feat: analyze repo

.editorconfig

@@ -0,0 +1,3 @@
+[*.{js,ts}]
+indent_style = space
+indent_size = 2

apps/backend/package.json

@@ -13,6 +13,7 @@
   "dependencies": {
     "@clickhouse/client": "1.11.2",
     "@fastify/cors": "11.0.1",
+    "@fastify/rate-limit": "10.3.0",
     "@founderpath/kysely-clickhouse": "1.7.0",
     "@octokit/rest": "22.0.0",
     "@specfy/stack-analyser": "1.27.2",
@@ -38,4 +39,4 @@
     "tsx": "4.20.3",
     "typescript": "5.8.3"
   }
-}
+}

apps/backend/src/app.ts

@@ -1,5 +1,6 @@
 /* eslint-disable @typescript-eslint/max-params */
 import cors from '@fastify/cors';
+import rateLimit from '@fastify/rate-limit';
 
 import { routes } from './routes/index.js';
 import { notFound, serverError } from './utils/apiErrors.js';
@@ -43,6 +44,16 @@ export default async function createApp(
     return notFound(res, `${req.method} ${req.url}`);
   });
 
+  await f.register(rateLimit, {
+    max: 100,
+    timeWindow: 60_000,
+    errorResponseBuilder: () => {
+      return {
+        error: { code: 'rate_limit_exceeded', status: 429 },
+      };
+    },
+  });
+
   f.removeAllContentTypeParsers();
   f.addContentTypeParser(
     'application/json',

apps/backend/src/db/migrationsDb/20250607000000_repositories_privates.ts

@@ -0,0 +1,25 @@
+import { sql } from 'kysely';
+
+import type { Database } from '../types.db.js';
+import type { Kysely } from 'kysely';
+
+export async function up(db: Kysely<Database>): Promise<void> {
+  await sql`
+     CREATE TABLE "repositories_analysis" (
+      id UUID DEFAULT gen_random_uuid(),
+      "repository_id" UUID NOT NULL,
+      "analysis" json NOT NULL DEFAULT '{}',
+      "last_manual_at" timestamptz NOT NULL DEFAULT CURRENT_TIMESTAMP,
+      PRIMARY KEY ("id")
+    )
+  `.execute(db);
+
+  await sql`
+    CREATE INDEX "idx_repositories_analysis_repository_id" ON "repositories_analysis" USING BTREE ("repository_id");
+  `.execute(db);
+
+  await sql`
+    ALTER TABLE "repositories"
+    ADD COLUMN "private" bool NOT NULL DEFAULT 'false';
+  `.execute(db);
+}

apps/backend/src/db/types.db.ts

@@ -1,5 +1,6 @@
 import type { AllowedLicensesLowercase } from '../types/stack.js';
-import type { ColumnType, Insertable, Selectable, Transaction, Updateable } from 'kysely';
+import type { AnalyserJson } from '@specfy/stack-analyser';
+import type { ColumnType, Insertable, Kysely, Selectable, Transaction, Updateable } from 'kysely';
 
 export type Timestamp = ColumnType<Date, Date, Date | string>;
 export type CreatedAt = ColumnType<string, Date | undefined, never>;
@@ -27,12 +28,24 @@ export interface RepositoriesTable {
   description: string;
   forks: number;
   repo_created_at: Timestamp;
+  private: boolean;
 }
 
 export type RepositoryRow = Selectable<RepositoriesTable>;
 export type RepositoryInsert = Insertable<RepositoriesTable>;
 export type RepositoryUpdate = Updateable<RepositoriesTable>;
 
+export interface RepositoriesAnalysisTable {
+  id: ColumnType<string, never, never>;
+  repository_id: string;
+  analysis: AnalyserJson;
+  last_manual_at: Timestamp;
+}
+
+export type RepositoryAnalysisRow = Selectable<RepositoriesAnalysisTable>;
+export type RepositoryAnalysisInsert = Insertable<RepositoriesAnalysisTable>;
+export type RepositoryAnalysisUpdate = Updateable<RepositoriesAnalysisTable>;
+
 export interface ProgressTable {
   date_week: string;
   progress: string;
@@ -91,8 +104,9 @@ export interface Database {
   progress: ProgressTable;
   licenses_info: LicensesInfoTable;
   repositories: RepositoriesTable;
+  repositories_analysis: RepositoriesAnalysisTable;
   cache: CacheTable;
   posts: PostsTable;
 }
 
-export type TX = Transaction<Database>;
+export type TX = Kysely<Database> | Transaction<Database>;

apps/backend/src/models/repositoriesAnalysis.ts

@@ -0,0 +1,69 @@
+import type {
+  RepositoryAnalysisInsert,
+  RepositoryAnalysisRow,
+  RepositoryAnalysisUpdate,
+  TX,
+} from '../db/types.db.js';
+
+export async function createRepositoryAnalysis(
+  trx: TX,
+  input: RepositoryAnalysisInsert
+): Promise<RepositoryAnalysisRow> {
+  return await trx
+    .insertInto('repositories_analysis')
+    .values(input)
+    .returningAll()
+    .executeTakeFirstOrThrow();
+}
+
+export async function getRepositoryAnalysis(
+  trx: TX,
+  repositoryId: string
+): Promise<RepositoryAnalysisRow | undefined> {
+  const row = await trx
+    .selectFrom('repositories_analysis')
+    .selectAll()
+    .where('repository_id', '=', repositoryId)
+    .executeTakeFirst();
+
+  return row;
+}
+
+export async function updateRepositoryAnalysis({
+  trx,
+  id,
+  input,
+}: {
+  trx: TX;
+  id: string;
+  input: RepositoryAnalysisUpdate;
+}): Promise<void> {
+  await trx
+    .updateTable('repositories_analysis')
+    .set({
+      analysis: input.analysis,
+    })
+    .where('id', '=', id)
+    .execute();
+}
+
+export async function upsertRepositoryAnalysis(
+  trx: TX,
+  repo: RepositoryAnalysisInsert
+): Promise<void> {
+  const row = await getRepositoryAnalysis(trx, repo.repository_id);
+
+  if (row) {
+    await trx
+      .updateTable('repositories_analysis')
+      .set({
+        analysis: repo.analysis,
+      })
+      .where('repository_id', '=', repo.repository_id)
+      .execute();
+
+    return;
+  }
+
+  await trx.insertInto('repositories_analysis').values(repo).execute();
+}

apps/backend/src/processor/analyzer.ts

@@ -11,7 +11,9 @@ import { $ } from 'execa';
 import { createLicenses, getLicensesByRepo } from '../models/licenses.js';
 import { getActiveWeek } from '../models/progress.js';
 import { updateRepository } from '../models/repositories.js';
+import { upsertRepositoryAnalysis } from '../models/repositoriesAnalysis.js';
 import { createTechnologies, getTechnologiesByRepo } from '../models/technologies.js';
+import { cleanAnalysis } from '../utils/analyzer.js';
 import { formatToClickhouseDatetime } from '../utils/date.js';
 import { envs } from '../utils/env.js';
 import { octokit } from '../utils/github.js';
@@ -71,7 +73,10 @@ export async function getPreviousAnalyzeIfStale(
   return { techs, licenses };
 }
 
-export async function analyze(repo: RepositoryRow, logger: Logger): Promise<Payload> {
+export async function analyze(
+  repo: Pick<RepositoryRow, 'branch' | 'name' | 'org'>,
+  logger: Logger
+): Promise<Payload> {
   const fullName = `${repo.org}/${repo.name}`;
   const dir = path.join(os.tmpdir(), 'getstack', repo.org, repo.name);
 
@@ -170,6 +175,11 @@ export async function saveAnalysis({
       last_analyzed_at: formatToClickhouseDatetime(new Date()),
     },
   });
+  await upsertRepositoryAnalysis(trx, {
+    repository_id: repo.id,
+    analysis: cleanAnalysis(res.toJson()),
+    last_manual_at: new Date(),
+  });
 }
 
 export async function savePreviousIfStale({

apps/backend/src/processor/cronList.ts

@@ -8,9 +8,9 @@ import {
   upsertRepository,
 } from '../models/repositories.js';
 import { algolia } from '../utils/algolia.js';
-import { formatToClickhouseDatetime, formatToDate, formatToYearWeek } from '../utils/date.js';
+import { formatToDate, formatToYearWeek } from '../utils/date.js';
 import { envs } from '../utils/env.js';
-import { octokit } from '../utils/github.js';
+import { githubToRepo, octokit } from '../utils/github.js';
 import { defaultLogger } from '../utils/logger.js';
 import { wait } from '../utils/wait.js';
 
@@ -120,29 +120,8 @@ export async function refreshOne(
 ): Promise<void> {
   const [org, name] = repo.full_name.split('/') as [string, string];
   const filtered = filter(repo);
-  await upsertRepository({
-    github_id: String(repo.id),
-    org,
-    name,
-    branch: repo.default_branch,
-    stars: repo.stargazers_count,
-    url: repo.html_url,
-    ignored: filtered === false ? 0 : 1,
-    ignored_reason: filtered === false ? 'ok' : filtered,
-    errored: 0,
-    last_fetched_at: formatToClickhouseDatetime(new Date('1970-01-01T00:00:00.000')),
-    last_analyzed_at: formatToClickhouseDatetime(new Date()),
-    size: repo.size,
-    avatar_url: repo.owner?.avatar_url || '',
-    homepage_url: repo.homepage
-      ? repo.homepage.startsWith('https:/')
-        ? repo.homepage
-        : `https://${repo.homepage}`
-      : '',
-    description: repo.description || '',
-    forks: repo.forks_count,
-    repo_created_at: formatToClickhouseDatetime(new Date(repo.created_at)),
-  });
+  await upsertRepository(githubToRepo(repo, filtered));
+
   await updateClickhouseRepository({
     id: String(repo.id),
     org,

apps/backend/src/routes/v1/categories/$name/getCategory.ts

@@ -1,4 +1,4 @@
-import { z } from 'zod';
+import * as z from 'zod';
 
 import { getOrCache } from '../../../../models/cache.js';
 import { getActiveWeek } from '../../../../models/progress.js';

apps/backend/src/routes/v1/categories/$name/leaderboard/getLeaderboard.ts

@@ -1,4 +1,4 @@
-import { z } from 'zod';
+import * as z from 'zod';
 
 import { getOrCache } from '../../../../../models/cache.js';
 import { getActiveWeek } from '../../../../../models/progress.js';

apps/backend/src/routes/v1/licenses/$license/getLicense.ts

@@ -1,4 +1,4 @@
-import { z } from 'zod';
+import * as z from 'zod';
 
 import { getOrCache } from '../../../../models/cache.js';
 import {
@@ -53,8 +53,8 @@ export const getApiLicense: FastifyPluginCallback = (fastify: FastifyInstance) =
     const repos =
       topRepos.length > 0
         ? await getRepositories({
-            ids: topRepos.map((row) => row.id),
-          })
+          ids: topRepos.map((row) => row.id),
+        })
         : [];
 
     reply.status(200).send({

apps/backend/src/routes/v1/newsletter/postSubscribe.ts

@@ -1,4 +1,4 @@
-import { z } from 'zod';
+import * as z from 'zod';
 
 import { serverError } from '../../../utils/apiErrors.js';
 import { envs } from '../../../utils/env.js';

apps/backend/src/routes/v1/posts/$id/getPost.ts

@@ -1,4 +1,4 @@
-import { z } from 'zod';
+import * as z from 'zod';
 
 import { db } from '../../../../db/client.js';
 import { notFound } from '../../../../utils/apiErrors.js';

apps/backend/src/routes/v1/repositories/$org/$name/getImage.ts

@@ -1,6 +1,6 @@
 import { listIndexed } from '@specfy/stack-analyser/dist/register.js';
 import sharp from 'sharp';
-import { z } from 'zod';
+import * as z from 'zod';
 
 import { getOrCache } from '../../../../../models/cache.js';
 import { getActiveWeek } from '../../../../../models/progress.js';
@@ -98,9 +98,9 @@ export const getRepositoryImage: FastifyPluginCallback = (fastify: FastifyInstan
     const tech =
       repo.ignored === 0
         ? await getOrCache({
-            keys: ['getTechnologiesByRepo', repo.id, weeks.currentWeek],
-            fn: () => getTechnologiesByRepo(repo, weeks.currentWeek),
-          })
+          keys: ['getTechnologiesByRepo', repo.id, weeks.currentWeek],
+          fn: () => getTechnologiesByRepo(repo, weeks.currentWeek),
+        })
         : [];
 
     // const stars = formatQuantity(repo.stars);

apps/backend/src/routes/v1/repositories/$org/$name/getRepository.ts

@@ -1,4 +1,4 @@
-import { z } from 'zod';
+import * as z from 'zod';
 
 import { getOrCache } from '../../../../../models/cache.js';
 import { getLicensesByRepo } from '../../../../../models/licenses.js';