diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..4e7b25e --- /dev/null +++ b/.editorconfig @@ -0,0 +1,3 @@ +[*.{js,ts}] +indent_style = space +indent_size = 2 \ No newline at end of file diff --git a/apps/backend/package.json b/apps/backend/package.json index a899069..22475d2 100644 --- a/apps/backend/package.json +++ b/apps/backend/package.json @@ -13,6 +13,7 @@ "dependencies": { "@clickhouse/client": "1.11.2", "@fastify/cors": "11.0.1", + "@fastify/rate-limit": "10.3.0", "@founderpath/kysely-clickhouse": "1.7.0", "@octokit/rest": "22.0.0", "@specfy/stack-analyser": "1.27.2", @@ -38,4 +39,4 @@ "tsx": "4.20.3", "typescript": "5.8.3" } -} \ No newline at end of file +} diff --git a/apps/backend/src/app.ts b/apps/backend/src/app.ts index 143c945..bf453a5 100644 --- a/apps/backend/src/app.ts +++ b/apps/backend/src/app.ts @@ -1,5 +1,6 @@ /* eslint-disable @typescript-eslint/max-params */ import cors from '@fastify/cors'; +import rateLimit from '@fastify/rate-limit'; import { routes } from './routes/index.js'; import { notFound, serverError } from './utils/apiErrors.js'; @@ -43,6 +44,16 @@ export default async function createApp( return notFound(res, `${req.method} ${req.url}`); }); + await f.register(rateLimit, { + max: 100, + timeWindow: 60_000, + errorResponseBuilder: () => { + return { + error: { code: 'rate_limit_exceeded', status: 429 }, + }; + }, + }); + f.removeAllContentTypeParsers(); f.addContentTypeParser( 'application/json', diff --git a/apps/backend/src/db/migrationsDb/20250607000000_repositories_privates.ts b/apps/backend/src/db/migrationsDb/20250607000000_repositories_privates.ts new file mode 100644 index 0000000..b1b7ae2 --- /dev/null +++ b/apps/backend/src/db/migrationsDb/20250607000000_repositories_privates.ts @@ -0,0 +1,25 @@ +import { sql } from 'kysely'; + +import type { Database } from '../types.db.js'; +import type { Kysely } from 'kysely'; + +export async function up(db: Kysely): Promise { + await sql` + CREATE TABLE "repositories_analysis" ( + id UUID DEFAULT gen_random_uuid(), + "repository_id" UUID NOT NULL, + "analysis" json NOT NULL DEFAULT '{}', + "last_manual_at" timestamptz NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY ("id") + ) + `.execute(db); + + await sql` + CREATE INDEX "idx_repositories_analysis_repository_id" ON "repositories_analysis" USING BTREE ("repository_id"); + `.execute(db); + + await sql` + ALTER TABLE "repositories" + ADD COLUMN "private" bool NOT NULL DEFAULT 'false'; + `.execute(db); +} diff --git a/apps/backend/src/db/types.db.ts b/apps/backend/src/db/types.db.ts index 86dfde0..af1302b 100644 --- a/apps/backend/src/db/types.db.ts +++ b/apps/backend/src/db/types.db.ts @@ -1,5 +1,6 @@ import type { AllowedLicensesLowercase } from '../types/stack.js'; -import type { ColumnType, Insertable, Selectable, Transaction, Updateable } from 'kysely'; +import type { AnalyserJson } from '@specfy/stack-analyser'; +import type { ColumnType, Insertable, Kysely, Selectable, Transaction, Updateable } from 'kysely'; export type Timestamp = ColumnType; export type CreatedAt = ColumnType; @@ -27,12 +28,24 @@ export interface RepositoriesTable { description: string; forks: number; repo_created_at: Timestamp; + private: boolean; } export type RepositoryRow = Selectable; export type RepositoryInsert = Insertable; export type RepositoryUpdate = Updateable; +export interface RepositoriesAnalysisTable { + id: ColumnType; + repository_id: string; + analysis: AnalyserJson; + last_manual_at: Timestamp; +} + +export type RepositoryAnalysisRow = Selectable; +export type RepositoryAnalysisInsert = Insertable; +export type RepositoryAnalysisUpdate = Updateable; + export interface ProgressTable { date_week: string; progress: string; @@ -91,8 +104,9 @@ export interface Database { progress: ProgressTable; licenses_info: LicensesInfoTable; repositories: RepositoriesTable; + repositories_analysis: RepositoriesAnalysisTable; cache: CacheTable; posts: PostsTable; } -export type TX = Transaction; +export type TX = Kysely | Transaction; diff --git a/apps/backend/src/models/repositoriesAnalysis.ts b/apps/backend/src/models/repositoriesAnalysis.ts new file mode 100644 index 0000000..336bf88 --- /dev/null +++ b/apps/backend/src/models/repositoriesAnalysis.ts @@ -0,0 +1,69 @@ +import type { + RepositoryAnalysisInsert, + RepositoryAnalysisRow, + RepositoryAnalysisUpdate, + TX, +} from '../db/types.db.js'; + +export async function createRepositoryAnalysis( + trx: TX, + input: RepositoryAnalysisInsert +): Promise { + return await trx + .insertInto('repositories_analysis') + .values(input) + .returningAll() + .executeTakeFirstOrThrow(); +} + +export async function getRepositoryAnalysis( + trx: TX, + repositoryId: string +): Promise { + const row = await trx + .selectFrom('repositories_analysis') + .selectAll() + .where('repository_id', '=', repositoryId) + .executeTakeFirst(); + + return row; +} + +export async function updateRepositoryAnalysis({ + trx, + id, + input, +}: { + trx: TX; + id: string; + input: RepositoryAnalysisUpdate; +}): Promise { + await trx + .updateTable('repositories_analysis') + .set({ + analysis: input.analysis, + }) + .where('id', '=', id) + .execute(); +} + +export async function upsertRepositoryAnalysis( + trx: TX, + repo: RepositoryAnalysisInsert +): Promise { + const row = await getRepositoryAnalysis(trx, repo.repository_id); + + if (row) { + await trx + .updateTable('repositories_analysis') + .set({ + analysis: repo.analysis, + }) + .where('repository_id', '=', repo.repository_id) + .execute(); + + return; + } + + await trx.insertInto('repositories_analysis').values(repo).execute(); +} diff --git a/apps/backend/src/processor/analyzer.ts b/apps/backend/src/processor/analyzer.ts index 7729d40..86e2e7a 100644 --- a/apps/backend/src/processor/analyzer.ts +++ b/apps/backend/src/processor/analyzer.ts @@ -11,7 +11,9 @@ import { $ } from 'execa'; import { createLicenses, getLicensesByRepo } from '../models/licenses.js'; import { getActiveWeek } from '../models/progress.js'; import { updateRepository } from '../models/repositories.js'; +import { upsertRepositoryAnalysis } from '../models/repositoriesAnalysis.js'; import { createTechnologies, getTechnologiesByRepo } from '../models/technologies.js'; +import { cleanAnalysis } from '../utils/analyzer.js'; import { formatToClickhouseDatetime } from '../utils/date.js'; import { envs } from '../utils/env.js'; import { octokit } from '../utils/github.js'; @@ -71,7 +73,10 @@ export async function getPreviousAnalyzeIfStale( return { techs, licenses }; } -export async function analyze(repo: RepositoryRow, logger: Logger): Promise { +export async function analyze( + repo: Pick, + logger: Logger +): Promise { const fullName = `${repo.org}/${repo.name}`; const dir = path.join(os.tmpdir(), 'getstack', repo.org, repo.name); @@ -170,6 +175,11 @@ export async function saveAnalysis({ last_analyzed_at: formatToClickhouseDatetime(new Date()), }, }); + await upsertRepositoryAnalysis(trx, { + repository_id: repo.id, + analysis: cleanAnalysis(res.toJson()), + last_manual_at: new Date(), + }); } export async function savePreviousIfStale({ diff --git a/apps/backend/src/processor/cronList.ts b/apps/backend/src/processor/cronList.ts index 2e6ce54..d112a09 100644 --- a/apps/backend/src/processor/cronList.ts +++ b/apps/backend/src/processor/cronList.ts @@ -8,9 +8,9 @@ import { upsertRepository, } from '../models/repositories.js'; import { algolia } from '../utils/algolia.js'; -import { formatToClickhouseDatetime, formatToDate, formatToYearWeek } from '../utils/date.js'; +import { formatToDate, formatToYearWeek } from '../utils/date.js'; import { envs } from '../utils/env.js'; -import { octokit } from '../utils/github.js'; +import { githubToRepo, octokit } from '../utils/github.js'; import { defaultLogger } from '../utils/logger.js'; import { wait } from '../utils/wait.js'; @@ -120,29 +120,8 @@ export async function refreshOne( ): Promise { const [org, name] = repo.full_name.split('/') as [string, string]; const filtered = filter(repo); - await upsertRepository({ - github_id: String(repo.id), - org, - name, - branch: repo.default_branch, - stars: repo.stargazers_count, - url: repo.html_url, - ignored: filtered === false ? 0 : 1, - ignored_reason: filtered === false ? 'ok' : filtered, - errored: 0, - last_fetched_at: formatToClickhouseDatetime(new Date('1970-01-01T00:00:00.000')), - last_analyzed_at: formatToClickhouseDatetime(new Date()), - size: repo.size, - avatar_url: repo.owner?.avatar_url || '', - homepage_url: repo.homepage - ? repo.homepage.startsWith('https:/') - ? repo.homepage - : `https://${repo.homepage}` - : '', - description: repo.description || '', - forks: repo.forks_count, - repo_created_at: formatToClickhouseDatetime(new Date(repo.created_at)), - }); + await upsertRepository(githubToRepo(repo, filtered)); + await updateClickhouseRepository({ id: String(repo.id), org, diff --git a/apps/backend/src/routes/v1/categories/$name/getCategory.ts b/apps/backend/src/routes/v1/categories/$name/getCategory.ts index e4a3f16..d39c7b0 100644 --- a/apps/backend/src/routes/v1/categories/$name/getCategory.ts +++ b/apps/backend/src/routes/v1/categories/$name/getCategory.ts @@ -1,4 +1,4 @@ -import { z } from 'zod'; +import * as z from 'zod'; import { getOrCache } from '../../../../models/cache.js'; import { getActiveWeek } from '../../../../models/progress.js'; diff --git a/apps/backend/src/routes/v1/categories/$name/leaderboard/getLeaderboard.ts b/apps/backend/src/routes/v1/categories/$name/leaderboard/getLeaderboard.ts index e94de7d..8813eb7 100644 --- a/apps/backend/src/routes/v1/categories/$name/leaderboard/getLeaderboard.ts +++ b/apps/backend/src/routes/v1/categories/$name/leaderboard/getLeaderboard.ts @@ -1,4 +1,4 @@ -import { z } from 'zod'; +import * as z from 'zod'; import { getOrCache } from '../../../../../models/cache.js'; import { getActiveWeek } from '../../../../../models/progress.js'; diff --git a/apps/backend/src/routes/v1/licenses/$license/getLicense.ts b/apps/backend/src/routes/v1/licenses/$license/getLicense.ts index 9d465d4..a43fe20 100644 --- a/apps/backend/src/routes/v1/licenses/$license/getLicense.ts +++ b/apps/backend/src/routes/v1/licenses/$license/getLicense.ts @@ -1,4 +1,4 @@ -import { z } from 'zod'; +import * as z from 'zod'; import { getOrCache } from '../../../../models/cache.js'; import { @@ -53,8 +53,8 @@ export const getApiLicense: FastifyPluginCallback = (fastify: FastifyInstance) = const repos = topRepos.length > 0 ? await getRepositories({ - ids: topRepos.map((row) => row.id), - }) + ids: topRepos.map((row) => row.id), + }) : []; reply.status(200).send({ diff --git a/apps/backend/src/routes/v1/newsletter/postSubscribe.ts b/apps/backend/src/routes/v1/newsletter/postSubscribe.ts index 1674210..94e9b82 100644 --- a/apps/backend/src/routes/v1/newsletter/postSubscribe.ts +++ b/apps/backend/src/routes/v1/newsletter/postSubscribe.ts @@ -1,4 +1,4 @@ -import { z } from 'zod'; +import * as z from 'zod'; import { serverError } from '../../../utils/apiErrors.js'; import { envs } from '../../../utils/env.js'; diff --git a/apps/backend/src/routes/v1/posts/$id/getPost.ts b/apps/backend/src/routes/v1/posts/$id/getPost.ts index 35bca27..18039ea 100644 --- a/apps/backend/src/routes/v1/posts/$id/getPost.ts +++ b/apps/backend/src/routes/v1/posts/$id/getPost.ts @@ -1,4 +1,4 @@ -import { z } from 'zod'; +import * as z from 'zod'; import { db } from '../../../../db/client.js'; import { notFound } from '../../../../utils/apiErrors.js'; diff --git a/apps/backend/src/routes/v1/repositories/$org/$name/getImage.ts b/apps/backend/src/routes/v1/repositories/$org/$name/getImage.ts index 22291fb..07cda4e 100644 --- a/apps/backend/src/routes/v1/repositories/$org/$name/getImage.ts +++ b/apps/backend/src/routes/v1/repositories/$org/$name/getImage.ts @@ -1,6 +1,6 @@ import { listIndexed } from '@specfy/stack-analyser/dist/register.js'; import sharp from 'sharp'; -import { z } from 'zod'; +import * as z from 'zod'; import { getOrCache } from '../../../../../models/cache.js'; import { getActiveWeek } from '../../../../../models/progress.js'; @@ -98,9 +98,9 @@ export const getRepositoryImage: FastifyPluginCallback = (fastify: FastifyInstan const tech = repo.ignored === 0 ? await getOrCache({ - keys: ['getTechnologiesByRepo', repo.id, weeks.currentWeek], - fn: () => getTechnologiesByRepo(repo, weeks.currentWeek), - }) + keys: ['getTechnologiesByRepo', repo.id, weeks.currentWeek], + fn: () => getTechnologiesByRepo(repo, weeks.currentWeek), + }) : []; // const stars = formatQuantity(repo.stars); diff --git a/apps/backend/src/routes/v1/repositories/$org/$name/getRepository.ts b/apps/backend/src/routes/v1/repositories/$org/$name/getRepository.ts index 4ca6869..9ac24d3 100644 --- a/apps/backend/src/routes/v1/repositories/$org/$name/getRepository.ts +++ b/apps/backend/src/routes/v1/repositories/$org/$name/getRepository.ts @@ -1,4 +1,4 @@ -import { z } from 'zod'; +import * as z from 'zod'; import { getOrCache } from '../../../../../models/cache.js'; import { getLicensesByRepo } from '../../../../../models/licenses.js'; diff --git a/apps/backend/src/routes/v1/repositories/$org/$name/postAnalyzeOne.ts b/apps/backend/src/routes/v1/repositories/$org/$name/postAnalyzeOne.ts index 66f1162..eba3ef2 100644 --- a/apps/backend/src/routes/v1/repositories/$org/$name/postAnalyzeOne.ts +++ b/apps/backend/src/routes/v1/repositories/$org/$name/postAnalyzeOne.ts @@ -1,12 +1,21 @@ -import { z } from 'zod'; +import * as z from 'zod'; import { db } from '../../../../../db/client.js'; -import { getRepository } from '../../../../../models/repositories.js'; -import { analyze, saveAnalysis, savePreviousIfStale } from '../../../../../processor/analyzer.js'; +import { createRepository, getRepository } from '../../../../../models/repositories.js'; +import { + createRepositoryAnalysis, + getRepositoryAnalysis, + updateRepositoryAnalysis, +} from '../../../../../models/repositoriesAnalysis.js'; +import { analyze } from '../../../../../processor/analyzer.js'; +import { cleanAnalysis } from '../../../../../utils/analyzer.js'; import { notFound } from '../../../../../utils/apiErrors.js'; -import { formatToYearWeek } from '../../../../../utils/date.js'; +import { envs } from '../../../../../utils/env.js'; +import { githubToRepo, octokit } from '../../../../../utils/github.js'; import { defaultLogger } from '../../../../../utils/logger.js'; +import type { APIPostAnalyzeOne } from '../../../../../types/endpoint.js'; +import type { RestEndpointMethodTypes } from '@octokit/rest'; import type { FastifyInstance, FastifyPluginCallback } from 'fastify'; const logger = defaultLogger.child({ svc: 'api.analyze' }); @@ -16,42 +25,81 @@ const schemaParams = z.object({ name: z.string().max(255), }); +const analysisTTL = 1000 * 60 * 60 * 24; + export const postAnalyzeOne: FastifyPluginCallback = (fastify: FastifyInstance) => { - fastify.post('/repositories/:org/:name/analyze', async (req, reply) => { - const valParams = schemaParams.safeParse(req.params); - if (valParams.error) { - return reply.status(400).send({ error: { code: 'invalid_params', status: 400 } }); - } + fastify.post( + '/repositories/:org/:name/analyze', + { + config: { + rateLimit: { max: 5, timeWindow: 60_000 }, + }, + }, + async (req, reply) => { + const valParams = schemaParams.safeParse(req.params); + if (valParams.error) { + return reply.status(400).send({ error: { code: '400_invalid_params', status: 400 } }); + } - const params = valParams.data; + const params = valParams.data; - logger.info(`Processing one ${params.org}/${params.name}`); + // First we fetch to know if it exists, if it's a private repo, etc. + let repoGithub: RestEndpointMethodTypes['repos']['get']['response']['data']; + try { + const res = await octokit.rest.repos.get({ owner: params.org, repo: params.name }); + repoGithub = res.data; + } catch (err) { + console.error(err); + return notFound(reply); + } - const repo = await getRepository(params); - if (!repo) { - return notFound(reply); - } + let repo = await getRepository(params); + if (!repo) { + // Ignore manual repo by default + const ignored = repoGithub.size > envs.ANALYZE_MAX_SIZE ? 'too_big' : 'manual'; + repo = await createRepository(githubToRepo(repoGithub, ignored)); + } else if (repo.ignored && repo.ignored_reason !== 'manual') { + return reply.status(200).send({ success: true, data: { repo, analysis: null } }); + } + + const repoAnalysis = await getRepositoryAnalysis(db, repo.id); + // Recently analyzed + if (repoAnalysis && repoAnalysis.last_manual_at.getTime() + analysisTTL > Date.now()) { + return reply + .status(200) + .send({ success: true, data: { repo, analysis: repoAnalysis.analysis } }); + } + + logger.info(`Processing one ${params.org}/${params.name}`); + const analysis = await analyze( + { + branch: repoGithub.default_branch, + name: repoGithub.name, + org: repoGithub.owner.login, + }, + logger + ); - const dateWeek = formatToYearWeek(new Date()); - try { await db.transaction().execute(async (trx) => { - const withPrevious = await savePreviousIfStale({ trx, repo, dateWeek }); - if (withPrevious) { - logger.info(`With previous`); + if (repoAnalysis) { + await updateRepositoryAnalysis({ + trx, + id: repoAnalysis.id, + input: { analysis: analysis.toJson(), last_manual_at: new Date() }, + }); } else { - const res = await analyze(repo, logger); - await saveAnalysis({ trx, repo, res, dateWeek }); - logger.info(`Done`); + await createRepositoryAnalysis(trx, { + repository_id: repo.id, + analysis: cleanAnalysis(analysis.toJson()), + last_manual_at: new Date(), + }); } }); - } catch (err) { - logger.error(`Failed to process`, err); - reply.status(500).send({ error: { code: 'failed_to_process', status: 500 } }); - return; - } - reply.status(200).send({ - success: true, - }); - }); + reply.status(200).send({ + success: true, + data: { repo, analysis: analysis.toJson() }, + }); + } + ); }; diff --git a/apps/backend/src/routes/v1/repositories/$org/$name/postRefresh.ts b/apps/backend/src/routes/v1/repositories/$org/$name/postRefresh.ts index b2caa6e..d1e7e6d 100644 --- a/apps/backend/src/routes/v1/repositories/$org/$name/postRefresh.ts +++ b/apps/backend/src/routes/v1/repositories/$org/$name/postRefresh.ts @@ -1,4 +1,4 @@ -import { z } from 'zod'; +import * as z from 'zod'; import { getRepository } from '../../../../../models/repositories.js'; import { refreshOne } from '../../../../../processor/cronList.js'; diff --git a/apps/backend/src/routes/v1/technologies/$name/getTechnology.ts b/apps/backend/src/routes/v1/technologies/$name/getTechnology.ts index 7a7aba5..bca5683 100644 --- a/apps/backend/src/routes/v1/technologies/$name/getTechnology.ts +++ b/apps/backend/src/routes/v1/technologies/$name/getTechnology.ts @@ -1,5 +1,5 @@ import { listIndexed } from '@specfy/stack-analyser/dist/common/techs.generated.js'; -import { z } from 'zod'; +import * as z from 'zod'; import { getOrCache } from '../../../../models/cache.js'; import { getActiveWeek } from '../../../../models/progress.js'; @@ -47,8 +47,8 @@ export const getTechnology: FastifyPluginCallback = (fastify: FastifyInstance) = const repos = topRepos.length > 0 ? await getRepositories({ - ids: topRepos.map((row) => row.id), - }) + ids: topRepos.map((row) => row.id), + }) : []; reply.status(200).send({ success: true, diff --git a/apps/backend/src/routes/v1/technologies/$name/related/getRelated.ts b/apps/backend/src/routes/v1/technologies/$name/related/getRelated.ts index 98a49b6..ec25f52 100644 --- a/apps/backend/src/routes/v1/technologies/$name/related/getRelated.ts +++ b/apps/backend/src/routes/v1/technologies/$name/related/getRelated.ts @@ -1,4 +1,4 @@ -import { z } from 'zod'; +import * as z from 'zod'; import { getOrCache } from '../../../../../models/cache.js'; import { getActiveWeek } from '../../../../../models/progress.js'; diff --git a/apps/backend/src/types/endpoint.ts b/apps/backend/src/types/endpoint.ts index b3ccdd0..0619f1a 100644 --- a/apps/backend/src/types/endpoint.ts +++ b/apps/backend/src/types/endpoint.ts @@ -1,4 +1,4 @@ -import type { Endpoint } from './api.js'; +import type { ApiError, Endpoint } from './api.js'; import type { LicenseRow, LicensesWeeklyRow, @@ -7,7 +7,7 @@ import type { TechnologyWeeklyRow, } from '../db/types.clickhouse.js'; import type { LicensesInfoTableRow, PostsRow, RepositoryRow } from '../db/types.db.js'; -import type { AllowedKeys, TechType } from '@specfy/stack-analyser'; +import type { AllowedKeys, AnalyserJson, TechType } from '@specfy/stack-analyser'; import type { AllowedLicenses } from '@specfy/stack-analyser/dist/types/licenses.js'; export interface TechnologyByCategoryByWeekWithTrend { @@ -177,3 +177,14 @@ export type APIGetPost = Endpoint<{ data: APIPost; }; }>; + +export type APIPostAnalyzeOne = Endpoint<{ + Path: '/1/repositories/:org/:name/analyze'; + Method: 'POST'; + Params: { org: string; name: string }; + Success: { + success: true; + data: { repo: RepositoryRow; analysis: AnalyserJson | null }; + }; + Error: ApiError<'failed_to_process'>; +}>; diff --git a/apps/backend/src/utils/analyzer.ts b/apps/backend/src/utils/analyzer.ts new file mode 100644 index 0000000..ad4ba59 --- /dev/null +++ b/apps/backend/src/utils/analyzer.ts @@ -0,0 +1,9 @@ +import type { AnalyserJson } from '@specfy/stack-analyser'; + +export function cleanAnalysis(analysis: AnalyserJson): AnalyserJson { + analysis.path = []; + for (const child of analysis.childs) { + child.path = []; + } + return analysis; +} diff --git a/apps/backend/src/utils/env.ts b/apps/backend/src/utils/env.ts index 86995df..d4a55b9 100644 --- a/apps/backend/src/utils/env.ts +++ b/apps/backend/src/utils/env.ts @@ -1,4 +1,4 @@ -import { z } from 'zod'; +import * as z from 'zod'; const bool = z .enum(['true', 'false', '']) diff --git a/apps/backend/src/utils/github.ts b/apps/backend/src/utils/github.ts index e8aaa4a..e864f5c 100644 --- a/apps/backend/src/utils/github.ts +++ b/apps/backend/src/utils/github.ts @@ -2,6 +2,45 @@ import { Octokit } from 'octokit'; import { envs } from './env.js'; +import type { RepositoryInsert } from '../db/types.db.js'; +import type { RestEndpointMethodTypes } from '@octokit/rest'; + export const octokit: Octokit = new Octokit({ auth: envs.GITHUB_TOKEN, }); + +export function fixGithubHomepageUrl(hp?: null | string): string { + if (!hp) { + return ''; + } + return hp.startsWith('https:/') ? hp : `https://${hp}`; +} + +export function githubToRepo( + repo: + | RestEndpointMethodTypes['repos']['get']['response']['data'] + | RestEndpointMethodTypes['search']['repos']['response']['data']['items'][0], + filtered: false | string +): RepositoryInsert { + const [org, name] = repo.full_name.split('/') as [string, string]; + return { + github_id: String(repo.id), + org, + name, + branch: repo.default_branch, + stars: repo.stargazers_count, + url: repo.html_url, + ignored: filtered === false ? 0 : 1, + ignored_reason: filtered === false ? 'ok' : filtered, + errored: 0, + last_fetched_at: new Date('1970-01-01T00:00:00.000'), + last_analyzed_at: new Date(), + size: repo.size, + avatar_url: repo.owner?.avatar_url || '', + homepage_url: fixGithubHomepageUrl(repo.homepage), + description: repo.description || '', + forks: repo.forks_count, + repo_created_at: new Date(repo.created_at), + private: repo.private, + }; +} diff --git a/apps/frontend/content-collections.ts b/apps/frontend/content-collections.ts index 3c49d2d..5ecd207 100644 --- a/apps/frontend/content-collections.ts +++ b/apps/frontend/content-collections.ts @@ -1,7 +1,7 @@ import { defineCollection, defineConfig } from '@content-collections/core'; import { compileMarkdown } from '@content-collections/markdown'; import { compileMDX } from '@content-collections/mdx'; -import { z } from 'zod'; +import * as z from 'zod'; const posts = defineCollection({ name: 'posts', diff --git a/apps/frontend/public/favicons/firecrawl.webp b/apps/frontend/public/favicons/firecrawl.webp index 3fb1d34..126e497 100644 Binary files a/apps/frontend/public/favicons/firecrawl.webp and b/apps/frontend/public/favicons/firecrawl.webp differ diff --git a/apps/frontend/src/api/useRepository.ts b/apps/frontend/src/api/useRepository.ts index f7fdbee..1f55408 100644 --- a/apps/frontend/src/api/useRepository.ts +++ b/apps/frontend/src/api/useRepository.ts @@ -1,5 +1,5 @@ /* eslint-disable @typescript-eslint/explicit-function-return-type */ -import { queryOptions, useQuery } from '@tanstack/react-query'; +import { queryOptions, useMutation, useQuery } from '@tanstack/react-query'; import { notFound } from '@tanstack/react-router'; import { ApiResError } from './api.js'; @@ -7,7 +7,7 @@ import { ALGOLIA_INDEX_NAME, API_URL } from '../lib/envs'; import { algolia } from '@/lib/algolia.js'; import type { AlgoliaRepositoryObject } from '@getstack/backend/src/types/algolia.js'; -import type { APIGetRepository } from '@getstack/backend/src/types/endpoint.js'; +import type { APIGetRepository, APIPostAnalyzeOne } from '@getstack/backend/src/types/endpoint.js'; export const useRepository = (opts: { org: string; name: string }) => { return useQuery(optionsGetRepository(opts)); @@ -50,3 +50,28 @@ export const useRepositorySearchAlgolia = ({ search }: { search: string }) => { }, }); }; + +export const useAnalyzeRepository = () => { + return useMutation({ + mutationFn: async (params) => { + const response = await fetch( + `${API_URL}/1/repositories/${params.org}/${params.name}/analyze`, + { + method: 'POST', + } + ); + + if (response.status === 404) { + // eslint-disable-next-line @typescript-eslint/only-throw-error + throw notFound(); + } + + const json = (await response.json()) as APIPostAnalyzeOne['Reply']; + if ('error' in json) { + throw new ApiResError(json); + } + + return json; + }, + }); +}; diff --git a/apps/frontend/src/components/Newsletter.tsx b/apps/frontend/src/components/Newsletter.tsx index a8e572a..fa59936 100644 --- a/apps/frontend/src/components/Newsletter.tsx +++ b/apps/frontend/src/components/Newsletter.tsx @@ -46,7 +46,7 @@ export const Newsletter: React.FC<{ title?: string }> = ({ title }) => { value={email} onChange={(e) => setEmail(e.target.value)} /> - diff --git a/apps/frontend/src/components/ui/input.tsx b/apps/frontend/src/components/ui/input.tsx index b34e014..06d7a54 100644 --- a/apps/frontend/src/components/ui/input.tsx +++ b/apps/frontend/src/components/ui/input.tsx @@ -8,7 +8,7 @@ function Input({ className, type, ...props }: React.ComponentProps<'input'>) { type={type} data-slot="input" className={cn( - 'file:text-foreground placeholder:text-muted-foreground selection:bg-primary selection:text-primary-foreground dark:bg-input/30 border-input flex h-9 w-full min-w-0 rounded-md border bg-transparent px-3 py-1 text-base shadow-xs transition-[color,box-shadow] outline-none file:inline-flex file:h-7 file:border-0 file:bg-transparent file:text-sm file:font-medium disabled:pointer-events-none disabled:cursor-not-allowed disabled:opacity-50 md:text-sm', + 'file:text-foreground placeholder:text-muted-foreground selection:bg-primary selection:text-primary-foreground dark:bg-input/30 border-input flex h-9 w-full min-w-0 rounded border bg-transparent px-3 py-1 text-base shadow-xs transition-[color,box-shadow] outline-none file:inline-flex file:h-7 file:border-0 file:bg-transparent file:text-sm file:font-medium disabled:pointer-events-none disabled:cursor-not-allowed disabled:opacity-50 md:text-sm', 'focus-visible:border-ring focus-visible:ring-ring/50 focus-visible:ring-[3px]', 'aria-invalid:ring-destructive/20 dark:aria-invalid:ring-destructive/40 aria-invalid:border-destructive', className diff --git a/apps/frontend/src/routeTree.gen.ts b/apps/frontend/src/routeTree.gen.ts index c5e3ef3..2d31b58 100644 --- a/apps/frontend/src/routeTree.gen.ts +++ b/apps/frontend/src/routeTree.gen.ts @@ -10,6 +10,7 @@ import { Route as rootRouteImport } from './routes/__root' import { Route as PrivateRouteImport } from './routes/private' +import { Route as AnalyzeRepositoryRouteImport } from './routes/analyze-repository' import { Route as AboutRouteImport } from './routes/about' import { Route as IndexRouteImport } from './routes/index' import { Route as LicensesIndexRouteImport } from './routes/licenses/index' @@ -25,6 +26,11 @@ const PrivateRoute = PrivateRouteImport.update({ path: '/private', getParentRoute: () => rootRouteImport, } as any) +const AnalyzeRepositoryRoute = AnalyzeRepositoryRouteImport.update({ + id: '/analyze-repository', + path: '/analyze-repository', + getParentRoute: () => rootRouteImport, +} as any) const AboutRoute = AboutRouteImport.update({ id: '/about', path: '/about', @@ -74,6 +80,7 @@ const OrgNameRoute = OrgNameRouteImport.update({ export interface FileRoutesByFullPath { '/': typeof IndexRoute '/about': typeof AboutRoute + '/analyze-repository': typeof AnalyzeRepositoryRoute '/private': typeof PrivateRoute '/$org/$name': typeof OrgNameRoute '/blog/$slug': typeof BlogSlugRoute @@ -86,6 +93,7 @@ export interface FileRoutesByFullPath { export interface FileRoutesByTo { '/': typeof IndexRoute '/about': typeof AboutRoute + '/analyze-repository': typeof AnalyzeRepositoryRoute '/private': typeof PrivateRoute '/$org/$name': typeof OrgNameRoute '/blog/$slug': typeof BlogSlugRoute @@ -99,6 +107,7 @@ export interface FileRoutesById { __root__: typeof rootRouteImport '/': typeof IndexRoute '/about': typeof AboutRoute + '/analyze-repository': typeof AnalyzeRepositoryRoute '/private': typeof PrivateRoute '/$org/$name': typeof OrgNameRoute '/blog/$slug': typeof BlogSlugRoute @@ -113,6 +122,7 @@ export interface FileRouteTypes { fullPaths: | '/' | '/about' + | '/analyze-repository' | '/private' | '/$org/$name' | '/blog/$slug' @@ -125,6 +135,7 @@ export interface FileRouteTypes { to: | '/' | '/about' + | '/analyze-repository' | '/private' | '/$org/$name' | '/blog/$slug' @@ -137,6 +148,7 @@ export interface FileRouteTypes { | '__root__' | '/' | '/about' + | '/analyze-repository' | '/private' | '/$org/$name' | '/blog/$slug' @@ -150,6 +162,7 @@ export interface FileRouteTypes { export interface RootRouteChildren { IndexRoute: typeof IndexRoute AboutRoute: typeof AboutRoute + AnalyzeRepositoryRoute: typeof AnalyzeRepositoryRoute PrivateRoute: typeof PrivateRoute OrgNameRoute: typeof OrgNameRoute BlogSlugRoute: typeof BlogSlugRoute @@ -169,6 +182,13 @@ declare module '@tanstack/react-router' { preLoaderRoute: typeof PrivateRouteImport parentRoute: typeof rootRouteImport } + '/analyze-repository': { + id: '/analyze-repository' + path: '/analyze-repository' + fullPath: '/analyze-repository' + preLoaderRoute: typeof AnalyzeRepositoryRouteImport + parentRoute: typeof rootRouteImport + } '/about': { id: '/about' path: '/about' @@ -238,6 +258,7 @@ declare module '@tanstack/react-router' { const rootRouteChildren: RootRouteChildren = { IndexRoute: IndexRoute, AboutRoute: AboutRoute, + AnalyzeRepositoryRoute: AnalyzeRepositoryRoute, PrivateRoute: PrivateRoute, OrgNameRoute: OrgNameRoute, BlogSlugRoute: BlogSlugRoute, diff --git a/apps/frontend/src/routes/analyze-repository.tsx b/apps/frontend/src/routes/analyze-repository.tsx new file mode 100644 index 0000000..4294dd9 --- /dev/null +++ b/apps/frontend/src/routes/analyze-repository.tsx @@ -0,0 +1,124 @@ +/* eslint-disable @typescript-eslint/no-misused-promises */ +import { IconLoader2 } from '@tabler/icons-react'; +import { Link, createFileRoute } from '@tanstack/react-router'; +import { useState } from 'react'; + +import { useAnalyzeRepository } from '@/api/useRepository'; +import { Button } from '@/components/ui/button'; +import { Input } from '@/components/ui/input'; +import { APP_URL } from '@/lib/envs'; +import { seo } from '@/lib/seo'; + +const highlights: { type: 'repo'; org: string; name: string }[] = [ + { type: 'repo', org: 'n8n-io', name: 'n8n' }, + { type: 'repo', org: 'NangoHQ', name: 'nango' }, + { type: 'repo', org: 'mendableai', name: 'firecrawl' }, +]; + +const RouteComponent: React.FC = () => { + const [repo, setRepo] = useState(''); + const [loading, setLoading] = useState(false); + const [error, setError] = useState(null); + const { data, mutateAsync: analyzeRepository } = useAnalyzeRepository(); + + const handleSubmit = async () => { + setLoading(true); + try { + const isUrl = new URL(repo); + if (isUrl.origin !== 'https://github.com') { + setLoading(false); + setError('Please enter a valid repository URL, like https://github.com/specfy/getstack'); + return; + } + + const [org, name] = isUrl.pathname.slice(1).split('/'); + if (!org || !name) { + setLoading(false); + setError('Please enter a valid repository URL, like https://github.com/specfy/getstack'); + return; + } + + setError(null); + await analyzeRepository({ org, name }); + } catch { + setError('An error occurred'); + return; + } finally { + setLoading(false); + } + }; + + return ( +
+

Analyzing any GitHub repository

+

+ Get the full tech stack of any GitHub repository in one click +

+
+
+ setRepo(e.target.value)} + disabled={loading} + /> + +
+ {error &&
{error}
} +
+
or try
+ {highlights.map((item) => { + return ( + + ); + })} +
+
+
+ ); +}; + +export const Route = createFileRoute('/analyze-repository')({ + head: () => { + const url = `${APP_URL}/analyze-repository`; + return { + meta: [ + ...seo({ + title: `Analyze your repository - getStack`, + description: `Analyze any GitHub repository and get its full tech stack in one click`, + url, + }), + ], + links: [{ rel: 'canonical', href: url }], + }; + }, + component: RouteComponent, +}); diff --git a/apps/frontend/src/routes/private.tsx b/apps/frontend/src/routes/private.tsx index 094d542..4c807fe 100644 --- a/apps/frontend/src/routes/private.tsx +++ b/apps/frontend/src/routes/private.tsx @@ -1,34 +1,7 @@ -import { createFileRoute } from '@tanstack/react-router'; - -import { Newsletter } from '@/components/Newsletter'; -import { APP_URL } from '@/lib/envs'; -import { seo } from '@/lib/seo'; - -const RouteComponent: React.FC = () => { - return ( -
-

Analyzing your private repo

-

Coming Soon, stay tuned!

-
- -
-
- ); -}; +import { createFileRoute, redirect } from '@tanstack/react-router'; export const Route = createFileRoute('/private')({ - head: () => { - const url = `${APP_URL}/private`; - return { - meta: [ - ...seo({ - title: `Analyze your repository - getStack`, - description: `Get the tech stack of any GitHub repository`, - url, - }), - ], - links: [{ rel: 'canonical', href: url }], - }; + loader: () => { + return redirect({ to: '/analyze-repository' }); }, - component: RouteComponent, }); diff --git a/package-lock.json b/package-lock.json index 45a4f96..4112872 100644 --- a/package-lock.json +++ b/package-lock.json @@ -24,6 +24,7 @@ "dependencies": { "@clickhouse/client": "1.11.2", "@fastify/cors": "11.0.1", + "@fastify/rate-limit": "10.3.0", "@founderpath/kysely-clickhouse": "1.7.0", "@octokit/rest": "22.0.0", "@specfy/stack-analyser": "1.27.2", @@ -2029,6 +2030,27 @@ "ipaddr.js": "^2.1.0" } }, + "node_modules/@fastify/rate-limit": { + "version": "10.3.0", + "resolved": "https://registry.npmjs.org/@fastify/rate-limit/-/rate-limit-10.3.0.tgz", + "integrity": "sha512-eIGkG9XKQs0nyynatApA3EVrojHOuq4l6fhB4eeCk4PIOeadvOJz9/4w3vGI44Go17uaXOWEcPkaD8kuKm7g6Q==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], + "license": "MIT", + "dependencies": { + "@lukeed/ms": "^2.0.2", + "fastify-plugin": "^5.0.0", + "toad-cache": "^3.7.0" + } + }, "node_modules/@floating-ui/core": { "version": "1.7.1", "resolved": "https://registry.npmjs.org/@floating-ui/core/-/core-1.7.1.tgz", @@ -2671,6 +2693,15 @@ "@jridgewell/sourcemap-codec": "^1.4.14" } }, + "node_modules/@lukeed/ms": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/@lukeed/ms/-/ms-2.0.2.tgz", + "integrity": "sha512-9I2Zn6+NJLfaGoz9jN3lpwDgAYvfGeNYdbAIjJOqzs4Tpc+VU3Jqq4IofSUBKajiDS8k9fZIg18/z13mpk1bsA==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, "node_modules/@mapbox/node-pre-gyp": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/@mapbox/node-pre-gyp/-/node-pre-gyp-2.0.0.tgz",