diff --git a/api/src/core/adapters/GitHub/api/repo.ts b/api/src/core/adapters/GitHub/api/repo.ts index 80c2f43c9..4b16a2b91 100644 --- a/api/src/core/adapters/GitHub/api/repo.ts +++ b/api/src/core/adapters/GitHub/api/repo.ts @@ -3,7 +3,7 @@ // SPDX-License-Identifier: MIT import { Octokit } from "@octokit/rest"; -import { Endpoints } from "@octokit/types"; +import { Endpoints, RequestParameters } from "@octokit/types"; export namespace GitHubAPI { export type Commit = Endpoints["GET /repos/{owner}/{repo}/commits/{ref}"]["response"]["data"]; @@ -16,6 +16,9 @@ export namespace GitHubAPI { export type SearchRepositoriesItem = Endpoints["GET /search/repositories"]["response"]["data"]["items"][0]; } +const GITHUB_API_API_TIMEOUT = 60 * 1000; +const GITHUB_API_LOG = "[GitHubAdapter]"; + const parseURL = (repoUrl: string | URL): { repo: string; owner: string } => { let repoUrlObj = typeof repoUrl === "string" ? URL.parse(repoUrl) : repoUrl; if (!repoUrlObj) throw Error("The provided URL could be parsed"); @@ -35,123 +38,74 @@ const parseURL = (repoUrl: string | URL): { repo: string; owner: string } => { export const gitHubEndpointMaker = (params?: { auth?: string }) => { const octokit = new Octokit(params?.auth ? { auth: params.auth } : {}); + const apiCall = async (url: string, options: RequestParameters) => { + try { + const resultApi = await octokit.request(url, options); + + if (resultApi.status === 429 || resultApi.status > 500) { + console.debug(`${GITHUB_API_LOG} Got ${resultApi.status} let's take a break for 60s`); + await new Promise(resolve => setTimeout(resolve, GITHUB_API_API_TIMEOUT)); + return apiCall(url, options); + } + + const remainingRequests = parseInt(resultApi.headers["x-ratelimit-remaining"] || "0"); + if (remainingRequests < 50) { + const now = new Date().valueOf(); + const resetTime = parseInt(resultApi.headers["x-ratelimit-reset"] || "0") * 1000; // Convert to ms + const timeToWait = resetTime - now; + console.debug( + `${GITHUB_API_LOG} Got ${resultApi.status} let's take a break for ${Math.round(timeToWait / 1000)}s` + ); + await new Promise(resolve => setTimeout(resolve, timeToWait)); + return apiCall(url, options); + } + + return resultApi.data; + } catch (error) { + console.error(error); + return undefined; + } + }; + return { repo: { issues: { getLastClosedIssue: async (params: { repoUrl: string | URL }) => { - try { - const { owner, repo } = parseURL(params.repoUrl); - const resIssues = await octokit.request("GET /repos/{owner}/{repo}/issues", { - owner, - repo, - headers: { - "X-GitHub-Api-Version": "2022-11-28" - }, - direction: "desc", - state: "closed" - }); - - return resIssues.data[0]; - } catch (error) { - return undefined; - } - } - }, - commits: { - getBySha: async (params: { - repoUrl: string | URL; - commit_sha: string; - }): Promise => { - try { - const { owner, repo } = parseURL(params.repoUrl); - const resCommit = await octokit.request("GET /repos/{owner}/{repo}/commits/{ref}", { - owner, - repo, - ref: params.commit_sha, - headers: { - "X-GitHub-Api-Version": "2022-11-28" - }, - direction: "desc" - }); - return resCommit.data; - } catch (error) { - return undefined; - } - }, - getLastCommit: async (params: { repoUrl: string | URL }): Promise => { - try { - const { owner, repo } = parseURL(params.repoUrl); - const resCommit = await octokit.request("GET /repos/{owner}/{repo}/commits", { - owner, - repo, - headers: { - "X-GitHub-Api-Version": "2022-11-28" - }, - direction: "desc" - }); - return resCommit.data[0]; - } catch (error) { - return undefined; - } - } - }, - mergeRequests: { - getLast: async (params: { repoUrl: string | URL }) => { - try { - const { owner, repo } = parseURL(params.repoUrl); - const resPull = await octokit.request("GET /repos/{owner}/{repo}/pulls", { - owner, - repo, - headers: { - "X-GitHub-Api-Version": "2022-11-28" - }, - direction: "desc", - state: "closed" - }); - - return resPull.data[0]; - } catch (error) { - return undefined; - } - } - }, - get: async (params: { repoUrl: string | URL }): Promise => { - try { const { owner, repo } = parseURL(params.repoUrl); - const resPull = await octokit.request("GET /repos/{owner}/{repo}", { + const url = "GET /repos/{owner}/{repo}/issues"; + return apiCall(url, { owner, repo, headers: { "X-GitHub-Api-Version": "2022-11-28" - } + }, + direction: "desc", + state: "closed" }); - - return resPull.data; - } catch (error) { - return undefined; } }, - getContributors: async (params: { repoUrl: string | URL }): Promise => { - try { + commits: { + getBySha: async (params: { + repoUrl: string | URL; + commit_sha: string; + }): Promise => { const { owner, repo } = parseURL(params.repoUrl); - const resPull = await octokit.request("GET /repos/{owner}/{repo}/contributors", { + const url = "GET /repos/{owner}/{repo}/commits/{ref}"; + const data = await apiCall(url, { owner, repo, + ref: params.commit_sha, headers: { "X-GitHub-Api-Version": "2022-11-28" }, direction: "desc" }); - - return resPull.data; - } catch (error) { - return undefined; - } - }, - getLanguages: async (params: { repoUrl: string | URL }): Promise => { - try { + return data[0]; + }, + getLastCommit: async (params: { repoUrl: string | URL }): Promise => { const { owner, repo } = parseURL(params.repoUrl); - const resPull = await octokit.request("GET /repos/{owner}/{repo}/languages", { + const url = "GET /repos/{owner}/{repo}/commits"; + const data = await apiCall(url, { owner, repo, headers: { @@ -159,81 +113,111 @@ export const gitHubEndpointMaker = (params?: { auth?: string }) => { }, direction: "desc" }); - - return resPull.data; - } catch (error) { - return undefined; + return data[0]; } }, - getTags: async (params: { repoUrl: string | URL }): Promise => { - try { + mergeRequests: { + getLast: async (params: { repoUrl: string | URL }) => { const { owner, repo } = parseURL(params.repoUrl); - const resPull = await octokit.request("GET /repos/{owner}/{repo}/tags", { + const url = "GET /repos/{owner}/{repo}/pulls"; + const data = await apiCall(url, { owner, repo, headers: { "X-GitHub-Api-Version": "2022-11-28" }, - direction: "desc" + direction: "desc", + state: "closed" }); - - return resPull.data; - } catch (error) { - return undefined; + return data[0]; } + }, + get: async (params: { repoUrl: string | URL }): Promise => { + const { owner, repo } = parseURL(params.repoUrl); + const url = "GET /repos/{owner}/{repo}"; + return apiCall(url, { + owner, + repo, + headers: { + "X-GitHub-Api-Version": "2022-11-28" + } + }); + }, + getContributors: async (params: { repoUrl: string | URL }): Promise => { + const { owner, repo } = parseURL(params.repoUrl); + const url = "GET /repos/{owner}/{repo}/contributors"; + return apiCall(url, { + owner, + repo, + headers: { + "X-GitHub-Api-Version": "2022-11-28" + }, + direction: "desc" + }); + }, + getLanguages: async (params: { repoUrl: string | URL }): Promise => { + const { owner, repo } = parseURL(params.repoUrl); + const url = "GET /repos/{owner}/{repo}/languages"; + return apiCall(url, { + owner, + repo, + headers: { + "X-GitHub-Api-Version": "2022-11-28" + }, + direction: "desc" + }); + }, + getTags: async (params: { repoUrl: string | URL }): Promise => { + const { owner, repo } = parseURL(params.repoUrl); + const url = "GET /repos/{owner}/{repo}/tags"; + return apiCall(url, { + owner, + repo, + headers: { + "X-GitHub-Api-Version": "2022-11-28" + }, + direction: "desc" + }); } }, users: { getById: async (account_id: number): Promise => { - try { - const resPull = await octokit.request("GET /user/{account_id}", { - account_id, - headers: { - "X-GitHub-Api-Version": "2022-11-28" - }, - direction: "desc", - state: "closed" - }); - - return resPull.data; - } catch (error) { - return undefined; - } + const url = "GET /user/{account_id}"; + return apiCall(url, { + account_id, + headers: { + "X-GitHub-Api-Version": "2022-11-28" + }, + direction: "desc", + state: "closed" + }); }, getByUsername: async (username: string): Promise => { - try { - const resPull = await octokit.request("GET /users/{username}", { - username, - headers: { - "X-GitHub-Api-Version": "2022-11-28" - }, - direction: "desc", - state: "closed" - }); - - return resPull.data; - } catch (error) { - return undefined; - } + const url = "GET /user/{username}"; + return apiCall(url, { + username, + headers: { + "X-GitHub-Api-Version": "2022-11-28" + }, + direction: "desc", + state: "closed" + }); } }, search: { repo: { searchByName: async (name: string): Promise => { - try { - const resPull = await octokit.request(`GET /search/repositories?q=${name} in:name`, { - headers: { - "X-GitHub-Api-Version": "2022-11-28" - }, - direction: "desc" - }); + const url = `GET /search/repositories?q=${name} in:name`; + const data = await apiCall(url, { + headers: { + "X-GitHub-Api-Version": "2022-11-28" + }, + direction: "desc" + }); - if (resPull.data.total_count === 0) return undefined; + if (data.total_count === 0) return undefined; - return resPull.data; - } catch (error) { - return undefined; - } + return data; } } } diff --git a/api/src/core/adapters/GitHub/getExternalData.ts b/api/src/core/adapters/GitHub/getExternalData.ts index 4780e626e..6d1f5930b 100644 --- a/api/src/core/adapters/GitHub/getExternalData.ts +++ b/api/src/core/adapters/GitHub/getExternalData.ts @@ -49,7 +49,7 @@ export const getGitHubSoftwareExternalData: GetSoftwareExternal = memoize( const versionCommitSha = repoTags?.[0]?.commit?.sha; const lastVersionCommit = versionCommitSha ? await gitHubApi.repo.commits.getBySha({ - repoUrl: externalId, + repoUrl, commit_sha: versionCommitSha }) : undefined; diff --git a/api/src/core/adapters/dbApi/kysely/createPgSoftwareRepository.ts b/api/src/core/adapters/dbApi/kysely/createPgSoftwareRepository.ts index d98303db8..6942a7215 100644 --- a/api/src/core/adapters/dbApi/kysely/createPgSoftwareRepository.ts +++ b/api/src/core/adapters/dbApi/kysely/createPgSoftwareRepository.ts @@ -644,6 +644,7 @@ export const createPgSoftwareRepository = (db: Kysely): SoftwareReposi externalId }) ) + .onConflict(oc => oc.columns(["sourceSlug", "externalId"]).doUpdateSet({ softwareId })) .execute(); } else { await trx diff --git a/api/src/core/adapters/dbApi/kysely/kysely.database.ts b/api/src/core/adapters/dbApi/kysely/kysely.database.ts index 5a1283aee..7089c5ee2 100644 --- a/api/src/core/adapters/dbApi/kysely/kysely.database.ts +++ b/api/src/core/adapters/dbApi/kysely/kysely.database.ts @@ -208,10 +208,16 @@ type SimilarExternalSoftwareExternalDataTable = { sourceSlug: string; }; +export type BotAgent = { + "User-Agent": string; + from?: string; +}; + export type SourceConfig = { queryTimeout?: number; auth?: string; rateLimitRetryDuration?: number; + agent?: BotAgent; }; type SourcesTable = { diff --git a/api/src/core/adapters/doiResolver.ts b/api/src/core/adapters/doiResolver.ts index 6e4f1745d..c73a46e31 100644 --- a/api/src/core/adapters/doiResolver.ts +++ b/api/src/core/adapters/doiResolver.ts @@ -85,8 +85,8 @@ export const resolveDOIIdentifier = async (doiIdentifier: SchemaIdentifier): Pro if (record) { return identifersUtils.makeZenodoIdentifer({ - zenodoId: record.id.toString(), - url: urlValue.data.value + zenodoId: record.conceptrecid.toString(), + url: record.links.parent }); } } diff --git a/api/src/core/adapters/zenodo/getZenodoExternalData.ts b/api/src/core/adapters/zenodo/getZenodoExternalData.ts index d4664d93b..f185417d1 100644 --- a/api/src/core/adapters/zenodo/getZenodoExternalData.ts +++ b/api/src/core/adapters/zenodo/getZenodoExternalData.ts @@ -20,28 +20,28 @@ export const getZenodoExternalData: GetSoftwareExternal = memoize( const zenodoApi = makeZenodoApi(source); - let record: Zenodo.Record | undefined = undefined; + const cleanId = externalId.includes("10.5281") ? externalId.split(".")[1] : externalId; - if (externalId.includes("10.5281")) { - record = await zenodoApi.records.getByDOI(externalId); - } else { - record = await zenodoApi.records.get(Number(externalId)); - } + try { + const record = await zenodoApi.records.get(Number(cleanId)); - const communities = await zenodoApi.records.getCommunities(Number(externalId)); - const communitiesRows = communities?.hits?.hits ?? []; + const communities = await zenodoApi.records.getCommunities(Number(externalId)); + const communitiesRows = communities?.hits?.hits ?? []; - if (!record) return undefined; - if (record.metadata.resource_type.type !== "software") { - console.error(`The record corresponding at ${externalId} is not a software`); - return undefined; - } + if (!record) return undefined; + if (record.metadata.resource_type.type !== "software") { + console.error(`The record corresponding at ${externalId} is not a software`); + return undefined; + } - const formatedExternalData = formatRecordToExternalData(record, communitiesRows, source); + const formatedExternalData = formatRecordToExternalData(record, communitiesRows, source); - formatedExternalData.identifiers = await populateFromDOIIdentifiers(formatedExternalData.identifiers ?? []); + formatedExternalData.identifiers = await populateFromDOIIdentifiers(formatedExternalData.identifiers ?? []); - return formatedExternalData; + return formatedExternalData; + } catch (err) { + throw err; + } } ); @@ -84,7 +84,7 @@ export const formatRecordToExternalData = ( sourceSlug: source.slug, authors: recordSoftwareItem.metadata.creators.map(creatorToPerson), name: { "en": recordSoftwareItem.metadata.title }, - description: { "en": recordSoftwareItem.metadata.description }, + description: { "en": recordSoftwareItem.metadata.description ?? "" }, isLibreSoftware: recordSoftwareItem.metadata.access_right === "open", image: undefined, url: undefined, @@ -110,6 +110,11 @@ export const formatRecordToExternalData = ( url: `https://zenodo.org/records/${recordSoftwareItem.id.toString()}`, additionalType: "Software" }), + identifersUtils.makeZenodoIdentifer({ + zenodoId: recordSoftwareItem.conceptdoi.toString(), + url: `https://zenodo.org/records/${recordSoftwareItem.conceptdoi.toString()}`, + additionalType: "Software" + }), ...(recordSoftwareItem?.metadata?.doi ? [identifersUtils.makeDOIIdentifier({ doi: recordSoftwareItem.metadata.doi })] : []), diff --git a/api/src/core/adapters/zenodo/getZenodoSoftwareForm.ts b/api/src/core/adapters/zenodo/getZenodoSoftwareForm.ts index c53db5e4d..6a0d4a42c 100644 --- a/api/src/core/adapters/zenodo/getZenodoSoftwareForm.ts +++ b/api/src/core/adapters/zenodo/getZenodoSoftwareForm.ts @@ -1,5 +1,5 @@ -// SPDX-FileCopyrightText: 2021-2025 DINUM -// SPDX-FileCopyrightText: 2024-2025 Université Grenoble Alpes +// SPDX-FileCopyrightText: 2021-2026 DINUM +// SPDX-FileCopyrightText: 2024-2026 Université Grenoble Alpes // SPDX-License-Identifier: MIT import memoize from "memoizee"; @@ -8,16 +8,24 @@ import { SoftwareFormData, Source } from "../../usecases/readWriteSillData"; import { makeZenodoApi } from "./zenodoAPI"; import { Zenodo } from "./zenodoAPI/type"; import { GetSoftwareFormData } from "../../ports/GetSoftwareFormData"; +import { SoftwareExternal } from "../../types/SoftwareTypes"; +import { createResolveLocalizedString } from "i18nifty/LocalizedString/reactless"; +import type { Language } from "../../ports/GetSoftwareExternalData"; + +const { resolveLocalizedString } = createResolveLocalizedString({ + "currentLanguage": "fr", + "fallbackLanguage": "en" +}); export const getZenodoSoftwareFormData: GetSoftwareFormData = memoize( async ({ externalId, source }: { externalId: string; source: Source }) => { if (source.kind !== "Zenodo" && source.url !== "https://zenodo.org/") throw new Error(`Not a Zenodo source, was : ${source.kind}`); - const zenodoApi = makeZenodoApi(); + const zenodoApi = makeZenodoApi(source); const record = await zenodoApi.records.get(Number(externalId)); - if (!record) return undefined; + if (!record || record.status == 410) return undefined; if (record.metadata.resource_type.type !== "software") throw new TypeError(`The record corresponding at ${externalId} is not a software`); @@ -25,6 +33,33 @@ export const getZenodoSoftwareFormData: GetSoftwareFormData = memoize( } ); +export const formatExternalDataToFormData = (zenodoExternalData: SoftwareExternal): SoftwareFormData => { + return { + name: resolveLocalizedString(zenodoExternalData.name), + nameOverride: null, + description: resolveLocalizedString(zenodoExternalData.description) ?? null, + operatingSystems: { "linux": false, "windows": false, "android": false, "ios": false, "mac": false }, + runtimePlatforms: [], + externalIdForSource: zenodoExternalData.externalId, + sourceSlug: zenodoExternalData.sourceSlug, + license: zenodoExternalData.license ?? null, + similarSoftwareExternalDataItems: [], + image: null, + keywords: zenodoExternalData.keywords ?? [], + customAttributes: undefined, + isLibreSoftware: null, + url: zenodoExternalData.url ?? null, + codeRepositoryUrl: zenodoExternalData.codeRepositoryUrl ?? null, + softwareHelp: null, + latestVersion: zenodoExternalData.latestVersion + ? { + version: zenodoExternalData.latestVersion.version ?? null, + releaseDate: zenodoExternalData.latestVersion.releaseDate ?? null + } + : null + }; +}; + export const formatRecordToSoftwareFormData = (recordSoftwareItem: Zenodo.Record, source: Source): SoftwareFormData => { const publicationIso = recordSoftwareItem.metadata.publication_date ? new Date(recordSoftwareItem.metadata.publication_date).toISOString() diff --git a/api/src/core/adapters/zenodo/getZenodoSoftwareOptions.ts b/api/src/core/adapters/zenodo/getZenodoSoftwareOptions.ts index 0bb54ac7e..f748da1da 100644 --- a/api/src/core/adapters/zenodo/getZenodoSoftwareOptions.ts +++ b/api/src/core/adapters/zenodo/getZenodoSoftwareOptions.ts @@ -1,5 +1,5 @@ -// SPDX-FileCopyrightText: 2021-2025 DINUM -// SPDX-FileCopyrightText: 2024-2025 Université Grenoble Alpes +// SPDX-FileCopyrightText: 2021-2026 DINUM +// SPDX-FileCopyrightText: 2024-2026 Université Grenoble Alpes // SPDX-License-Identifier: MIT import { GetSoftwareExternalDataOptions, SoftwareExternalDataOption } from "../../ports/GetSoftwareExternalDataOptions"; @@ -11,10 +11,10 @@ export const getZenodoSoftwareOptions: GetSoftwareExternalDataOptions = async ({ if (source.kind !== "Zenodo" && source.url !== "https://zenodo.org/") throw new Error(`Not a Zenodo source, was : ${source.kind}`); - const zenodoApi = makeZenodoApi(); + const zenodoApi = makeZenodoApi(source); const records = await zenodoApi.records.getByNameAndType(queryString, "software"); - if (!records.hits || records.hits.hits.length === 0) return []; + if (!records || !records.hits || records.hits.hits.length === 0) return []; return records.hits.hits.map(record => formatRecordToSoftwareOption(record, source)); }; diff --git a/api/src/core/adapters/zenodo/zenodoAPI/index.ts b/api/src/core/adapters/zenodo/zenodoAPI/index.ts index 13a29455f..75a2bb4ba 100644 --- a/api/src/core/adapters/zenodo/zenodoAPI/index.ts +++ b/api/src/core/adapters/zenodo/zenodoAPI/index.ts @@ -1,5 +1,5 @@ -// SPDX-FileCopyrightText: 2021-2025 DINUM -// SPDX-FileCopyrightText: 2024-2025 Université Grenoble Alpes +// SPDX-FileCopyrightText: 2021-2026 DINUM +// SPDX-FileCopyrightText: 2024-2026 Université Grenoble Alpes // SPDX-License-Identifier: MIT import { convertSourceConfigToRequestInit } from "../../../../tools/sourceConfig"; @@ -9,124 +9,95 @@ import { Zenodo } from "./type"; const ZENODO_API_API_TIMEOUT = 60 * 1000; +const LOG = "[ZenodoWrapper]"; + export const makeZenodoApi = (source?: Source) => { const timeOutBreak = source?.configuration?.rateLimitRetryDuration ?? ZENODO_API_API_TIMEOUT; const auth = source?.configuration?.auth; - const config = source?.configuration ? convertSourceConfigToRequestInit(source.configuration) : {}; + const config = source?.configuration + ? convertSourceConfigToRequestInit(source.configuration) + : { + headers: { + "User-Agent": "Catalogi/1.55.8 (+https://github.com/codegouvfr/catalogi;" + } + }; - const getRecord = async (zenodoRecordId: number): Promise => { - const url = `https://zenodo.org/api/records/${zenodoRecordId}`; + const apiCall = async (url: string): Promise => { + console.debug(`${LOG} Zenodo call ${url}`); const res = await fetch(url, config).catch(err => { console.error(err); throw new Error(err); }); - if (res.status === 404) { - console.debug(`Could not find records : ${zenodoRecordId}`); + if (res.status === 403) { + console.debug(`${LOG} Got ${res.status} Forbidden resolve the url : ${url}`); return undefined; } - if (res.status === 429) { - await new Promise(resolve => setTimeout(resolve, timeOutBreak)); - return getRecord(zenodoRecordId); - } - - return res.json(); - }; - const getRecordByDOI = async (zenodoDOI: string): Promise => { - const url = `https://zenodo.org/doi/${zenodoDOI}`; - - const res = await fetch(url, config).catch(err => { - console.error(err); - throw new Error(err); - }); - - if (res.status === 404) { - console.debug(`Could not find records : ${zenodoDOI}`); + if (res.status === 404 || res.status === 410) { + console.debug(`${LOG} Got ${res.status} Could not resolve the url : ${url}`); return undefined; } - if (res.status === 429) { + if (res.status === 429 || res.status > 500) { + console.debug(`${LOG} Got ${res.status} let's take a break for 60s`); await new Promise(resolve => setTimeout(resolve, timeOutBreak)); - return getRecordByDOI(zenodoDOI); + return apiCall(url); } - if (url !== res.url) { - const urlParsed = res.url.split("/"); - return getRecord(Number(urlParsed[urlParsed.length - 1])); + if (res.status !== 200) { + throw new Error(LOG + res.status.toString()); } - return undefined; + try { + return res.json(); + } catch (err) { + console.error(LOG, err); + throw new Error(LOG + err); + } }; - const getRecordByName = async (name: string, type: string): Promise> => { - const url = `https://zenodo.org/api/records?q=title:${name} AND type:${type}`; + const getRecord = async (zenodoRecordId: number): Promise => { + const url = `https://zenodo.org/api/records/${zenodoRecordId}`; - const res = await fetch(url, config).catch(err => { - console.error(err); - throw new Error(err); - }); + return apiCall(url); + }; + const getRecordByDOI = async (zenodoDOI: string): Promise => { + const doiSplit = zenodoDOI.split("/"); + const doiReference = doiSplit[1].split("."); - if (res.status === 404) { - throw new Error(`Could find endpoint`); - } + if (doiSplit[0] !== "10.5281" || doiReference[0] !== "zenodo") + throw new Error(`${LOG} The doi (${zenodoDOI}) isn't valid.`); - if (res.status === 429) { - await new Promise(resolve => setTimeout(resolve, timeOutBreak)); - return getRecordByName(name, type); - } + const url = `https://zenodo.org/api/records/${doiReference[1]}`; - return res.json(); + return apiCall(url); }; - const getAllSoftware = async (params: { page?: number; date: Date }): Promise> => { + const getRecordByName = async (name: string, type: string): Promise | undefined> => { + const url = `https://zenodo.org/api/records?q=title:${name} AND type:${type}`; + + return apiCall(url); + }; + + const getAllSoftware = async (params: { + page?: number; + date: Date; + }): Promise | undefined> => { const { page = 1 } = params; const size = auth ? 100 : 25; const formatedDate: string = formatDateToYYYMMDD(params.date); const url = `https://zenodo.org/api/records?q=type:software AND created:[${formatedDate} TO *]&size=${size}&page=${page}&sort=-mostrecent&all_versions=false`; - const res = await fetch(encodeURI(url), config).catch(err => { - console.error(err); - throw new Error(err); - }); - - if (res.status === 404) { - throw new Error(`Could find endpoint`); - } - - if (res.status === 429 || res.status === 504) { - await new Promise(resolve => setTimeout(resolve, timeOutBreak)); - return getAllSoftware(params); - } - - if (res.status !== 200) { - throw new Error(res.status.toString()); - } - - return res.json(); + return apiCall(url); }; const getCommunities = async (zenodoRecordId: number): Promise | undefined> => { const url = `https://zenodo.org/api/records/${zenodoRecordId}/communities`; - const res = await fetch(url, config).catch(err => { - console.error(err); - throw new Error(err); - }); - - if (res.status === 404) { - console.debug(`Could not find commuties for record : ${zenodoRecordId}`); - return undefined; - } - - if (res.status === 429) { - await new Promise(resolve => setTimeout(resolve, timeOutBreak)); - return getCommunities(zenodoRecordId); - } - - return res.json(); + return apiCall(url); }; return { diff --git a/api/src/core/adapters/zenodo/zenodoAPI/type.ts b/api/src/core/adapters/zenodo/zenodoAPI/type.ts index 7f8765d39..bdb72b2f7 100644 --- a/api/src/core/adapters/zenodo/zenodoAPI/type.ts +++ b/api/src/core/adapters/zenodo/zenodoAPI/type.ts @@ -1,5 +1,5 @@ -// SPDX-FileCopyrightText: 2021-2025 DINUM -// SPDX-FileCopyrightText: 2024-2025 Université Grenoble Alpes +// SPDX-FileCopyrightText: 2021-2026 DINUM +// SPDX-FileCopyrightText: 2024-2026 Université Grenoble Alpes // SPDX-License-Identifier: MIT export namespace Zenodo { @@ -156,7 +156,7 @@ export namespace Zenodo { files: File[]; swh: SWH; owners: Owner[]; - status: string; + status: string | number; stats: Stats; state: string; submitted: boolean; diff --git a/api/src/core/usecases/importFromSource.ts b/api/src/core/usecases/importFromSource.ts index 3634605a3..60647a131 100644 --- a/api/src/core/usecases/importFromSource.ts +++ b/api/src/core/usecases/importFromSource.ts @@ -5,14 +5,14 @@ import { DbApiV2 } from "../ports/DbApiV2"; import { makeHalAPIGateway } from "../adapters/hal/HalAPI"; import { makeCreateSofware } from "./createSoftware"; -import { Source } from "./readWriteSillData"; -import { GetSoftwareFormData } from "../ports/GetSoftwareFormData"; +import { SoftwareFormData, Source } from "./readWriteSillData"; import { resolveAdapterFromSource } from "../adapters/resolveAdapter"; import { makeZenodoApi } from "../adapters/zenodo/zenodoAPI"; -import { USER_INPUT_SOURCE_SLUG } from "../adapters/dbApi/kysely/kysely.database"; -import { formatRecordToSoftwareFormData } from "../adapters/zenodo/getZenodoSoftwareForm"; +import { ExternalDataOriginKind, USER_INPUT_SOURCE_SLUG } from "../adapters/dbApi/kysely/kysely.database"; +import { formatExternalDataToFormData, formatRecordToSoftwareFormData } from "../adapters/zenodo/getZenodoSoftwareForm"; import { saveExternalData } from "./refreshExternalData"; import { formatRecordToExternalData } from "../adapters/zenodo/getZenodoExternalData"; +import { SoftwareExternal } from "../types/SoftwareTypes"; export type ImportFromSource = (params: { userEmail: string; @@ -50,13 +50,7 @@ export const importFromSource: (dbApi: DbApiV2) => ImportFromSource = (dbApi: Db console.info(`[UC:Import] Importing ${softwareIds.length} software packages from ${source.slug}`); for (const externalId of softwareIds) { - const newId = await checkSoftware( - dbApi, - source, - externalId, - sourceGateway.software.getSoftwareForm, - userId - ); + const newId = await checkSoftware(dbApi, source, externalId, userId); result.push(newId); } } else if (["Zenodo"].includes(source.kind)) { @@ -92,24 +86,59 @@ const resolveAllIdsAccordingToSource = async (source: Source): Promise } }; -const checkSoftware = async ( - dbApi: DbApiV2, - source: Source, - externalId: string, - getSoftwareForm: GetSoftwareFormData, - userId: number -) => { - // Get software form from source - const softwareForm = await getSoftwareForm({ externalId, source }); - if (!softwareForm || !softwareForm.name) { +const SOURCE_EXDATA_TO_FORM: ExternalDataOriginKind[] = ["Zenodo"]; + +const checkSoftware = async (dbApi: DbApiV2, source: Source, externalId: string, userId: number) => { + const sourceGateway = resolveAdapterFromSource(source); + if (!sourceGateway?.software?.getSoftwareForm) + throw new Error("[UC:Import] Import if not possible from a secondary source"); + + const sources = await dbApi.source.getAll(); + + const exData = await dbApi.softwareExternalData.get({ sourceSlug: source.slug, externalId }); + + if (exData?.softwareId !== undefined) { + console.info(`[UC:Import] Importing (${externalId}) from ${source.slug} : Already existing`); + return exData.softwareId; + } + + let formFromSource: SoftwareFormData | undefined = undefined; + let dataFromSource: SoftwareExternal | undefined = undefined; + if (SOURCE_EXDATA_TO_FORM.includes(source.kind) && sourceGateway.softwareExtra !== undefined) { + dataFromSource = await sourceGateway.softwareExtra.getSoftwareExternal({ externalId, source }); + + if (dataFromSource) { + if (source.kind === "Zenodo") formFromSource = formatExternalDataToFormData(dataFromSource); + } + } + + if (!formFromSource) { + // Get software form from source + formFromSource = await sourceGateway.software.getSoftwareForm({ externalId, source }); + } + + if (!formFromSource || !formFromSource.name) { return undefined; } console.info( - `[UC:Import] Importing ${softwareForm.name}(${externalId}) from ${source.slug} : Adding software and externalData ` + `[UC:Import] Importing ${formFromSource.name}(${formFromSource.externalIdForSource}) from ${formFromSource.sourceSlug} : Adding software and externalData ` ); + const createSoftware = makeCreateSofware({ dbApi, withUserInput: false }); - return createSoftware({ formData: softwareForm, userId }); + const createdId = await createSoftware({ formData: formFromSource, userId }); + + if (dataFromSource && createdId) { + await saveExternalData({ + dbApi, + sourceSlug: source.slug, + sources, + externalData: dataFromSource, + externalId: dataFromSource.externalId + }); + } + + return createdId; }; const directImportFromSource = async (params: { dbApi: DbApiV2; source: Source; userId: number }) => { @@ -134,7 +163,7 @@ const directImportFromSource = async (params: { dbApi: DbApiV2; source: Source; date: source.lastImport ?? new Date("1968") }); - if (softwareRecords.hits.hits.length === 0) { + if (!softwareRecords || softwareRecords?.hits?.hits.length === 0) { end = false; continue; } diff --git a/api/src/entrypoints/import.ts b/api/src/entrypoints/import.ts index 7510329e4..eeda5ee17 100644 --- a/api/src/entrypoints/import.ts +++ b/api/src/entrypoints/import.ts @@ -5,6 +5,8 @@ import { env } from "../env"; import { startImportService } from "../rpc/import"; import commandLineArgs from "command-line-args"; +import { readFileSync } from "fs"; +import { resolve } from "path"; const optionDefinitions = [ { name: "source", alias: "s", type: String, multiple: false, defaultOption: true }, @@ -13,10 +15,59 @@ const optionDefinitions = [ const options = commandLineArgs(optionDefinitions); -startImportService({ - env, - args: { - sourceSlug: options.source, - externalIdsToImport: options.externalIdsToImport +const parseExternalIds = (externalIdsInput: string | string[]): Promise => { + const inputs = Array.isArray(externalIdsInput) ? externalIdsInput : [externalIdsInput]; + + const allIds: string[] = []; + + for (const input of inputs) { + if (!input) continue; + + if (input.endsWith(".json") || input.startsWith("/") || input.startsWith("./")) { + try { + const filePath = resolve(input); + const fileContent = readFileSync(filePath, "utf-8"); + const parsed = JSON.parse(fileContent); + + // Handle both {["id1","id2"]} and ["id1","id2"] formats + if (Array.isArray(parsed)) { + allIds.push(...parsed); + } else if (typeof parsed === "object" && parsed !== null) { + const listOfIds = parsed.ids || parsed.externalIds; + if (Array.isArray(listOfIds)) { + allIds.push(...listOfIds); + } + } + } catch (error) { + console.error(`[Entrypoint:Import] Error reading file ${input}:`, error); + throw error; + } + } else { + // Treat as comma-separated string + allIds.push( + ...input + .split(",") + .map(id => id.trim()) + .filter(id => id) + ); + } } -}).then(() => console.info("[Entrypoint:Import] Import sucessuful ✅ Closing import")); + + return Promise.resolve(allIds); +}; + +parseExternalIds(options.externalIdsToImport) + .then(externalIds => { + return startImportService({ + env, + args: { + sourceSlug: options.source, + externalIdsToImport: externalIds + } + }); + }) + .then(() => console.info("[Entrypoint:Import] Import successful ✅ Closing import")) + .catch(error => { + console.error("[Entrypoint:Import] Error during import:", error); + process.exit(1); + }); diff --git a/api/src/tools/sourceConfig.ts b/api/src/tools/sourceConfig.ts index bfa7d1fad..89923d225 100644 --- a/api/src/tools/sourceConfig.ts +++ b/api/src/tools/sourceConfig.ts @@ -4,11 +4,25 @@ import { SourceConfig } from "../core/adapters/dbApi/kysely/kysely.database"; import { Source } from "../lib/ApiTypes"; +import { projectVersion } from "./projectVersion"; export const convertSourceConfigToRequestInit = (sourceConfig: SourceConfig | undefined): RequestInit => { if (!sourceConfig) return {}; + const headers: HeadersInit = {}; + + if (sourceConfig.auth) { + headers.Authorization = `Bearer ${sourceConfig.auth}`; + } + + if (sourceConfig.agent) { + headers["User-Agent"] = sourceConfig.agent["User-Agent"]; + if (sourceConfig.agent.from) headers.from = sourceConfig.agent.from; + } else { + headers["User-Agent"] = `Catalogi/${projectVersion} (+https://github.com/codegouvfr/catalogi;`; + } + return { - ...(sourceConfig.auth ? { headers: { Authorization: `Bearer ${sourceConfig.auth}` } } : {}), + headers, ...(sourceConfig.queryTimeout ? { signal: AbortSignal.timeout(sourceConfig.queryTimeout) } : {}) }; };