{ "name": "DocuQuery 1 - Ingestion", "nodes": [ { "parameters": { "event": "fileCreated", "triggerOn": "specificFolder", "folderToWatch": { "__rl": true, "value": "={{ $env.DRIVE_FOLDER_ID }}", "mode": "id" }, "pollTimes": { "item": [ { "mode": "everyX", "unit": "minutes", "value": 5 } ] }, "options": {} }, "id": "c8e406f3-3ade-4ce7-ade0-32744cbde91d", "name": "Drive Trigger: File Created", "type": "n8n-nodes-base.googleDriveTrigger", "typeVersion": 1, "position": [ 0, 0 ], "notes": "Credential: Google Drive OAuth2" }, { "parameters": { "event": "fileUpdated", "triggerOn": "specificFolder", "folderToWatch": { "__rl": true, "value": "={{ $env.DRIVE_FOLDER_ID }}", "mode": "id" }, "pollTimes": { "item": [ { "mode": "everyX", "unit": "minutes", "value": 5 } ] }, "options": {} }, "id": "f066848e-1ba7-4711-a73a-54b1b2bafb47", "name": "Drive Trigger: File Updated", "type": "n8n-nodes-base.googleDriveTrigger", "typeVersion": 1, "position": [ 0, 190 ], "notes": "Fires on metadata-only edits too. \"Decide Action\" filters those out." }, { "parameters": {}, "id": "e7d08059-eff8-4c62-adc0-3c90dceb53af", "name": "Manual Trigger: Backfill Folder", "type": "n8n-nodes-base.manualTrigger", "typeVersion": 1, "position": [ 0, 380 ], "notes": "Run once to index everything already sitting in the folder." }, { "parameters": { "rule": { "interval": [ { "field": "days", "daysInterval": 1, "triggerAtHour": 3, "triggerAtMinute": 0 } ] } }, "id": "4c9e14ed-e2a3-4278-ac55-ee4bb8d0cb1c", "name": "Schedule: Nightly Sweep", "type": "n8n-nodes-base.scheduleTrigger", "typeVersion": 1.2, "position": [ 0, 570 ], "notes": "Reconciliation: catches files the poll trigger missed and retries earlier failures, bounded by MAX_INGEST_RETRIES." }, { "parameters": { "mode": "runOnceForAllItems", "jsCode": "// --- inlined from src/lib/config.js ---------------------------\n/**\n * Environment reading + validation shared by every workflow.\n *\n * Workflows fail fast with one readable message naming the missing variable,\n * rather than throwing \"undefined is not a string\" six nodes downstream.\n */\n\n/**\n * @param {object} env Usually n8n's `$env`.\n * @param {object[]} spec [{ key, name, required, default, type }]\n */\nfunction readConfig(env, spec) {\n const source = env ?? {};\n const out = {};\n const missing = [];\n const invalid = [];\n\n for (const entry of spec) {\n const raw = source[entry.key];\n let value = raw === undefined || raw === null || raw === '' ? entry.default : raw;\n\n if (value === undefined || value === null || value === '') {\n if (entry.required) missing.push(entry.key);\n out[entry.name] = entry.default ?? null;\n continue;\n }\n\n switch (entry.type) {\n case 'number': {\n const n = Number(value);\n if (!Number.isFinite(n)) invalid.push(`${entry.key} must be a number (got \"${value}\")`);\n value = n;\n break;\n }\n case 'boolean':\n value = value === true || String(value).trim().toLowerCase() === 'true' || String(value) === '1';\n break;\n case 'json':\n if (typeof value === 'string') {\n try {\n value = JSON.parse(value);\n } catch {\n invalid.push(`${entry.key} must be valid JSON (got \"${String(value).slice(0, 40)}\")`);\n value = entry.default;\n }\n }\n break;\n case 'list':\n value = String(value)\n .split(',')\n .map((v) => v.trim())\n .filter(Boolean);\n break;\n case 'url':\n value = String(value).trim().replace(/\\/+$/, '');\n if (!/^https?:\\/\\//i.test(value)) {\n invalid.push(`${entry.key} must start with https:// (got \"${value}\")`);\n }\n break;\n default:\n value = String(value).trim();\n }\n\n out[entry.name] = value;\n }\n\n const problems = [];\n if (missing.length) problems.push(`missing: ${missing.join(', ')}`);\n if (invalid.length) problems.push(invalid.join('; '));\n if (problems.length) {\n throw new Error(\n `DocuQuery configuration problem - ${problems.join(' | ')}. ` +\n `Set these in your n8n environment (see .env.example) and restart n8n.`,\n );\n }\n\n return out;\n}\n\n/**\n * n8n hides `$env` when N8N_BLOCK_ENV_ACCESS_IN_NODE is on. Detect it here so\n * the failure names the setting instead of surfacing as \"$env is not defined\".\n */\nfunction requireEnv(env) {\n if (!env || typeof env !== 'object') {\n throw new Error(\n 'This node cannot read environment variables. Set ' +\n 'N8N_BLOCK_ENV_ACCESS_IN_NODE=false on the n8n container and restart it.',\n );\n }\n return env;\n}\n\nconst INGESTION_CONFIG_SPEC = [\n { key: 'DRIVE_FOLDER_ID', name: 'driveFolderId', required: true },\n { key: 'PINECONE_INDEX_HOST', name: 'pineconeHost', required: true, type: 'url' },\n { key: 'PINECONE_NAMESPACE', name: 'pineconeNamespace', default: 'docuquery' },\n { key: 'PINECONE_DIMENSION', name: 'pineconeDimension', type: 'number', default: 1536 },\n { key: 'EMBEDDING_MODEL', name: 'embeddingModel', default: 'text-embedding-3-small' },\n { key: 'EMBED_BATCH_SIZE', name: 'embedBatchSize', type: 'number', default: 96 },\n { key: 'CHUNK_TOKENS', name: 'chunkTokens', type: 'number', default: 500 },\n { key: 'CHUNK_OVERLAP_TOKENS', name: 'overlapTokens', type: 'number', default: 50 },\n { key: 'AIRTABLE_BASE_ID', name: 'airtableBaseId', required: true },\n { key: 'AIRTABLE_DOCUMENTS_TABLE', name: 'documentsTable', default: 'Documents' },\n { key: 'MAX_INGEST_RETRIES', name: 'maxRetries', type: 'number', default: 3 },\n { key: 'DEPARTMENTS', name: 'departments', type: 'list', default: 'General' },\n { key: 'DEFAULT_DEPARTMENT', name: 'defaultDepartment', default: 'General' },\n { key: 'DEPARTMENT_FOLDER_MAP', name: 'departmentFolderMap', type: 'json', default: {} },\n];\n\nconst QUERY_CONFIG_SPEC = [\n { key: 'PINECONE_INDEX_HOST', name: 'pineconeHost', required: true, type: 'url' },\n { key: 'PINECONE_NAMESPACE', name: 'pineconeNamespace', default: 'docuquery' },\n { key: 'EMBEDDING_MODEL', name: 'embeddingModel', default: 'text-embedding-3-small' },\n { key: 'LLM_PROVIDER', name: 'llmProvider', default: 'openai' },\n { key: 'LLM_MODEL', name: 'llmModel', default: 'gpt-4o' },\n { key: 'LLM_MAX_TOKENS', name: 'llmMaxTokens', type: 'number', default: 900 },\n { key: 'LLM_TEMPERATURE', name: 'llmTemperature', type: 'number', default: 0 },\n { key: 'LLM_BASE_URL', name: 'llmBaseUrl', default: '' },\n { key: 'RETRIEVAL_TOP_K', name: 'topK', type: 'number', default: 8 },\n { key: 'RETRIEVAL_MIN_SCORE', name: 'minScore', type: 'number', default: 0.32 },\n { key: 'RETRIEVAL_MAX_PER_DOC', name: 'maxPerDoc', type: 'number', default: 4 },\n { key: 'RETRIEVAL_MAX_CONTEXT_TOKENS', name: 'maxContextTokens', type: 'number', default: 6000 },\n { key: 'DEPARTMENTS', name: 'departments', type: 'list', default: 'General' },\n { key: 'SLACK_SIGNING_SECRET', name: 'slackSigningSecret', default: '' },\n { key: 'SLACK_TEAM_ID', name: 'slackTeamId', default: '' },\n { key: 'SLACK_ALLOW_UNVERIFIED', name: 'slackAllowUnverified', type: 'boolean', default: false },\n { key: 'QUERY_API_TOKEN', name: 'queryApiToken', default: '' },\n];\n// --- end src/lib/config.js\n\n/**\n * Node: \"Validate Config\" (Ingestion)\n *\n * Fails the run immediately, with a message naming the variable, rather than\n * letting a missing id surface as a confusing 404 six nodes later. Items pass\n * through untouched.\n */\n\nconst env = requireEnv(typeof $env === 'undefined' ? null : $env);\nreadConfig(env, INGESTION_CONFIG_SPEC);\n\nconst items = $input.all();\nreturn items.length ? items : [{ json: {} }];\n" }, "id": "a75b41e8-fa4c-49f2-a635-e4ea2e8efa75", "name": "Validate Config", "type": "n8n-nodes-base.code", "typeVersion": 2, "position": [ 300, 285 ], "notes": "Fails the run with one readable message if an env var is missing." }, { "parameters": { "conditions": { "options": { "caseSensitive": true, "leftValue": "", "typeValidation": "loose", "version": 2 }, "combinator": "and", "conditions": [ { "id": "7e9712c7-a82d-4fb9-afde-ae343f3888af", "leftValue": "={{ $json.id }}", "rightValue": "", "operator": { "type": "string", "operation": "notEmpty", "singleValue": true } } ] }, "looseTypeValidation": true, "options": {} }, "id": "297b107d-0c8a-49b6-a2b3-d0f1d985d2f2", "name": "Has File Id?", "type": "n8n-nodes-base.if", "typeVersion": 2.2, "position": [ 600, 285 ], "notes": "Trigger events arrive with a file id; manual and scheduled runs do not." }, { "parameters": { "resource": "file", "operation": "get", "fileId": { "__rl": true, "value": "={{ $json.id }}", "mode": "id" }, "options": { "fields": [ "*" ] } }, "id": "6964d78f-57fc-4ad1-ab59-cb45d41061a0", "name": "Drive: Get File Metadata", "type": "n8n-nodes-base.googleDrive", "typeVersion": 3, "position": [ 900, 95 ], "notes": "Needed for md5Checksum, which trigger payloads omit.", "retryOnFail": true, "maxTries": 3, "waitBetweenTries": 2000 }, { "parameters": { "resource": "fileFolder", "searchMethod": "query", "queryString": "={{ \"'\" + $env.DRIVE_FOLDER_ID + \"' in parents and mimeType = 'application/pdf' and trashed = false\" }}", "returnAll": true, "filter": {}, "options": { "fields": [ "*" ] } }, "id": "8a856eb9-dfe2-435b-a6c3-0427644a0019", "name": "Drive: List PDFs In Folder", "type": "n8n-nodes-base.googleDrive", "typeVersion": 3, "position": [ 900, 475 ], "retryOnFail": true, "maxTries": 3, "waitBetweenTries": 2000 }, { "parameters": { "mode": "runOnceForAllItems", "jsCode": "// --- inlined from src/lib/config.js ---------------------------\n/**\n * Environment reading + validation shared by every workflow.\n *\n * Workflows fail fast with one readable message naming the missing variable,\n * rather than throwing \"undefined is not a string\" six nodes downstream.\n */\n\n/**\n * @param {object} env Usually n8n's `$env`.\n * @param {object[]} spec [{ key, name, required, default, type }]\n */\nfunction readConfig(env, spec) {\n const source = env ?? {};\n const out = {};\n const missing = [];\n const invalid = [];\n\n for (const entry of spec) {\n const raw = source[entry.key];\n let value = raw === undefined || raw === null || raw === '' ? entry.default : raw;\n\n if (value === undefined || value === null || value === '') {\n if (entry.required) missing.push(entry.key);\n out[entry.name] = entry.default ?? null;\n continue;\n }\n\n switch (entry.type) {\n case 'number': {\n const n = Number(value);\n if (!Number.isFinite(n)) invalid.push(`${entry.key} must be a number (got \"${value}\")`);\n value = n;\n break;\n }\n case 'boolean':\n value = value === true || String(value).trim().toLowerCase() === 'true' || String(value) === '1';\n break;\n case 'json':\n if (typeof value === 'string') {\n try {\n value = JSON.parse(value);\n } catch {\n invalid.push(`${entry.key} must be valid JSON (got \"${String(value).slice(0, 40)}\")`);\n value = entry.default;\n }\n }\n break;\n case 'list':\n value = String(value)\n .split(',')\n .map((v) => v.trim())\n .filter(Boolean);\n break;\n case 'url':\n value = String(value).trim().replace(/\\/+$/, '');\n if (!/^https?:\\/\\//i.test(value)) {\n invalid.push(`${entry.key} must start with https:// (got \"${value}\")`);\n }\n break;\n default:\n value = String(value).trim();\n }\n\n out[entry.name] = value;\n }\n\n const problems = [];\n if (missing.length) problems.push(`missing: ${missing.join(', ')}`);\n if (invalid.length) problems.push(invalid.join('; '));\n if (problems.length) {\n throw new Error(\n `DocuQuery configuration problem - ${problems.join(' | ')}. ` +\n `Set these in your n8n environment (see .env.example) and restart n8n.`,\n );\n }\n\n return out;\n}\n\n/**\n * n8n hides `$env` when N8N_BLOCK_ENV_ACCESS_IN_NODE is on. Detect it here so\n * the failure names the setting instead of surfacing as \"$env is not defined\".\n */\nfunction requireEnv(env) {\n if (!env || typeof env !== 'object') {\n throw new Error(\n 'This node cannot read environment variables. Set ' +\n 'N8N_BLOCK_ENV_ACCESS_IN_NODE=false on the n8n container and restart it.',\n );\n }\n return env;\n}\n\nconst INGESTION_CONFIG_SPEC = [\n { key: 'DRIVE_FOLDER_ID', name: 'driveFolderId', required: true },\n { key: 'PINECONE_INDEX_HOST', name: 'pineconeHost', required: true, type: 'url' },\n { key: 'PINECONE_NAMESPACE', name: 'pineconeNamespace', default: 'docuquery' },\n { key: 'PINECONE_DIMENSION', name: 'pineconeDimension', type: 'number', default: 1536 },\n { key: 'EMBEDDING_MODEL', name: 'embeddingModel', default: 'text-embedding-3-small' },\n { key: 'EMBED_BATCH_SIZE', name: 'embedBatchSize', type: 'number', default: 96 },\n { key: 'CHUNK_TOKENS', name: 'chunkTokens', type: 'number', default: 500 },\n { key: 'CHUNK_OVERLAP_TOKENS', name: 'overlapTokens', type: 'number', default: 50 },\n { key: 'AIRTABLE_BASE_ID', name: 'airtableBaseId', required: true },\n { key: 'AIRTABLE_DOCUMENTS_TABLE', name: 'documentsTable', default: 'Documents' },\n { key: 'MAX_INGEST_RETRIES', name: 'maxRetries', type: 'number', default: 3 },\n { key: 'DEPARTMENTS', name: 'departments', type: 'list', default: 'General' },\n { key: 'DEFAULT_DEPARTMENT', name: 'defaultDepartment', default: 'General' },\n { key: 'DEPARTMENT_FOLDER_MAP', name: 'departmentFolderMap', type: 'json', default: {} },\n];\n\nconst QUERY_CONFIG_SPEC = [\n { key: 'PINECONE_INDEX_HOST', name: 'pineconeHost', required: true, type: 'url' },\n { key: 'PINECONE_NAMESPACE', name: 'pineconeNamespace', default: 'docuquery' },\n { key: 'EMBEDDING_MODEL', name: 'embeddingModel', default: 'text-embedding-3-small' },\n { key: 'LLM_PROVIDER', name: 'llmProvider', default: 'openai' },\n { key: 'LLM_MODEL', name: 'llmModel', default: 'gpt-4o' },\n { key: 'LLM_MAX_TOKENS', name: 'llmMaxTokens', type: 'number', default: 900 },\n { key: 'LLM_TEMPERATURE', name: 'llmTemperature', type: 'number', default: 0 },\n { key: 'LLM_BASE_URL', name: 'llmBaseUrl', default: '' },\n { key: 'RETRIEVAL_TOP_K', name: 'topK', type: 'number', default: 8 },\n { key: 'RETRIEVAL_MIN_SCORE', name: 'minScore', type: 'number', default: 0.32 },\n { key: 'RETRIEVAL_MAX_PER_DOC', name: 'maxPerDoc', type: 'number', default: 4 },\n { key: 'RETRIEVAL_MAX_CONTEXT_TOKENS', name: 'maxContextTokens', type: 'number', default: 6000 },\n { key: 'DEPARTMENTS', name: 'departments', type: 'list', default: 'General' },\n { key: 'SLACK_SIGNING_SECRET', name: 'slackSigningSecret', default: '' },\n { key: 'SLACK_TEAM_ID', name: 'slackTeamId', default: '' },\n { key: 'SLACK_ALLOW_UNVERIFIED', name: 'slackAllowUnverified', type: 'boolean', default: false },\n { key: 'QUERY_API_TOKEN', name: 'queryApiToken', default: '' },\n];\n// --- end src/lib/config.js\n\n// --- inlined from src/lib/departments.js ----------------------\n/**\n * Where a document's `department` metadata tag comes from.\n *\n * Resolution order, first hit wins:\n * 1. DEPARTMENT_FOLDER_MAP - a Drive sub-folder id pinned to a department.\n * 2. A bracketed filename prefix: \"[HR] Leave Policy.pdf\"\n * 3. A dashed filename prefix that matches a known department:\n * \"Finance - Expense SOP.pdf\"\n * 4. DEFAULT_DEPARTMENT.\n *\n * This tag is what makes \"/ask dept:HR ...\" able to filter at the vector level\n * instead of retrieving across everything and hoping.\n */\n\nfunction canonicaliseDepartment(value, known = []) {\n const raw = String(value ?? '').trim();\n if (!raw) return null;\n const match = known.find((k) => k.toLowerCase() === raw.toLowerCase());\n if (match) return match;\n return raw\n .split(/\\s+/)\n .map((w) => (w.length <= 3 ? w.toUpperCase() : w[0].toUpperCase() + w.slice(1).toLowerCase()))\n .join(' ');\n}\n\nfunction resolveDepartment({\n filename = '',\n parents = [],\n folderMap = {},\n known = [],\n fallback = 'General',\n} = {}) {\n for (const parent of parents ?? []) {\n if (folderMap && folderMap[parent]) {\n return canonicaliseDepartment(folderMap[parent], known);\n }\n }\n\n const bracketed = /^\\s*[[(]\\s*([A-Za-z][A-Za-z &/-]{0,30}?)\\s*[\\])]\\s*/.exec(filename);\n if (bracketed) return canonicaliseDepartment(bracketed[1], known);\n\n const dashed = /^\\s*([A-Za-z][A-Za-z &]{1,24}?)\\s+[-–]\\s+\\S/.exec(filename);\n if (dashed) {\n const candidate = dashed[1].trim();\n if (known.some((k) => k.toLowerCase() === candidate.toLowerCase())) {\n return canonicaliseDepartment(candidate, known);\n }\n }\n\n return canonicaliseDepartment(fallback, known) ?? 'General';\n}\n\n/** Strip a \"[HR] \" / \"Finance - \" prefix for a cleaner citation label. */\nfunction displayTitle(filename = '') {\n return String(filename)\n .replace(/\\.pdf$/i, '')\n .replace(/^\\s*[[(]\\s*[A-Za-z][A-Za-z &/-]{0,30}?\\s*[\\])]\\s*/, '')\n .replace(/[_]+/g, ' ')\n .trim();\n}\n// --- end src/lib/departments.js\n\n/**\n * Node: \"Normalize File Event\"\n *\n * Three entry points reach this node - the two Drive triggers, and the folder\n * listing used by the manual backfill and the scheduled sweep. They all carry\n * a Drive file resource, but not the same subset of fields, so normalise once\n * here and let everything downstream depend on one shape.\n *\n * Non-PDFs are dropped rather than failed: someone putting a spreadsheet in the\n * watched folder is not an error condition.\n */\n\nconst cfg = readConfig(requireEnv($env), INGESTION_CONFIG_SPEC);\n\nconst out = [];\nconst skipped = [];\n\nfor (const item of $input.all()) {\n const file = item.json ?? {};\n const docId = file.id ?? file.fileId;\n if (!docId) continue;\n\n const filename = String(file.name ?? file.originalFilename ?? 'untitled.pdf');\n const mimeType = String(file.mimeType ?? '');\n const isPdf = mimeType === 'application/pdf' || /\\.pdf$/i.test(filename);\n\n if (!isPdf) {\n skipped.push(`${filename} (${mimeType || 'unknown type'})`);\n continue;\n }\n if (file.trashed === true) {\n skipped.push(`${filename} (in trash)`);\n continue;\n }\n\n const parents = Array.isArray(file.parents) ? file.parents : [];\n\n out.push({\n json: {\n doc_id: String(docId),\n filename,\n title: displayTitle(filename),\n mime_type: mimeType || 'application/pdf',\n size: Number(file.size ?? 0) || null,\n drive_md5: file.md5Checksum ?? null,\n web_view_link: file.webViewLink ?? `https://drive.google.com/file/d/${docId}/view`,\n created_time: file.createdTime ?? null,\n modified_time: file.modifiedTime ?? null,\n upload_date: String(file.createdTime ?? file.modifiedTime ?? new Date().toISOString()).slice(0, 10),\n parents,\n department: resolveDepartment({\n filename,\n parents,\n folderMap: cfg.departmentFolderMap,\n known: cfg.departments,\n fallback: cfg.defaultDepartment,\n }),\n embedding_model: cfg.embeddingModel,\n index_namespace: cfg.pineconeNamespace,\n },\n });\n}\n\nif (skipped.length) {\n console.log(`Skipped ${skipped.length} non-PDF file(s): ${skipped.join(', ')}`);\n}\n\nreturn out;\n" }, "id": "fbb69d89-ed27-4837-a398-ee7a615b9d29", "name": "Normalize File Event", "type": "n8n-nodes-base.code", "typeVersion": 2, "position": [ 1200, 285 ], "notes": "One shape for all entry points. Drops non-PDFs and trashed files." }, { "parameters": { "mode": "runOnceForAllItems", "jsCode": "// --- inlined from src/lib/airtable.js -------------------------\n/**\n * Airtable helpers for the ingestion state table.\n *\n * The pipeline talks to Airtable over plain HTTP rather than through the\n * Airtable node, so that the record shape lives here in reviewable code instead\n * of inside a node's resource-mapper UI state.\n */\n\n/** Airtable formula strings are single-quoted; escape backslashes then quotes. */\nfunction escapeFormulaValue(value) {\n return String(value ?? '')\n .replace(/\\\\/g, '\\\\\\\\')\n .replace(/'/g, \"\\\\'\");\n}\n\nfunction lookupFormula(docIds) {\n const ids = [...new Set((docIds ?? []).filter(Boolean).map(String))];\n if (!ids.length) return { formula: 'FALSE()', doc_ids: [] };\n const clauses = ids.map((id) => `{doc_id} = '${escapeFormulaValue(id)}'`);\n return {\n formula: clauses.length === 1 ? clauses[0] : `OR(${clauses.join(', ')})`,\n doc_ids: ids,\n };\n}\n\n/** Airtable v2 responses nest under `fields`; flatten and keep the record id. */\nfunction flattenRecords(response) {\n const records = response?.records ?? (Array.isArray(response) ? response : []);\n return records.map((record) => ({\n record_id: record.id ?? null,\n ...(record.fields ?? record),\n }));\n}\n\nfunction indexByDocId(records) {\n const map = new Map();\n for (const record of records) {\n if (record.doc_id) map.set(String(record.doc_id), record);\n }\n return map;\n}\n\n/** Fields written when a document is successfully indexed. */\nfunction indexedFields(doc, nowIso) {\n return {\n doc_id: doc.doc_id,\n filename: doc.filename,\n drive_url: doc.web_view_link ?? null,\n department: doc.department ?? null,\n drive_md5: doc.drive_md5 ?? null,\n content_hash: doc.content_hash ?? null,\n status: 'Indexed',\n chunk_count: doc.chunk_count ?? 0,\n page_count: doc.page_count ?? 0,\n char_count: doc.char_count ?? 0,\n embedding_model: doc.embedding_model ?? null,\n index_namespace: doc.index_namespace ?? null,\n upload_date: doc.upload_date ? String(doc.upload_date).slice(0, 10) : null,\n modified_time: doc.modified_time ?? null,\n last_indexed: nowIso,\n last_checked: nowIso,\n retry_count: 0,\n error_notes: '',\n };\n}\n\n/** A document seen but deliberately not re-embedded. */\nfunction skippedFields(doc, nowIso) {\n return {\n doc_id: doc.doc_id,\n filename: doc.filename,\n drive_url: doc.web_view_link ?? null,\n department: doc.department ?? null,\n drive_md5: doc.drive_md5 ?? null,\n status: doc.existing?.status === 'Indexed' ? 'Indexed' : 'Skipped',\n last_checked: nowIso,\n };\n}\n\nfunction failedFields(doc, error, nowIso) {\n return {\n doc_id: doc.doc_id,\n filename: doc.filename ?? null,\n drive_url: doc.web_view_link ?? null,\n department: doc.department ?? null,\n drive_md5: doc.drive_md5 ?? null,\n status: 'Failed',\n retry_count: Number(doc.retry_count ?? 0) + 1,\n error_notes: String(error ?? 'unknown error').slice(0, 5000),\n last_checked: nowIso,\n };\n}\n\nfunction logFields({ docId, filename, event, detail, executionId, workflow }, nowIso) {\n return {\n timestamp: nowIso,\n doc_id: docId ?? null,\n filename: filename ?? null,\n event,\n detail: String(detail ?? '').slice(0, 5000),\n execution_id: executionId ?? null,\n workflow: workflow ?? null,\n };\n}\n\n/** Body for Airtable's upsert-on-PATCH. `typecast` lets new select options through. */\nfunction upsertBody(fieldsList, mergeOn = ['doc_id']) {\n return {\n performUpsert: { fieldsToMergeOn: mergeOn },\n typecast: true,\n records: fieldsList.map((fields) => ({ fields })),\n };\n}\n\nfunction createBody(fieldsList) {\n return { typecast: true, records: fieldsList.map((fields) => ({ fields })) };\n}\n// --- end src/lib/airtable.js\n\n// --- inlined from src/lib/vectors.js --------------------------\n/**\n * Pinecone vector construction, batching, and stale-vector cleanup.\n *\n * Vector ids are deterministic: `{doc_id}#{chunk_index}`. Two consequences the\n * pipeline depends on:\n * - Re-indexing a document overwrites its vectors rather than duplicating\n * them, because the ids collide by design.\n * - When a revised document produces *fewer* chunks than before, the leftover\n * tail ids can be computed and deleted without listing the index. (Delete\n * by metadata filter is not supported on Pinecone serverless indexes, so\n * computing ids is not just an optimisation - it is the supported route.)\n */\n\nconst UPSERT_BATCH_SIZE = 100;\nconst DELETE_BATCH_SIZE = 1000;\nconst MAX_METADATA_TEXT = 8000;\n\nfunction vectorId(docId, chunkIndex) {\n return `${docId}#${chunkIndex}`;\n}\n\n/** Pinecone rejects null metadata values, so drop rather than send them. */\nfunction compact(metadata) {\n const out = {};\n for (const [key, value] of Object.entries(metadata)) {\n if (value === null || value === undefined || value === '') continue;\n if (Array.isArray(value) && !value.length) continue;\n out[key] = value;\n }\n return out;\n}\n\n/**\n * `upload_ts` exists alongside `upload_date` because Pinecone can only do\n * range comparisons ($gte/$lt) on numbers - a date string supports equality\n * filtering and nothing else.\n */\nfunction buildVector(doc, chunk, embedding) {\n const uploadDate = doc.upload_date ? String(doc.upload_date).slice(0, 10) : null;\n const uploadTs = uploadDate ? Math.floor(new Date(`${uploadDate}T00:00:00Z`).getTime() / 1000) : null;\n\n return {\n id: vectorId(doc.doc_id, chunk.chunk_index),\n values: embedding,\n metadata: compact({\n doc_id: doc.doc_id,\n filename: doc.filename,\n title: doc.title ?? doc.filename,\n department: doc.department,\n drive_url: doc.web_view_link,\n upload_date: uploadDate,\n upload_ts: Number.isFinite(uploadTs) ? uploadTs : null,\n chunk_index: chunk.chunk_index,\n chunk_count: doc.chunk_count,\n page_start: chunk.page_start,\n page_end: chunk.page_end,\n section: chunk.section,\n heading_path: chunk.heading_path,\n content_hash: doc.content_hash,\n text: String(chunk.text ?? '').slice(0, MAX_METADATA_TEXT),\n }),\n };\n}\n\nfunction batch(items, size) {\n const out = [];\n for (let i = 0; i < items.length; i += size) out.push(items.slice(i, i + size));\n return out;\n}\n\n/**\n * Ids left behind when a document shrinks. The `+ margin` tail covers the case\n * where the previous chunk count was never recorded (a run that died mid-way),\n * at the cost of a few no-op deletes - Pinecone ignores ids that do not exist.\n */\nfunction staleVectorIds(docId, newChunkCount, previousChunkCount = 0, margin = 25) {\n const upper = Math.max(Number(previousChunkCount) || 0, newChunkCount + margin);\n const ids = [];\n for (let i = newChunkCount; i < upper; i += 1) ids.push(vectorId(docId, i));\n return ids;\n}\n\nfunction assertDimensions(vectors, expected) {\n if (!expected) return;\n const actual = vectors[0]?.values?.length;\n if (actual && actual !== Number(expected)) {\n throw new Error(\n `Embedding dimension mismatch: the model returned ${actual}-dimensional vectors but ` +\n `PINECONE_DIMENSION is ${expected}. Recreate the index with dimension ${actual}, ` +\n `or switch EMBEDDING_MODEL back to one that emits ${expected} dimensions.`,\n );\n }\n}\n// --- end src/lib/vectors.js\n\n/**\n * Node: \"Build Airtable Lookup\"\n *\n * One filterByFormula covering every file in this run, instead of one Airtable\n * request per file. A folder sweep over 200 documents becomes two calls.\n *\n * Batched at 100 because that is Airtable's page size: a group of at most 100\n * doc_ids can match at most 100 records, so no result ever needs pagination.\n */\n\nconst LOOKUP_BATCH_SIZE = 100;\n\nconst docIds = [...new Set($input.all().map((item) => item.json.doc_id).filter(Boolean))];\n\nif (!docIds.length) return [{ json: { ...lookupFormula([]), count: 0, batch_index: 0 } }];\n\nreturn batch(docIds, LOOKUP_BATCH_SIZE).map((group, index) => {\n const { formula, doc_ids: ids } = lookupFormula(group);\n return { json: { formula, doc_ids: ids, count: ids.length, batch_index: index } };\n});\n" }, "id": "a77d1f57-6fb1-4591-af35-0699e9d89cc2", "name": "Build Airtable Lookup", "type": "n8n-nodes-base.code", "typeVersion": 2, "position": [ 1500, 285 ] }, { "parameters": { "options": { "timeout": 30000 }, "method": "POST", "url": "=https://api.airtable.com/v0/{{ $env.AIRTABLE_BASE_ID }}/{{ encodeURIComponent($env.AIRTABLE_DOCUMENTS_TABLE || \"Documents\") }}/listRecords", "authentication": "predefinedCredentialType", "nodeCredentialType": "airtableTokenApi", "sendBody": true, "specifyBody": "json", "jsonBody": "={{ JSON.stringify({ filterByFormula: $json.formula, pageSize: 100 }) }}" }, "id": "5880a061-3638-44c1-ab3e-f96ebd6fd72f", "name": "Airtable: Lookup Documents", "type": "n8n-nodes-base.httpRequest", "typeVersion": 4.2, "position": [ 1800, 285 ], "notes": "POST /listRecords rather than GET: a folder sweep builds a formula far longer than a URL can carry.", "retryOnFail": true, "maxTries": 3, "waitBetweenTries": 2000, "alwaysOutputData": true }, { "parameters": { "mode": "runOnceForAllItems", "jsCode": "// --- inlined from src/lib/config.js ---------------------------\n/**\n * Environment reading + validation shared by every workflow.\n *\n * Workflows fail fast with one readable message naming the missing variable,\n * rather than throwing \"undefined is not a string\" six nodes downstream.\n */\n\n/**\n * @param {object} env Usually n8n's `$env`.\n * @param {object[]} spec [{ key, name, required, default, type }]\n */\nfunction readConfig(env, spec) {\n const source = env ?? {};\n const out = {};\n const missing = [];\n const invalid = [];\n\n for (const entry of spec) {\n const raw = source[entry.key];\n let value = raw === undefined || raw === null || raw === '' ? entry.default : raw;\n\n if (value === undefined || value === null || value === '') {\n if (entry.required) missing.push(entry.key);\n out[entry.name] = entry.default ?? null;\n continue;\n }\n\n switch (entry.type) {\n case 'number': {\n const n = Number(value);\n if (!Number.isFinite(n)) invalid.push(`${entry.key} must be a number (got \"${value}\")`);\n value = n;\n break;\n }\n case 'boolean':\n value = value === true || String(value).trim().toLowerCase() === 'true' || String(value) === '1';\n break;\n case 'json':\n if (typeof value === 'string') {\n try {\n value = JSON.parse(value);\n } catch {\n invalid.push(`${entry.key} must be valid JSON (got \"${String(value).slice(0, 40)}\")`);\n value = entry.default;\n }\n }\n break;\n case 'list':\n value = String(value)\n .split(',')\n .map((v) => v.trim())\n .filter(Boolean);\n break;\n case 'url':\n value = String(value).trim().replace(/\\/+$/, '');\n if (!/^https?:\\/\\//i.test(value)) {\n invalid.push(`${entry.key} must start with https:// (got \"${value}\")`);\n }\n break;\n default:\n value = String(value).trim();\n }\n\n out[entry.name] = value;\n }\n\n const problems = [];\n if (missing.length) problems.push(`missing: ${missing.join(', ')}`);\n if (invalid.length) problems.push(invalid.join('; '));\n if (problems.length) {\n throw new Error(\n `DocuQuery configuration problem - ${problems.join(' | ')}. ` +\n `Set these in your n8n environment (see .env.example) and restart n8n.`,\n );\n }\n\n return out;\n}\n\n/**\n * n8n hides `$env` when N8N_BLOCK_ENV_ACCESS_IN_NODE is on. Detect it here so\n * the failure names the setting instead of surfacing as \"$env is not defined\".\n */\nfunction requireEnv(env) {\n if (!env || typeof env !== 'object') {\n throw new Error(\n 'This node cannot read environment variables. Set ' +\n 'N8N_BLOCK_ENV_ACCESS_IN_NODE=false on the n8n container and restart it.',\n );\n }\n return env;\n}\n\nconst INGESTION_CONFIG_SPEC = [\n { key: 'DRIVE_FOLDER_ID', name: 'driveFolderId', required: true },\n { key: 'PINECONE_INDEX_HOST', name: 'pineconeHost', required: true, type: 'url' },\n { key: 'PINECONE_NAMESPACE', name: 'pineconeNamespace', default: 'docuquery' },\n { key: 'PINECONE_DIMENSION', name: 'pineconeDimension', type: 'number', default: 1536 },\n { key: 'EMBEDDING_MODEL', name: 'embeddingModel', default: 'text-embedding-3-small' },\n { key: 'EMBED_BATCH_SIZE', name: 'embedBatchSize', type: 'number', default: 96 },\n { key: 'CHUNK_TOKENS', name: 'chunkTokens', type: 'number', default: 500 },\n { key: 'CHUNK_OVERLAP_TOKENS', name: 'overlapTokens', type: 'number', default: 50 },\n { key: 'AIRTABLE_BASE_ID', name: 'airtableBaseId', required: true },\n { key: 'AIRTABLE_DOCUMENTS_TABLE', name: 'documentsTable', default: 'Documents' },\n { key: 'MAX_INGEST_RETRIES', name: 'maxRetries', type: 'number', default: 3 },\n { key: 'DEPARTMENTS', name: 'departments', type: 'list', default: 'General' },\n { key: 'DEFAULT_DEPARTMENT', name: 'defaultDepartment', default: 'General' },\n { key: 'DEPARTMENT_FOLDER_MAP', name: 'departmentFolderMap', type: 'json', default: {} },\n];\n\nconst QUERY_CONFIG_SPEC = [\n { key: 'PINECONE_INDEX_HOST', name: 'pineconeHost', required: true, type: 'url' },\n { key: 'PINECONE_NAMESPACE', name: 'pineconeNamespace', default: 'docuquery' },\n { key: 'EMBEDDING_MODEL', name: 'embeddingModel', default: 'text-embedding-3-small' },\n { key: 'LLM_PROVIDER', name: 'llmProvider', default: 'openai' },\n { key: 'LLM_MODEL', name: 'llmModel', default: 'gpt-4o' },\n { key: 'LLM_MAX_TOKENS', name: 'llmMaxTokens', type: 'number', default: 900 },\n { key: 'LLM_TEMPERATURE', name: 'llmTemperature', type: 'number', default: 0 },\n { key: 'LLM_BASE_URL', name: 'llmBaseUrl', default: '' },\n { key: 'RETRIEVAL_TOP_K', name: 'topK', type: 'number', default: 8 },\n { key: 'RETRIEVAL_MIN_SCORE', name: 'minScore', type: 'number', default: 0.32 },\n { key: 'RETRIEVAL_MAX_PER_DOC', name: 'maxPerDoc', type: 'number', default: 4 },\n { key: 'RETRIEVAL_MAX_CONTEXT_TOKENS', name: 'maxContextTokens', type: 'number', default: 6000 },\n { key: 'DEPARTMENTS', name: 'departments', type: 'list', default: 'General' },\n { key: 'SLACK_SIGNING_SECRET', name: 'slackSigningSecret', default: '' },\n { key: 'SLACK_TEAM_ID', name: 'slackTeamId', default: '' },\n { key: 'SLACK_ALLOW_UNVERIFIED', name: 'slackAllowUnverified', type: 'boolean', default: false },\n { key: 'QUERY_API_TOKEN', name: 'queryApiToken', default: '' },\n];\n// --- end src/lib/config.js\n\n// --- inlined from src/lib/airtable.js -------------------------\n/**\n * Airtable helpers for the ingestion state table.\n *\n * The pipeline talks to Airtable over plain HTTP rather than through the\n * Airtable node, so that the record shape lives here in reviewable code instead\n * of inside a node's resource-mapper UI state.\n */\n\n/** Airtable formula strings are single-quoted; escape backslashes then quotes. */\nfunction escapeFormulaValue(value) {\n return String(value ?? '')\n .replace(/\\\\/g, '\\\\\\\\')\n .replace(/'/g, \"\\\\'\");\n}\n\nfunction lookupFormula(docIds) {\n const ids = [...new Set((docIds ?? []).filter(Boolean).map(String))];\n if (!ids.length) return { formula: 'FALSE()', doc_ids: [] };\n const clauses = ids.map((id) => `{doc_id} = '${escapeFormulaValue(id)}'`);\n return {\n formula: clauses.length === 1 ? clauses[0] : `OR(${clauses.join(', ')})`,\n doc_ids: ids,\n };\n}\n\n/** Airtable v2 responses nest under `fields`; flatten and keep the record id. */\nfunction flattenRecords(response) {\n const records = response?.records ?? (Array.isArray(response) ? response : []);\n return records.map((record) => ({\n record_id: record.id ?? null,\n ...(record.fields ?? record),\n }));\n}\n\nfunction indexByDocId(records) {\n const map = new Map();\n for (const record of records) {\n if (record.doc_id) map.set(String(record.doc_id), record);\n }\n return map;\n}\n\n/** Fields written when a document is successfully indexed. */\nfunction indexedFields(doc, nowIso) {\n return {\n doc_id: doc.doc_id,\n filename: doc.filename,\n drive_url: doc.web_view_link ?? null,\n department: doc.department ?? null,\n drive_md5: doc.drive_md5 ?? null,\n content_hash: doc.content_hash ?? null,\n status: 'Indexed',\n chunk_count: doc.chunk_count ?? 0,\n page_count: doc.page_count ?? 0,\n char_count: doc.char_count ?? 0,\n embedding_model: doc.embedding_model ?? null,\n index_namespace: doc.index_namespace ?? null,\n upload_date: doc.upload_date ? String(doc.upload_date).slice(0, 10) : null,\n modified_time: doc.modified_time ?? null,\n last_indexed: nowIso,\n last_checked: nowIso,\n retry_count: 0,\n error_notes: '',\n };\n}\n\n/** A document seen but deliberately not re-embedded. */\nfunction skippedFields(doc, nowIso) {\n return {\n doc_id: doc.doc_id,\n filename: doc.filename,\n drive_url: doc.web_view_link ?? null,\n department: doc.department ?? null,\n drive_md5: doc.drive_md5 ?? null,\n status: doc.existing?.status === 'Indexed' ? 'Indexed' : 'Skipped',\n last_checked: nowIso,\n };\n}\n\nfunction failedFields(doc, error, nowIso) {\n return {\n doc_id: doc.doc_id,\n filename: doc.filename ?? null,\n drive_url: doc.web_view_link ?? null,\n department: doc.department ?? null,\n drive_md5: doc.drive_md5 ?? null,\n status: 'Failed',\n retry_count: Number(doc.retry_count ?? 0) + 1,\n error_notes: String(error ?? 'unknown error').slice(0, 5000),\n last_checked: nowIso,\n };\n}\n\nfunction logFields({ docId, filename, event, detail, executionId, workflow }, nowIso) {\n return {\n timestamp: nowIso,\n doc_id: docId ?? null,\n filename: filename ?? null,\n event,\n detail: String(detail ?? '').slice(0, 5000),\n execution_id: executionId ?? null,\n workflow: workflow ?? null,\n };\n}\n\n/** Body for Airtable's upsert-on-PATCH. `typecast` lets new select options through. */\nfunction upsertBody(fieldsList, mergeOn = ['doc_id']) {\n return {\n performUpsert: { fieldsToMergeOn: mergeOn },\n typecast: true,\n records: fieldsList.map((fields) => ({ fields })),\n };\n}\n\nfunction createBody(fieldsList) {\n return { typecast: true, records: fieldsList.map((fields) => ({ fields })) };\n}\n// --- end src/lib/airtable.js\n\n/**\n * Node: \"Decide Action\"\n *\n * Tier one of duplicate-ingestion protection: compare Drive's own md5 checksum\n * against the one recorded at last index. Google Drive triggers fire on\n * metadata-only changes too (a rename, a sharing change, a comment), so without\n * this every trivial edit would re-embed a whole document.\n *\n * Tier two - hashing the extracted text - happens after extraction in \"Chunk\n * Document\", and catches the reverse case: a file whose bytes changed but whose\n * text did not (re-export, re-compression, metadata rewrite).\n *\n * A document that has failed MAX_INGEST_RETRIES times at the same checksum is\n * left alone. Bounded retries; a corrupt file cannot loop forever.\n */\n\nconst cfg = readConfig(requireEnv($env), INGESTION_CONFIG_SPEC);\n\nconst files = $('Normalize File Event').all().map((item) => item.json);\n// The lookup is batched at 100 ids per request, so there may be several\n// responses to merge.\nconst existingByDoc = indexByDocId(\n $input.all().flatMap((item) => flattenRecords(item.json)),\n);\n\nconst decisions = files.map((file) => {\n const existing = existingByDoc.get(file.doc_id) ?? null;\n const retryCount = Number(existing?.retry_count ?? 0);\n const status = String(existing?.status ?? '').toLowerCase();\n const sameChecksum = Boolean(\n file.drive_md5 && existing?.drive_md5 && file.drive_md5 === existing.drive_md5,\n );\n\n let action = 'index';\n let reason = 'not previously indexed';\n\n if (existing) {\n if (sameChecksum && status === 'indexed') {\n action = 'skip';\n reason = 'Drive checksum unchanged since last index';\n } else if (sameChecksum && status === 'failed' && retryCount >= cfg.maxRetries) {\n action = 'skip';\n reason = `failed ${retryCount} times on identical content; giving up until the file changes`;\n } else if (sameChecksum) {\n action = 'reindex';\n reason = `previous run left status \"${existing.status ?? 'unknown'}\"`;\n } else {\n action = 'reindex';\n reason = existing.drive_md5 ? 'Drive checksum changed' : 'no checksum on record';\n }\n }\n\n return {\n json: {\n ...file,\n existing,\n existing_record_id: existing?.record_id ?? null,\n previous_chunk_count: Number(existing?.chunk_count ?? 0),\n retry_count: retryCount,\n action,\n reason,\n needs_processing: action !== 'skip',\n run_id: $execution.id,\n },\n };\n});\n\nconst counts = decisions.reduce((acc, d) => {\n acc[d.json.action] = (acc[d.json.action] ?? 0) + 1;\n return acc;\n}, {});\nconsole.log(`Decided ${decisions.length} document(s): ${JSON.stringify(counts)}`);\n\nreturn decisions;\n" }, "id": "796973dd-d997-48f3-af9e-c4e0626ad8a7", "name": "Decide Action", "type": "n8n-nodes-base.code", "typeVersion": 2, "position": [ 2100, 285 ], "notes": "Duplicate protection, tier 1: compare Drive md5 against the last indexed checksum." }, { "parameters": { "batchSize": 1, "options": {} }, "id": "7d71acbd-d0fe-4848-a2ac-6fcc19da0e30", "name": "Loop Over Documents", "type": "n8n-nodes-base.splitInBatches", "typeVersion": 3, "position": [ 2400, 285 ], "notes": "One document per iteration, so a failure is isolated to that document." }, { "parameters": { "mode": "runOnceForAllItems", "jsCode": "/**\n * Node: \"Run Summary\"\n *\n * Runs once on the loop's \"done\" output. Produces the ops-channel message, and\n * decides whether there is anything worth saying: a sweep that finds nothing\n * changed should stay quiet, a sweep with failures should not.\n */\n\n/**\n * Referencing a node that never ran in this execution throws, and on a clean\n * run neither the failure branch nor (on an all-skipped sweep) the success\n * branch will have run. Treat \"never executed\" as \"nothing to report\".\n */\nfunction itemsFrom(nodeName) {\n try {\n return $(nodeName).all().map((item) => item.json);\n } catch {\n return [];\n }\n}\n\nconst decisions = itemsFrom('Decide Action');\n\nconst counts = { index: 0, reindex: 0, skip: 0 };\nfor (const decision of decisions) counts[decision.action] = (counts[decision.action] ?? 0) + 1;\n\nconst indexed = itemsFrom('Summarise Ingestion');\nconst failures = itemsFrom('Build Failure Record');\n\nconst chunks = indexed.reduce((total, doc) => total + Number(doc.chunk_count ?? 0), 0);\nconst vectors = indexed.reduce((total, doc) => total + Number(doc.vectors_upserted ?? 0), 0);\n\nconst lines = [\n `*DocuQuery ingestion run* - ${decisions.length} document(s) seen`,\n indexed.length\n ? `:white_check_mark: Indexed ${indexed.length}: ${indexed.map((d) => d.filename).join(', ')} (${chunks} chunks, ${vectors} vectors)`\n : null,\n counts.skip ? `:fast_forward: Skipped ${counts.skip} unchanged` : null,\n failures.length\n ? `:x: Failed ${failures.length}: ${failures.map((f) => `${f.filename} - ${f.error}`).join('; ')}`\n : null,\n];\n\nreturn [\n {\n json: {\n documents_seen: decisions.length,\n indexed: indexed.length,\n skipped: counts.skip ?? 0,\n failed: failures.length,\n chunks,\n vectors,\n // Stay silent on a no-op sweep; alert on anything else.\n notify: indexed.length > 0 || failures.length > 0,\n text: lines.filter(Boolean).join('\\n'),\n },\n },\n];\n" }, "id": "7257730b-f8cd-4f46-a67f-69be6344e8ca", "name": "Run Summary", "type": "n8n-nodes-base.code", "typeVersion": 2, "position": [ 2700, 0 ] }, { "parameters": { "conditions": { "options": { "caseSensitive": true, "leftValue": "", "typeValidation": "loose", "version": 2 }, "combinator": "and", "conditions": [ { "id": "af081670-0399-44bc-a437-582ef7f0c2df", "leftValue": "={{ $json.notify }}", "rightValue": "", "operator": { "type": "boolean", "operation": "true", "singleValue": true } } ] }, "looseTypeValidation": true, "options": {} }, "id": "066496d9-39f5-491a-aa31-f243502349e7", "name": "Notify?", "type": "n8n-nodes-base.if", "typeVersion": 2.2, "position": [ 3000, 0 ], "notes": "A sweep that changed nothing stays quiet." }, { "parameters": { "options": { "timeout": 15000 }, "method": "POST", "url": "https://slack.com/api/chat.postMessage", "authentication": "predefinedCredentialType", "nodeCredentialType": "slackApi", "sendBody": true, "specifyBody": "json", "jsonBody": "={{ JSON.stringify({ channel: $env.SLACK_OPS_CHANNEL, text: $json.text, unfurl_links: false }) }}" }, "id": "c440af78-9dc5-4e51-a64b-72affa5fc9e2", "name": "Slack: Post Run Summary", "type": "n8n-nodes-base.httpRequest", "typeVersion": 4.2, "position": [ 3300, 0 ], "retryOnFail": true, "maxTries": 2, "waitBetweenTries": 2000, "onError": "continueRegularOutput", "notes": "Credential: Slack API (bot token, chat:write)" }, { "parameters": {}, "id": "0f512686-b247-427f-aa1c-28f70a6e37cc", "name": "Ingestion Complete", "type": "n8n-nodes-base.noOp", "typeVersion": 1, "position": [ 3600, 0 ] }, { "parameters": { "conditions": { "options": { "caseSensitive": true, "leftValue": "", "typeValidation": "loose", "version": 2 }, "combinator": "and", "conditions": [ { "id": "844b1024-cae3-4829-a200-51f7fe1cb6a6", "leftValue": "={{ $json.needs_processing }}", "rightValue": "", "operator": { "type": "boolean", "operation": "true", "singleValue": true } } ] }, "looseTypeValidation": true, "options": {} }, "id": "f8ef6982-3383-4b2f-a6eb-19cb783b19fc", "name": "Needs Processing?", "type": "n8n-nodes-base.if", "typeVersion": 2.2, "position": [ 2700, 475 ] }, { "parameters": { "resource": "file", "operation": "download", "fileId": { "__rl": true, "value": "={{ $json.doc_id }}", "mode": "id" }, "options": { "binaryPropertyName": "data" } }, "id": "57459c0e-e719-4785-ac6f-8ca5e2d43edb", "name": "Drive: Download File", "type": "n8n-nodes-base.googleDrive", "typeVersion": 3, "position": [ 3000, 380 ], "onError": "continueErrorOutput", "retryOnFail": true, "maxTries": 3, "waitBetweenTries": 3000 }, { "parameters": { "operation": "pdf", "binaryPropertyName": "data", "options": { "joinPages": false } }, "id": "53f18064-1a7d-413b-a84c-3006f02ecb2f", "name": "Extract PDF Text", "type": "n8n-nodes-base.extractFromFile", "typeVersion": 1, "position": [ 3300, 380 ], "onError": "continueErrorOutput", "notes": "joinPages: false keeps page boundaries, which is where citation page numbers come from." }, { "parameters": { "mode": "runOnceForAllItems", "jsCode": "// --- inlined from src/lib/config.js ---------------------------\n/**\n * Environment reading + validation shared by every workflow.\n *\n * Workflows fail fast with one readable message naming the missing variable,\n * rather than throwing \"undefined is not a string\" six nodes downstream.\n */\n\n/**\n * @param {object} env Usually n8n's `$env`.\n * @param {object[]} spec [{ key, name, required, default, type }]\n */\nfunction readConfig(env, spec) {\n const source = env ?? {};\n const out = {};\n const missing = [];\n const invalid = [];\n\n for (const entry of spec) {\n const raw = source[entry.key];\n let value = raw === undefined || raw === null || raw === '' ? entry.default : raw;\n\n if (value === undefined || value === null || value === '') {\n if (entry.required) missing.push(entry.key);\n out[entry.name] = entry.default ?? null;\n continue;\n }\n\n switch (entry.type) {\n case 'number': {\n const n = Number(value);\n if (!Number.isFinite(n)) invalid.push(`${entry.key} must be a number (got \"${value}\")`);\n value = n;\n break;\n }\n case 'boolean':\n value = value === true || String(value).trim().toLowerCase() === 'true' || String(value) === '1';\n break;\n case 'json':\n if (typeof value === 'string') {\n try {\n value = JSON.parse(value);\n } catch {\n invalid.push(`${entry.key} must be valid JSON (got \"${String(value).slice(0, 40)}\")`);\n value = entry.default;\n }\n }\n break;\n case 'list':\n value = String(value)\n .split(',')\n .map((v) => v.trim())\n .filter(Boolean);\n break;\n case 'url':\n value = String(value).trim().replace(/\\/+$/, '');\n if (!/^https?:\\/\\//i.test(value)) {\n invalid.push(`${entry.key} must start with https:// (got \"${value}\")`);\n }\n break;\n default:\n value = String(value).trim();\n }\n\n out[entry.name] = value;\n }\n\n const problems = [];\n if (missing.length) problems.push(`missing: ${missing.join(', ')}`);\n if (invalid.length) problems.push(invalid.join('; '));\n if (problems.length) {\n throw new Error(\n `DocuQuery configuration problem - ${problems.join(' | ')}. ` +\n `Set these in your n8n environment (see .env.example) and restart n8n.`,\n );\n }\n\n return out;\n}\n\n/**\n * n8n hides `$env` when N8N_BLOCK_ENV_ACCESS_IN_NODE is on. Detect it here so\n * the failure names the setting instead of surfacing as \"$env is not defined\".\n */\nfunction requireEnv(env) {\n if (!env || typeof env !== 'object') {\n throw new Error(\n 'This node cannot read environment variables. Set ' +\n 'N8N_BLOCK_ENV_ACCESS_IN_NODE=false on the n8n container and restart it.',\n );\n }\n return env;\n}\n\nconst INGESTION_CONFIG_SPEC = [\n { key: 'DRIVE_FOLDER_ID', name: 'driveFolderId', required: true },\n { key: 'PINECONE_INDEX_HOST', name: 'pineconeHost', required: true, type: 'url' },\n { key: 'PINECONE_NAMESPACE', name: 'pineconeNamespace', default: 'docuquery' },\n { key: 'PINECONE_DIMENSION', name: 'pineconeDimension', type: 'number', default: 1536 },\n { key: 'EMBEDDING_MODEL', name: 'embeddingModel', default: 'text-embedding-3-small' },\n { key: 'EMBED_BATCH_SIZE', name: 'embedBatchSize', type: 'number', default: 96 },\n { key: 'CHUNK_TOKENS', name: 'chunkTokens', type: 'number', default: 500 },\n { key: 'CHUNK_OVERLAP_TOKENS', name: 'overlapTokens', type: 'number', default: 50 },\n { key: 'AIRTABLE_BASE_ID', name: 'airtableBaseId', required: true },\n { key: 'AIRTABLE_DOCUMENTS_TABLE', name: 'documentsTable', default: 'Documents' },\n { key: 'MAX_INGEST_RETRIES', name: 'maxRetries', type: 'number', default: 3 },\n { key: 'DEPARTMENTS', name: 'departments', type: 'list', default: 'General' },\n { key: 'DEFAULT_DEPARTMENT', name: 'defaultDepartment', default: 'General' },\n { key: 'DEPARTMENT_FOLDER_MAP', name: 'departmentFolderMap', type: 'json', default: {} },\n];\n\nconst QUERY_CONFIG_SPEC = [\n { key: 'PINECONE_INDEX_HOST', name: 'pineconeHost', required: true, type: 'url' },\n { key: 'PINECONE_NAMESPACE', name: 'pineconeNamespace', default: 'docuquery' },\n { key: 'EMBEDDING_MODEL', name: 'embeddingModel', default: 'text-embedding-3-small' },\n { key: 'LLM_PROVIDER', name: 'llmProvider', default: 'openai' },\n { key: 'LLM_MODEL', name: 'llmModel', default: 'gpt-4o' },\n { key: 'LLM_MAX_TOKENS', name: 'llmMaxTokens', type: 'number', default: 900 },\n { key: 'LLM_TEMPERATURE', name: 'llmTemperature', type: 'number', default: 0 },\n { key: 'LLM_BASE_URL', name: 'llmBaseUrl', default: '' },\n { key: 'RETRIEVAL_TOP_K', name: 'topK', type: 'number', default: 8 },\n { key: 'RETRIEVAL_MIN_SCORE', name: 'minScore', type: 'number', default: 0.32 },\n { key: 'RETRIEVAL_MAX_PER_DOC', name: 'maxPerDoc', type: 'number', default: 4 },\n { key: 'RETRIEVAL_MAX_CONTEXT_TOKENS', name: 'maxContextTokens', type: 'number', default: 6000 },\n { key: 'DEPARTMENTS', name: 'departments', type: 'list', default: 'General' },\n { key: 'SLACK_SIGNING_SECRET', name: 'slackSigningSecret', default: '' },\n { key: 'SLACK_TEAM_ID', name: 'slackTeamId', default: '' },\n { key: 'SLACK_ALLOW_UNVERIFIED', name: 'slackAllowUnverified', type: 'boolean', default: false },\n { key: 'QUERY_API_TOKEN', name: 'queryApiToken', default: '' },\n];\n// --- end src/lib/config.js\n\n// --- inlined from src/lib/hash.js -----------------------------\n/**\n * Content fingerprinting for duplicate-ingestion protection.\n *\n * Prefers node:crypto (needs NODE_FUNCTION_ALLOW_BUILTIN=crypto on the n8n\n * container). If that is unavailable the FNV-1a fallback still gives a stable\n * \"has this text changed?\" answer, which is all the pipeline needs - it is a\n * change detector, not a security primitive.\n */\n\nfunction fnv1a64(str) {\n const prime = 0x100000001b3n;\n const mask = 0xffffffffffffffffn;\n let hash = 0xcbf29ce484222325n;\n for (let i = 0; i < str.length; i += 1) {\n hash ^= BigInt(str.charCodeAt(i) & 0xffff);\n hash = (hash * prime) & mask;\n }\n return hash.toString(16).padStart(16, '0');\n}\n\nfunction sha256Hex(text, cryptoImpl) {\n const input = String(text ?? '');\n let lib = cryptoImpl;\n if (!lib) {\n try {\n // eslint-disable-next-line no-undef\n lib = require('crypto');\n } catch {\n lib = null;\n }\n }\n if (lib && typeof lib.createHash === 'function') {\n return `sha256:${lib.createHash('sha256').update(input, 'utf8').digest('hex')}`;\n }\n return `fnv1a:${fnv1a64(input)}:${input.length}`;\n}\n// --- end src/lib/hash.js\n\n// --- inlined from src/lib/chunker.js --------------------------\n/**\n * PDF text cleaning + semantic chunking.\n *\n * Runs inside the \"Chunk Document\" n8n Code node (inlined at build time) and is\n * unit-tested directly from test/chunker.test.js. Keep it dependency-free.\n *\n * The pipeline is: pages -> drop repeated headers/footers -> reflow wrapped\n * lines -> paragraph blocks tagged with page + heading -> pack into ~500-token\n * chunks that prefer to break at headings and carry an overlap tail.\n */\n\nconst CHUNK_DEFAULTS = {\n chunkTokens: 500,\n overlapTokens: 50,\n minChunkTokens: 40,\n charsPerToken: 4,\n boilerplateThreshold: 0.6,\n edgeLines: 3,\n maxHeadingLength: 90,\n};\n\n// Written as escapes so the source stays free of literal control bytes.\nconst NBSP_RE = new RegExp(String.fromCharCode(0xa0), 'g');\nconst CONTROL_RE = new RegExp(\n `[${Array.from({ length: 32 }, (_, i) => i)\n .filter((i) => i !== 9 && i !== 10 && i !== 13)\n .map((i) => String.fromCharCode(i))\n .join('')}]`,\n 'g',\n);\nconst NON_LETTERS_RE = /[^A-Za-z]/g;\n\nconst PAGE_NUMBER_RE =\n /^\\s*(?:-\\s*)?(?:page\\s*)?\\d{1,4}(?:\\s*(?:of|\\/)\\s*\\d{1,4})?(?:\\s*-)?\\s*$/i;\n\n/** Rough token estimate. Good enough for budgeting; never used for billing. */\nfunction estimateTokens(text, charsPerToken = CHUNK_DEFAULTS.charsPerToken) {\n const trimmed = String(text ?? '').trim();\n if (!trimmed) return 0;\n return Math.max(1, Math.ceil(trimmed.length / charsPerToken));\n}\n\n/**\n * Normalise whatever the Extract From File node handed us into a page array.\n * With `joinPages: false` it gives an array; older versions give one string, in\n * which case we can only recover page boundaries if form feeds survived.\n */\nfunction toPages(raw) {\n if (Array.isArray(raw)) {\n return raw.map((p) => (typeof p === 'string' ? p : String(p?.text ?? '')));\n }\n if (raw && typeof raw === 'object' && raw.text !== undefined) return toPages(raw.text);\n const text = String(raw ?? '');\n if (text.includes('\\f')) return text.split('\\f');\n return [text];\n}\n\nfunction normaliseForCompare(line) {\n return line.replace(/\\s+/g, ' ').trim().toLowerCase().replace(/\\d+/g, '#');\n}\n\n/**\n * Find header/footer lines that repeat across most pages (\"ACME Corp -\n * Confidential\", \"Employee Handbook v3\", \"Page 4 of 12\"). Only the first and\n * last few lines of each page are candidates, so body text that happens to\n * repeat is left alone.\n */\nfunction findBoilerplate(pages, options = {}) {\n const { boilerplateThreshold, edgeLines, maxHeadingLength } = {\n ...CHUNK_DEFAULTS,\n ...options,\n };\n // Two pages is the minimum useful evidence: a line on both is very likely a\n // running header. With one page there is nothing to compare against, and\n // guessing would strip the document title.\n if (pages.length < 2) return new Set();\n\n const counts = new Map();\n for (const page of pages) {\n const lines = page.split('\\n').map((l) => l.trim()).filter(Boolean);\n const edges = [...lines.slice(0, edgeLines), ...lines.slice(-edgeLines)];\n const seenOnPage = new Set();\n for (const line of edges) {\n if (line.length > maxHeadingLength) continue;\n const key = normaliseForCompare(line);\n if (!key || seenOnPage.has(key)) continue;\n seenOnPage.add(key);\n counts.set(key, (counts.get(key) ?? 0) + 1);\n }\n }\n\n const minHits = Math.max(2, Math.ceil(pages.length * boilerplateThreshold));\n return new Set(\n [...counts.entries()].filter(([, count]) => count >= minHits).map(([key]) => key),\n );\n}\n\nfunction isListItem(line) {\n return /^\\s*(?:[-–—•*·]|\\(?[a-z0-9]{1,3}[.)])\\s+\\S/i.test(line);\n}\n\nfunction isHeading(line, maxLength = CHUNK_DEFAULTS.maxHeadingLength) {\n const t = String(line ?? '').trim();\n if (!t || t.length > maxLength) return false;\n if (/[.,;]$/.test(t)) return false;\n // \"3.2 Eligibility\", \"4) Scope\"\n if (/^\\d+(?:\\.\\d+)*[.)]?\\s+\\S/.test(t)) return true;\n // \"Section 4\", \"Appendix B\", \"Article III\"\n if (/^(?:article|section|appendix|chapter|part|schedule|exhibit|annex)\\s+[\\divxlcm]/i.test(t)) {\n return true;\n }\n const letters = t.replace(NON_LETTERS_RE, '');\n if (letters.length >= 3 && t === t.toUpperCase() && !/[!?]$/.test(t)) return true;\n\n // Title case: a short line with no terminal punctuation, where most of the\n // substantial words are capitalised. The capitalisation ratio is what keeps\n // ordinary prose out - \"The company provides a laptop\" opens with a capital\n // and has no full stop, but only one of its words is capitalised, and\n // treating it as a heading would split a paragraph down the middle.\n if (\n /^[A-Z][A-Za-z0-9 ,&/’'()\\-]{2,60}$/.test(t) &&\n !/[.!?]$/.test(t) &&\n t.split(/\\s+/).length <= 9\n ) {\n const significant = t\n .split(/\\s+/)\n .filter((w) => w.replace(NON_LETTERS_RE, '').length > 3);\n if (!significant.length) return false;\n const capitalised = significant.filter((w) => /^[A-Z]/.test(w)).length;\n return capitalised / significant.length >= 0.6;\n }\n return false;\n}\n\n/** Depth for the heading breadcrumb: \"2.1.3 Foo\" is depth 3, ALL CAPS is depth 1. */\nfunction headingDepth(line) {\n const numbered = /^(\\d+(?:\\.\\d+)*)/.exec(line.trim());\n if (numbered) return numbered[1].split('.').length;\n const letters = line.replace(NON_LETTERS_RE, '');\n if (letters && line.trim() === line.trim().toUpperCase()) return 1;\n return 2;\n}\n\nfunction cleanPage(page, boilerplate, options) {\n const { edgeLines } = { ...CHUNK_DEFAULTS, ...options };\n const lines = page.split('\\n');\n const kept = [];\n lines.forEach((rawLine, index) => {\n const line = rawLine\n .replace(NBSP_RE, ' ')\n .replace(CONTROL_RE, '')\n .trimEnd();\n const trimmed = line.trim();\n const nearEdge = index < edgeLines || index >= lines.length - edgeLines;\n if (trimmed && PAGE_NUMBER_RE.test(trimmed)) return;\n if (trimmed && nearEdge && boilerplate.has(normaliseForCompare(trimmed))) return;\n kept.push(line);\n });\n return kept.join('\\n');\n}\n\n/** Re-join lines that PDF extraction wrapped mid-sentence. */\nfunction reflow(text) {\n const out = [];\n for (const rawLine of text.split('\\n')) {\n const line = rawLine.trim();\n if (!line) {\n if (out.length && out[out.length - 1] !== '') out.push('');\n continue;\n }\n const prev = out.length ? out[out.length - 1] : '';\n const continuation =\n prev &&\n prev !== '' &&\n !isHeading(prev) &&\n !isHeading(line) &&\n !isListItem(line) &&\n !/[.:;!?]$/.test(prev) &&\n /^[a-z(“\"'\\d]/.test(line);\n if (continuation) out[out.length - 1] = `${prev} ${line}`;\n else out.push(line);\n }\n return out.join('\\n').replace(/\\n{3,}/g, '\\n\\n').trim();\n}\n\nfunction dehyphenate(text) {\n return text.replace(/([A-Za-z])-\\n\\s*([a-z])/g, '$1$2');\n}\n\n/**\n * Split cleaned page text into paragraphs.\n *\n * Blank lines end a paragraph, but so does a heading - on either side. Some\n * extractors emit blank lines between paragraphs and some emit none at all\n * (the difference depends on how the PDF encodes vertical spacing), and a\n * document that arrives as one unbroken run of lines must still produce\n * sections. Treating headings as boundaries in their own right makes the split\n * work either way.\n */\nfunction toParagraphs(text, options = {}) {\n const { maxHeadingLength } = { ...CHUNK_DEFAULTS, ...options };\n const paragraphs = [];\n let current = [];\n\n const flush = () => {\n if (current.length) paragraphs.push(current.join('\\n'));\n current = [];\n };\n\n for (const rawLine of text.split('\\n')) {\n const line = rawLine.trim();\n if (!line) {\n flush();\n continue;\n }\n if (isHeading(line, maxHeadingLength)) {\n flush();\n paragraphs.push(line);\n continue;\n }\n current.push(line);\n }\n flush();\n\n return paragraphs;\n}\n\n/**\n * Turn pages into paragraph-sized blocks, each tagged with its page number and\n * the heading it sits under. This is the unit the packer works with.\n */\nfunction toBlocks(pages, boilerplate, options = {}) {\n const opts = { ...CHUNK_DEFAULTS, ...options };\n const blocks = [];\n const headingStack = [];\n\n pages.forEach((page, pageIndex) => {\n const cleaned = reflow(dehyphenate(cleanPage(page, boilerplate, opts)));\n if (!cleaned) return;\n for (const paragraph of toParagraphs(cleaned, opts)) {\n const text = paragraph.trim();\n if (!text) continue;\n const heading = !text.includes('\\n') && isHeading(text, opts.maxHeadingLength);\n if (heading) {\n const depth = headingDepth(text);\n headingStack.length = Math.min(headingStack.length, depth - 1);\n headingStack[depth - 1] = text;\n for (let i = 0; i < headingStack.length; i += 1) {\n if (headingStack[i] === undefined) headingStack[i] = '';\n }\n }\n blocks.push({\n text,\n page: pages.length > 1 ? pageIndex + 1 : null,\n isHeading: heading,\n section: headingStack.filter(Boolean).slice(-1)[0] ?? null,\n headingPath: headingStack.filter(Boolean).join(' › ') || null,\n tokens: estimateTokens(text, opts.charsPerToken),\n });\n }\n });\n\n return blocks;\n}\n\nfunction splitSentences(text) {\n return text.split(/(?<=[.!?])\\s+(?=[A-Z“\"(\\d])/).filter(Boolean);\n}\n\n/** Break a block bigger than one chunk, preferring sentence edges. */\nfunction splitOversizedBlock(block, maxTokens, charsPerToken) {\n const maxChars = maxTokens * charsPerToken;\n const pieces = [];\n let current = '';\n const flush = () => {\n if (current.trim()) {\n pieces.push({\n ...block,\n text: current.trim(),\n tokens: estimateTokens(current, charsPerToken),\n });\n }\n current = '';\n };\n\n for (const sentence of splitSentences(block.text)) {\n if (sentence.length > maxChars) {\n flush();\n // A single sentence longer than a whole chunk: hard-split on word edges.\n let rest = sentence;\n while (rest.length > maxChars) {\n let cut = rest.lastIndexOf(' ', maxChars);\n if (cut < maxChars * 0.5) cut = maxChars;\n pieces.push({ ...block, text: rest.slice(0, cut).trim(), tokens: maxTokens });\n rest = rest.slice(cut).trimStart();\n }\n current = rest;\n continue;\n }\n if (current && current.length + sentence.length + 1 > maxChars) flush();\n current = current ? `${current} ${sentence}` : sentence;\n }\n flush();\n return pieces.length ? pieces : [block];\n}\n\n/** Tail of a chunk to prepend to the next one, cut at a sentence edge. */\nfunction overlapTail(text, overlapTokens, charsPerToken) {\n const chars = overlapTokens * charsPerToken;\n if (chars <= 0 || !text) return '';\n if (text.length <= chars) return text.trim();\n let tail = text.slice(-chars);\n const boundary = tail.search(/[.!?]\\s+/);\n if (boundary >= 0 && boundary < tail.length - 20) {\n tail = tail.slice(boundary + 1).trimStart();\n } else {\n const space = tail.indexOf(' ');\n if (space >= 0) tail = tail.slice(space + 1);\n }\n return tail.trim();\n}\n\n/**\n * @param {string|string[]} raw Text from the Extract From File node.\n * @returns {{chunks: object[], normalisedText: string, stats: object}}\n */\nfunction chunkDocument(raw, options = {}) {\n const opts = { ...CHUNK_DEFAULTS, ...options };\n const pages = toPages(raw);\n const boilerplate = findBoilerplate(pages, opts);\n const rawBlocks = toBlocks(pages, boilerplate, opts);\n\n const blocks = [];\n for (const block of rawBlocks) {\n if (block.tokens > opts.chunkTokens) {\n blocks.push(...splitOversizedBlock(block, opts.chunkTokens, opts.charsPerToken));\n } else {\n blocks.push(block);\n }\n }\n\n const chunks = [];\n // The overlap prefix each chunk was given, parallel to `chunks`. Only needed\n // so a later merge can strip text the previous chunk already ends with.\n const overlaps = [];\n let parts = [];\n let tokens = 0;\n let carry = '';\n\n const flush = () => {\n // A heading stranded at the end of a chunk belongs to the next one - it\n // titles content this chunk does not contain.\n const stranded = [];\n while (parts.length > 1 && parts[parts.length - 1].isHeading) {\n stranded.unshift(parts.pop());\n }\n\n const body = parts.map((p) => p.text).join('\\n\\n').trim();\n if (body) {\n const text = carry ? `${carry}\\n\\n${body}` : body;\n const pageNumbers = parts.map((p) => p.page).filter((p) => p != null);\n // The section a chunk is \"in\" is the one its prose sits under, not a\n // heading that happens to be its last line.\n const anchor = parts.find((p) => !p.isHeading) ?? parts[0];\n chunks.push({\n chunk_index: chunks.length,\n text,\n tokens: estimateTokens(text, opts.charsPerToken),\n chars: text.length,\n page_start: pageNumbers.length ? Math.min(...pageNumbers) : null,\n page_end: pageNumbers.length ? Math.max(...pageNumbers) : null,\n section: anchor.section ?? null,\n heading_path: anchor.headingPath ?? null,\n has_overlap: Boolean(carry),\n });\n overlaps.push(carry);\n carry = overlapTail(text, opts.overlapTokens, opts.charsPerToken);\n }\n\n parts = stranded;\n tokens = stranded.reduce((sum, p) => sum + p.tokens, 0);\n };\n\n for (let i = 0; i < blocks.length; i += 1) {\n const block = blocks[i];\n // A heading landing mid-chunk starts a new one when the current chunk is\n // already substantial, or when the heading and the paragraph it introduces\n // would not fit together. This is what makes the split \"semantic\".\n if (block.isHeading && parts.length) {\n const next = blocks[i + 1];\n const projected = tokens + block.tokens + (next && !next.isHeading ? next.tokens : 0);\n if (tokens > opts.chunkTokens * 0.6 || projected > opts.chunkTokens) flush();\n }\n if (tokens + block.tokens > opts.chunkTokens && parts.length) flush();\n parts.push(block);\n tokens += block.tokens;\n }\n // flush() can hand headings back to `parts`, so drain rather than call once.\n while (parts.length) flush();\n\n // A stubby trailing chunk is worse than a slightly oversized previous one.\n if (chunks.length > 1) {\n const tail = chunks[chunks.length - 1];\n if (tail.tokens < opts.minChunkTokens) {\n const prev = chunks[chunks.length - 2];\n // Drop the tail chunk's overlap prefix - prev already ends with it.\n const tailOverlap = overlaps[chunks.length - 1] ?? '';\n const tailText =\n tailOverlap && tail.text.startsWith(tailOverlap)\n ? tail.text.slice(tailOverlap.length).trimStart()\n : tail.text;\n prev.text = `${prev.text}\\n\\n${tailText}`;\n prev.tokens = estimateTokens(prev.text, opts.charsPerToken);\n prev.chars = prev.text.length;\n prev.page_end = tail.page_end ?? prev.page_end;\n chunks.pop();\n overlaps.pop();\n }\n }\n\n const normalisedText = rawBlocks.map((b) => b.text).join('\\n\\n');\n\n return {\n chunks,\n normalisedText,\n stats: {\n pages: pages.length,\n paginated: pages.length > 1,\n blocks: rawBlocks.length,\n boilerplate_lines_removed: boilerplate.size,\n chars: normalisedText.length,\n chunks: chunks.length,\n avg_chunk_tokens: chunks.length\n ? Math.round(chunks.reduce((sum, c) => sum + c.tokens, 0) / chunks.length)\n : 0,\n },\n };\n}\n// --- end src/lib/chunker.js\n\n/**\n * Node: \"Chunk Document\"\n *\n * Cleans the extracted PDF text, splits it into ~500-token chunks with overlap,\n * and fingerprints the *normalised* text.\n *\n * Hashing after cleaning rather than hashing the raw bytes is deliberate: it\n * makes the fingerprint track meaning, not encoding. A PDF re-exported from the\n * same source has different bytes and an identical content hash, so it costs\n * nothing to re-embed - which is the point.\n */\n\nconst cfg = readConfig(requireEnv($env), INGESTION_CONFIG_SPEC);\nconst doc = $('Loop Over Documents').first().json;\nconst extracted = $input.first().json;\n\nconst result = chunkDocument(extracted.text ?? extracted.pages ?? '', {\n chunkTokens: cfg.chunkTokens,\n overlapTokens: cfg.overlapTokens,\n});\n\nif (!result.chunks.length) {\n throw new Error(\n `No extractable text in \"${doc.filename}\". The PDF is almost certainly a scan ` +\n `with no OCR layer, or it is empty. OCR it, or remove it from the watched folder.`,\n );\n}\n\nconst contentHash = sha256Hex(result.normalisedText);\nconst contentChanged = !doc.existing || doc.existing.content_hash !== contentHash;\n\nconsole.log(\n `${doc.filename}: ${result.stats.pages} page(s) -> ${result.chunks.length} chunk(s), ` +\n `avg ${result.stats.avg_chunk_tokens} tokens, ` +\n `${result.stats.boilerplate_lines_removed} boilerplate line(s) removed, ` +\n `content ${contentChanged ? 'CHANGED' : 'unchanged'}`,\n);\n\nreturn [\n {\n json: {\n ...doc,\n content_hash: contentHash,\n content_changed: contentChanged,\n chunk_count: result.chunks.length,\n page_count: result.stats.pages,\n char_count: result.stats.chars,\n paginated: result.stats.paginated,\n chunk_stats: result.stats,\n chunks: result.chunks,\n },\n },\n];\n" }, "id": "f66854d2-5487-4fe1-a94e-6b2c600113b0", "name": "Chunk Document", "type": "n8n-nodes-base.code", "typeVersion": 2, "position": [ 3600, 380 ], "onError": "continueErrorOutput", "notes": "Cleaning, chunking, and duplicate protection tier 2: hash the normalised text, so a re-export with identical content is not re-embedded." }, { "parameters": { "conditions": { "options": { "caseSensitive": true, "leftValue": "", "typeValidation": "loose", "version": 2 }, "combinator": "and", "conditions": [ { "id": "a7896882-6bb3-4fa4-a1dd-5146cccb57ac", "leftValue": "={{ $json.content_changed }}", "rightValue": "", "operator": { "type": "boolean", "operation": "true", "singleValue": true } } ] }, "looseTypeValidation": true, "options": {} }, "id": "82527a40-6cb9-4777-a928-65d609b65550", "name": "Content Changed?", "type": "n8n-nodes-base.if", "typeVersion": 2.2, "position": [ 3900, 380 ] }, { "parameters": { "options": { "timeout": 30000 }, "method": "PATCH", "url": "=https://api.airtable.com/v0/{{ $env.AIRTABLE_BASE_ID }}/{{ encodeURIComponent($env.AIRTABLE_DOCUMENTS_TABLE || \"Documents\") }}", "authentication": "predefinedCredentialType", "nodeCredentialType": "airtableTokenApi", "sendBody": true, "specifyBody": "json", "jsonBody": "={{ JSON.stringify({ performUpsert: { fieldsToMergeOn: [\"doc_id\"] }, typecast: true, records: [{ fields: { doc_id: $json.doc_id, filename: $json.filename, department: $json.department, status: \"Indexing\", last_checked: new Date().toISOString() } }] }) }}" }, "id": "8b871dfe-31eb-42af-afa9-1b4dd531f246", "name": "Airtable: Mark Indexing", "type": "n8n-nodes-base.httpRequest", "typeVersion": 4.2, "position": [ 4200, 285 ], "notes": "Marks the row in-flight, so a crashed run is visibly stuck rather than silently gone.", "retryOnFail": true, "maxTries": 3, "waitBetweenTries": 2000 }, { "parameters": { "mode": "runOnceForAllItems", "jsCode": "// --- inlined from src/lib/config.js ---------------------------\n/**\n * Environment reading + validation shared by every workflow.\n *\n * Workflows fail fast with one readable message naming the missing variable,\n * rather than throwing \"undefined is not a string\" six nodes downstream.\n */\n\n/**\n * @param {object} env Usually n8n's `$env`.\n * @param {object[]} spec [{ key, name, required, default, type }]\n */\nfunction readConfig(env, spec) {\n const source = env ?? {};\n const out = {};\n const missing = [];\n const invalid = [];\n\n for (const entry of spec) {\n const raw = source[entry.key];\n let value = raw === undefined || raw === null || raw === '' ? entry.default : raw;\n\n if (value === undefined || value === null || value === '') {\n if (entry.required) missing.push(entry.key);\n out[entry.name] = entry.default ?? null;\n continue;\n }\n\n switch (entry.type) {\n case 'number': {\n const n = Number(value);\n if (!Number.isFinite(n)) invalid.push(`${entry.key} must be a number (got \"${value}\")`);\n value = n;\n break;\n }\n case 'boolean':\n value = value === true || String(value).trim().toLowerCase() === 'true' || String(value) === '1';\n break;\n case 'json':\n if (typeof value === 'string') {\n try {\n value = JSON.parse(value);\n } catch {\n invalid.push(`${entry.key} must be valid JSON (got \"${String(value).slice(0, 40)}\")`);\n value = entry.default;\n }\n }\n break;\n case 'list':\n value = String(value)\n .split(',')\n .map((v) => v.trim())\n .filter(Boolean);\n break;\n case 'url':\n value = String(value).trim().replace(/\\/+$/, '');\n if (!/^https?:\\/\\//i.test(value)) {\n invalid.push(`${entry.key} must start with https:// (got \"${value}\")`);\n }\n break;\n default:\n value = String(value).trim();\n }\n\n out[entry.name] = value;\n }\n\n const problems = [];\n if (missing.length) problems.push(`missing: ${missing.join(', ')}`);\n if (invalid.length) problems.push(invalid.join('; '));\n if (problems.length) {\n throw new Error(\n `DocuQuery configuration problem - ${problems.join(' | ')}. ` +\n `Set these in your n8n environment (see .env.example) and restart n8n.`,\n );\n }\n\n return out;\n}\n\n/**\n * n8n hides `$env` when N8N_BLOCK_ENV_ACCESS_IN_NODE is on. Detect it here so\n * the failure names the setting instead of surfacing as \"$env is not defined\".\n */\nfunction requireEnv(env) {\n if (!env || typeof env !== 'object') {\n throw new Error(\n 'This node cannot read environment variables. Set ' +\n 'N8N_BLOCK_ENV_ACCESS_IN_NODE=false on the n8n container and restart it.',\n );\n }\n return env;\n}\n\nconst INGESTION_CONFIG_SPEC = [\n { key: 'DRIVE_FOLDER_ID', name: 'driveFolderId', required: true },\n { key: 'PINECONE_INDEX_HOST', name: 'pineconeHost', required: true, type: 'url' },\n { key: 'PINECONE_NAMESPACE', name: 'pineconeNamespace', default: 'docuquery' },\n { key: 'PINECONE_DIMENSION', name: 'pineconeDimension', type: 'number', default: 1536 },\n { key: 'EMBEDDING_MODEL', name: 'embeddingModel', default: 'text-embedding-3-small' },\n { key: 'EMBED_BATCH_SIZE', name: 'embedBatchSize', type: 'number', default: 96 },\n { key: 'CHUNK_TOKENS', name: 'chunkTokens', type: 'number', default: 500 },\n { key: 'CHUNK_OVERLAP_TOKENS', name: 'overlapTokens', type: 'number', default: 50 },\n { key: 'AIRTABLE_BASE_ID', name: 'airtableBaseId', required: true },\n { key: 'AIRTABLE_DOCUMENTS_TABLE', name: 'documentsTable', default: 'Documents' },\n { key: 'MAX_INGEST_RETRIES', name: 'maxRetries', type: 'number', default: 3 },\n { key: 'DEPARTMENTS', name: 'departments', type: 'list', default: 'General' },\n { key: 'DEFAULT_DEPARTMENT', name: 'defaultDepartment', default: 'General' },\n { key: 'DEPARTMENT_FOLDER_MAP', name: 'departmentFolderMap', type: 'json', default: {} },\n];\n\nconst QUERY_CONFIG_SPEC = [\n { key: 'PINECONE_INDEX_HOST', name: 'pineconeHost', required: true, type: 'url' },\n { key: 'PINECONE_NAMESPACE', name: 'pineconeNamespace', default: 'docuquery' },\n { key: 'EMBEDDING_MODEL', name: 'embeddingModel', default: 'text-embedding-3-small' },\n { key: 'LLM_PROVIDER', name: 'llmProvider', default: 'openai' },\n { key: 'LLM_MODEL', name: 'llmModel', default: 'gpt-4o' },\n { key: 'LLM_MAX_TOKENS', name: 'llmMaxTokens', type: 'number', default: 900 },\n { key: 'LLM_TEMPERATURE', name: 'llmTemperature', type: 'number', default: 0 },\n { key: 'LLM_BASE_URL', name: 'llmBaseUrl', default: '' },\n { key: 'RETRIEVAL_TOP_K', name: 'topK', type: 'number', default: 8 },\n { key: 'RETRIEVAL_MIN_SCORE', name: 'minScore', type: 'number', default: 0.32 },\n { key: 'RETRIEVAL_MAX_PER_DOC', name: 'maxPerDoc', type: 'number', default: 4 },\n { key: 'RETRIEVAL_MAX_CONTEXT_TOKENS', name: 'maxContextTokens', type: 'number', default: 6000 },\n { key: 'DEPARTMENTS', name: 'departments', type: 'list', default: 'General' },\n { key: 'SLACK_SIGNING_SECRET', name: 'slackSigningSecret', default: '' },\n { key: 'SLACK_TEAM_ID', name: 'slackTeamId', default: '' },\n { key: 'SLACK_ALLOW_UNVERIFIED', name: 'slackAllowUnverified', type: 'boolean', default: false },\n { key: 'QUERY_API_TOKEN', name: 'queryApiToken', default: '' },\n];\n// --- end src/lib/config.js\n\n// --- inlined from src/lib/vectors.js --------------------------\n/**\n * Pinecone vector construction, batching, and stale-vector cleanup.\n *\n * Vector ids are deterministic: `{doc_id}#{chunk_index}`. Two consequences the\n * pipeline depends on:\n * - Re-indexing a document overwrites its vectors rather than duplicating\n * them, because the ids collide by design.\n * - When a revised document produces *fewer* chunks than before, the leftover\n * tail ids can be computed and deleted without listing the index. (Delete\n * by metadata filter is not supported on Pinecone serverless indexes, so\n * computing ids is not just an optimisation - it is the supported route.)\n */\n\nconst UPSERT_BATCH_SIZE = 100;\nconst DELETE_BATCH_SIZE = 1000;\nconst MAX_METADATA_TEXT = 8000;\n\nfunction vectorId(docId, chunkIndex) {\n return `${docId}#${chunkIndex}`;\n}\n\n/** Pinecone rejects null metadata values, so drop rather than send them. */\nfunction compact(metadata) {\n const out = {};\n for (const [key, value] of Object.entries(metadata)) {\n if (value === null || value === undefined || value === '') continue;\n if (Array.isArray(value) && !value.length) continue;\n out[key] = value;\n }\n return out;\n}\n\n/**\n * `upload_ts` exists alongside `upload_date` because Pinecone can only do\n * range comparisons ($gte/$lt) on numbers - a date string supports equality\n * filtering and nothing else.\n */\nfunction buildVector(doc, chunk, embedding) {\n const uploadDate = doc.upload_date ? String(doc.upload_date).slice(0, 10) : null;\n const uploadTs = uploadDate ? Math.floor(new Date(`${uploadDate}T00:00:00Z`).getTime() / 1000) : null;\n\n return {\n id: vectorId(doc.doc_id, chunk.chunk_index),\n values: embedding,\n metadata: compact({\n doc_id: doc.doc_id,\n filename: doc.filename,\n title: doc.title ?? doc.filename,\n department: doc.department,\n drive_url: doc.web_view_link,\n upload_date: uploadDate,\n upload_ts: Number.isFinite(uploadTs) ? uploadTs : null,\n chunk_index: chunk.chunk_index,\n chunk_count: doc.chunk_count,\n page_start: chunk.page_start,\n page_end: chunk.page_end,\n section: chunk.section,\n heading_path: chunk.heading_path,\n content_hash: doc.content_hash,\n text: String(chunk.text ?? '').slice(0, MAX_METADATA_TEXT),\n }),\n };\n}\n\nfunction batch(items, size) {\n const out = [];\n for (let i = 0; i < items.length; i += size) out.push(items.slice(i, i + size));\n return out;\n}\n\n/**\n * Ids left behind when a document shrinks. The `+ margin` tail covers the case\n * where the previous chunk count was never recorded (a run that died mid-way),\n * at the cost of a few no-op deletes - Pinecone ignores ids that do not exist.\n */\nfunction staleVectorIds(docId, newChunkCount, previousChunkCount = 0, margin = 25) {\n const upper = Math.max(Number(previousChunkCount) || 0, newChunkCount + margin);\n const ids = [];\n for (let i = newChunkCount; i < upper; i += 1) ids.push(vectorId(docId, i));\n return ids;\n}\n\nfunction assertDimensions(vectors, expected) {\n if (!expected) return;\n const actual = vectors[0]?.values?.length;\n if (actual && actual !== Number(expected)) {\n throw new Error(\n `Embedding dimension mismatch: the model returned ${actual}-dimensional vectors but ` +\n `PINECONE_DIMENSION is ${expected}. Recreate the index with dimension ${actual}, ` +\n `or switch EMBEDDING_MODEL back to one that emits ${expected} dimensions.`,\n );\n }\n}\n// --- end src/lib/vectors.js\n\n/**\n * Node: \"Batch Chunks For Embedding\"\n *\n * The embeddings API takes an array of inputs. One request per chunk would turn\n * a 60-chunk document into 60 round trips and 60 chances to hit a rate limit;\n * batches of ~96 make it one or two.\n */\n\nconst cfg = readConfig(requireEnv($env), INGESTION_CONFIG_SPEC);\n// Read the document from its own node rather than $input: the node feeding this\n// one is an Airtable write, whose output is an Airtable response.\nconst doc = $('Chunk Document').first().json;\n\nconst chunks = doc.chunks ?? [];\nif (!chunks.length) throw new Error(`\"${doc.filename}\" produced no chunks to embed.`);\n\nreturn batch(chunks, cfg.embedBatchSize).map((group, index) => ({\n json: {\n doc_id: doc.doc_id,\n filename: doc.filename,\n model: cfg.embeddingModel,\n batch_index: index,\n // Offset of this batch's first chunk, so embeddings can be paired back up.\n from: index * cfg.embedBatchSize,\n input: group.map((chunk) => chunk.text),\n },\n}));\n" }, "id": "dd19a0a0-fd0b-4534-a87f-2c757fd312ab", "name": "Batch Chunks For Embedding", "type": "n8n-nodes-base.code", "typeVersion": 2, "position": [ 4500, 285 ] }, { "parameters": { "options": { "timeout": 120000, "batching": { "batch": { "batchSize": 1, "batchInterval": 500 } } }, "method": "POST", "url": "https://api.openai.com/v1/embeddings", "authentication": "predefinedCredentialType", "nodeCredentialType": "openAiApi", "sendBody": true, "specifyBody": "json", "jsonBody": "={{ JSON.stringify({ model: $json.model, input: $json.input }) }}" }, "id": "50636e5a-0922-474d-a30a-d2a8e70ddf17", "name": "OpenAI: Create Embeddings", "type": "n8n-nodes-base.httpRequest", "typeVersion": 4.2, "position": [ 4800, 285 ], "onError": "continueErrorOutput", "retryOnFail": true, "maxTries": 4, "waitBetweenTries": 5000, "notes": "Credential: OpenAI. Retries cover 429s; batchInterval spaces the calls out." }, { "parameters": { "mode": "runOnceForAllItems", "jsCode": "// --- inlined from src/lib/config.js ---------------------------\n/**\n * Environment reading + validation shared by every workflow.\n *\n * Workflows fail fast with one readable message naming the missing variable,\n * rather than throwing \"undefined is not a string\" six nodes downstream.\n */\n\n/**\n * @param {object} env Usually n8n's `$env`.\n * @param {object[]} spec [{ key, name, required, default, type }]\n */\nfunction readConfig(env, spec) {\n const source = env ?? {};\n const out = {};\n const missing = [];\n const invalid = [];\n\n for (const entry of spec) {\n const raw = source[entry.key];\n let value = raw === undefined || raw === null || raw === '' ? entry.default : raw;\n\n if (value === undefined || value === null || value === '') {\n if (entry.required) missing.push(entry.key);\n out[entry.name] = entry.default ?? null;\n continue;\n }\n\n switch (entry.type) {\n case 'number': {\n const n = Number(value);\n if (!Number.isFinite(n)) invalid.push(`${entry.key} must be a number (got \"${value}\")`);\n value = n;\n break;\n }\n case 'boolean':\n value = value === true || String(value).trim().toLowerCase() === 'true' || String(value) === '1';\n break;\n case 'json':\n if (typeof value === 'string') {\n try {\n value = JSON.parse(value);\n } catch {\n invalid.push(`${entry.key} must be valid JSON (got \"${String(value).slice(0, 40)}\")`);\n value = entry.default;\n }\n }\n break;\n case 'list':\n value = String(value)\n .split(',')\n .map((v) => v.trim())\n .filter(Boolean);\n break;\n case 'url':\n value = String(value).trim().replace(/\\/+$/, '');\n if (!/^https?:\\/\\//i.test(value)) {\n invalid.push(`${entry.key} must start with https:// (got \"${value}\")`);\n }\n break;\n default:\n value = String(value).trim();\n }\n\n out[entry.name] = value;\n }\n\n const problems = [];\n if (missing.length) problems.push(`missing: ${missing.join(', ')}`);\n if (invalid.length) problems.push(invalid.join('; '));\n if (problems.length) {\n throw new Error(\n `DocuQuery configuration problem - ${problems.join(' | ')}. ` +\n `Set these in your n8n environment (see .env.example) and restart n8n.`,\n );\n }\n\n return out;\n}\n\n/**\n * n8n hides `$env` when N8N_BLOCK_ENV_ACCESS_IN_NODE is on. Detect it here so\n * the failure names the setting instead of surfacing as \"$env is not defined\".\n */\nfunction requireEnv(env) {\n if (!env || typeof env !== 'object') {\n throw new Error(\n 'This node cannot read environment variables. Set ' +\n 'N8N_BLOCK_ENV_ACCESS_IN_NODE=false on the n8n container and restart it.',\n );\n }\n return env;\n}\n\nconst INGESTION_CONFIG_SPEC = [\n { key: 'DRIVE_FOLDER_ID', name: 'driveFolderId', required: true },\n { key: 'PINECONE_INDEX_HOST', name: 'pineconeHost', required: true, type: 'url' },\n { key: 'PINECONE_NAMESPACE', name: 'pineconeNamespace', default: 'docuquery' },\n { key: 'PINECONE_DIMENSION', name: 'pineconeDimension', type: 'number', default: 1536 },\n { key: 'EMBEDDING_MODEL', name: 'embeddingModel', default: 'text-embedding-3-small' },\n { key: 'EMBED_BATCH_SIZE', name: 'embedBatchSize', type: 'number', default: 96 },\n { key: 'CHUNK_TOKENS', name: 'chunkTokens', type: 'number', default: 500 },\n { key: 'CHUNK_OVERLAP_TOKENS', name: 'overlapTokens', type: 'number', default: 50 },\n { key: 'AIRTABLE_BASE_ID', name: 'airtableBaseId', required: true },\n { key: 'AIRTABLE_DOCUMENTS_TABLE', name: 'documentsTable', default: 'Documents' },\n { key: 'MAX_INGEST_RETRIES', name: 'maxRetries', type: 'number', default: 3 },\n { key: 'DEPARTMENTS', name: 'departments', type: 'list', default: 'General' },\n { key: 'DEFAULT_DEPARTMENT', name: 'defaultDepartment', default: 'General' },\n { key: 'DEPARTMENT_FOLDER_MAP', name: 'departmentFolderMap', type: 'json', default: {} },\n];\n\nconst QUERY_CONFIG_SPEC = [\n { key: 'PINECONE_INDEX_HOST', name: 'pineconeHost', required: true, type: 'url' },\n { key: 'PINECONE_NAMESPACE', name: 'pineconeNamespace', default: 'docuquery' },\n { key: 'EMBEDDING_MODEL', name: 'embeddingModel', default: 'text-embedding-3-small' },\n { key: 'LLM_PROVIDER', name: 'llmProvider', default: 'openai' },\n { key: 'LLM_MODEL', name: 'llmModel', default: 'gpt-4o' },\n { key: 'LLM_MAX_TOKENS', name: 'llmMaxTokens', type: 'number', default: 900 },\n { key: 'LLM_TEMPERATURE', name: 'llmTemperature', type: 'number', default: 0 },\n { key: 'LLM_BASE_URL', name: 'llmBaseUrl', default: '' },\n { key: 'RETRIEVAL_TOP_K', name: 'topK', type: 'number', default: 8 },\n { key: 'RETRIEVAL_MIN_SCORE', name: 'minScore', type: 'number', default: 0.32 },\n { key: 'RETRIEVAL_MAX_PER_DOC', name: 'maxPerDoc', type: 'number', default: 4 },\n { key: 'RETRIEVAL_MAX_CONTEXT_TOKENS', name: 'maxContextTokens', type: 'number', default: 6000 },\n { key: 'DEPARTMENTS', name: 'departments', type: 'list', default: 'General' },\n { key: 'SLACK_SIGNING_SECRET', name: 'slackSigningSecret', default: '' },\n { key: 'SLACK_TEAM_ID', name: 'slackTeamId', default: '' },\n { key: 'SLACK_ALLOW_UNVERIFIED', name: 'slackAllowUnverified', type: 'boolean', default: false },\n { key: 'QUERY_API_TOKEN', name: 'queryApiToken', default: '' },\n];\n// --- end src/lib/config.js\n\n// --- inlined from src/lib/vectors.js --------------------------\n/**\n * Pinecone vector construction, batching, and stale-vector cleanup.\n *\n * Vector ids are deterministic: `{doc_id}#{chunk_index}`. Two consequences the\n * pipeline depends on:\n * - Re-indexing a document overwrites its vectors rather than duplicating\n * them, because the ids collide by design.\n * - When a revised document produces *fewer* chunks than before, the leftover\n * tail ids can be computed and deleted without listing the index. (Delete\n * by metadata filter is not supported on Pinecone serverless indexes, so\n * computing ids is not just an optimisation - it is the supported route.)\n */\n\nconst UPSERT_BATCH_SIZE = 100;\nconst DELETE_BATCH_SIZE = 1000;\nconst MAX_METADATA_TEXT = 8000;\n\nfunction vectorId(docId, chunkIndex) {\n return `${docId}#${chunkIndex}`;\n}\n\n/** Pinecone rejects null metadata values, so drop rather than send them. */\nfunction compact(metadata) {\n const out = {};\n for (const [key, value] of Object.entries(metadata)) {\n if (value === null || value === undefined || value === '') continue;\n if (Array.isArray(value) && !value.length) continue;\n out[key] = value;\n }\n return out;\n}\n\n/**\n * `upload_ts` exists alongside `upload_date` because Pinecone can only do\n * range comparisons ($gte/$lt) on numbers - a date string supports equality\n * filtering and nothing else.\n */\nfunction buildVector(doc, chunk, embedding) {\n const uploadDate = doc.upload_date ? String(doc.upload_date).slice(0, 10) : null;\n const uploadTs = uploadDate ? Math.floor(new Date(`${uploadDate}T00:00:00Z`).getTime() / 1000) : null;\n\n return {\n id: vectorId(doc.doc_id, chunk.chunk_index),\n values: embedding,\n metadata: compact({\n doc_id: doc.doc_id,\n filename: doc.filename,\n title: doc.title ?? doc.filename,\n department: doc.department,\n drive_url: doc.web_view_link,\n upload_date: uploadDate,\n upload_ts: Number.isFinite(uploadTs) ? uploadTs : null,\n chunk_index: chunk.chunk_index,\n chunk_count: doc.chunk_count,\n page_start: chunk.page_start,\n page_end: chunk.page_end,\n section: chunk.section,\n heading_path: chunk.heading_path,\n content_hash: doc.content_hash,\n text: String(chunk.text ?? '').slice(0, MAX_METADATA_TEXT),\n }),\n };\n}\n\nfunction batch(items, size) {\n const out = [];\n for (let i = 0; i < items.length; i += size) out.push(items.slice(i, i + size));\n return out;\n}\n\n/**\n * Ids left behind when a document shrinks. The `+ margin` tail covers the case\n * where the previous chunk count was never recorded (a run that died mid-way),\n * at the cost of a few no-op deletes - Pinecone ignores ids that do not exist.\n */\nfunction staleVectorIds(docId, newChunkCount, previousChunkCount = 0, margin = 25) {\n const upper = Math.max(Number(previousChunkCount) || 0, newChunkCount + margin);\n const ids = [];\n for (let i = newChunkCount; i < upper; i += 1) ids.push(vectorId(docId, i));\n return ids;\n}\n\nfunction assertDimensions(vectors, expected) {\n if (!expected) return;\n const actual = vectors[0]?.values?.length;\n if (actual && actual !== Number(expected)) {\n throw new Error(\n `Embedding dimension mismatch: the model returned ${actual}-dimensional vectors but ` +\n `PINECONE_DIMENSION is ${expected}. Recreate the index with dimension ${actual}, ` +\n `or switch EMBEDDING_MODEL back to one that emits ${expected} dimensions.`,\n );\n }\n}\n// --- end src/lib/vectors.js\n\n/**\n * Node: \"Build Pinecone Vectors\"\n *\n * Pairs each embedding back to its chunk, attaches the retrieval metadata, and\n * batches for upsert.\n *\n * Pairing is by position: the HTTP Request node emits exactly one output item\n * per input item, in order, so response[i] belongs to request[i]. Within a\n * response, OpenAI returns each embedding tagged with its own `index`, which is\n * relative to that request - hence `from + index`.\n */\n\nconst cfg = readConfig(requireEnv($env), INGESTION_CONFIG_SPEC);\nconst doc = $('Chunk Document').first().json;\nconst requests = $('Batch Chunks For Embedding').all().map((item) => item.json);\nconst responses = $input.all();\n\nif (responses.length !== requests.length) {\n throw new Error(\n `Embedding response count (${responses.length}) does not match request count ` +\n `(${requests.length}) for \"${doc.filename}\".`,\n );\n}\n\nconst vectors = [];\nlet promptTokens = 0;\n\nresponses.forEach((item, requestIndex) => {\n const request = requests[requestIndex];\n const body = item.json ?? {};\n\n if (body.error) {\n const message = body.error.message ?? JSON.stringify(body.error);\n throw new Error(`Embedding batch ${requestIndex} failed for \"${doc.filename}\": ${message}`);\n }\n if (!Array.isArray(body.data) || body.data.length !== request.input.length) {\n throw new Error(\n `Embedding batch ${requestIndex} returned ${body.data?.length ?? 0} vectors for ` +\n `${request.input.length} chunk(s) of \"${doc.filename}\".`,\n );\n }\n\n promptTokens += Number(body.usage?.total_tokens ?? 0);\n\n for (const entry of body.data) {\n const chunk = doc.chunks[request.from + entry.index];\n if (!chunk) {\n throw new Error(`No chunk at index ${request.from + entry.index} for \"${doc.filename}\".`);\n }\n vectors.push(buildVector(doc, chunk, entry.embedding));\n }\n});\n\nassertDimensions(vectors, cfg.pineconeDimension);\n\nconsole.log(\n `${doc.filename}: embedded ${vectors.length} chunk(s) ` +\n `(${promptTokens} tokens, ${vectors[0].values.length} dimensions)`,\n);\n\nreturn batch(vectors, UPSERT_BATCH_SIZE).map((group, index) => ({\n json: {\n doc_id: doc.doc_id,\n filename: doc.filename,\n batch_index: index,\n namespace: cfg.pineconeNamespace,\n embedding_tokens: index === 0 ? promptTokens : 0,\n vectors: group,\n },\n}));\n" }, "id": "65c257b1-9de8-4eab-a6ec-e73236ff6c97", "name": "Build Pinecone Vectors", "type": "n8n-nodes-base.code", "typeVersion": 2, "position": [ 5100, 285 ], "notes": "Pairs embeddings to chunks and attaches the metadata retrieval filters on." }, { "parameters": { "options": { "timeout": 120000 }, "method": "POST", "url": "={{ $env.PINECONE_INDEX_HOST }}/vectors/upsert", "authentication": "genericCredentialType", "genericAuthType": "httpHeaderAuth", "sendHeaders": true, "specifyHeaders": "keypair", "headerParameters": { "parameters": [ { "name": "X-Pinecone-API-Version", "value": "={{ $env.PINECONE_API_VERSION || \"2025-01\" }}" } ] }, "sendBody": true, "specifyBody": "json", "jsonBody": "={{ JSON.stringify({ namespace: $json.namespace, vectors: $json.vectors }) }}" }, "id": "b989d8e0-eba6-40c7-a283-71e2cf19f5f9", "name": "Pinecone: Upsert Vectors", "type": "n8n-nodes-base.httpRequest", "typeVersion": 4.2, "position": [ 5400, 285 ], "onError": "continueErrorOutput", "retryOnFail": true, "maxTries": 4, "waitBetweenTries": 5000, "notes": "Credential: Header Auth with header name \"Api-Key\" and your Pinecone key." }, { "parameters": { "mode": "runOnceForAllItems", "jsCode": "// --- inlined from src/lib/config.js ---------------------------\n/**\n * Environment reading + validation shared by every workflow.\n *\n * Workflows fail fast with one readable message naming the missing variable,\n * rather than throwing \"undefined is not a string\" six nodes downstream.\n */\n\n/**\n * @param {object} env Usually n8n's `$env`.\n * @param {object[]} spec [{ key, name, required, default, type }]\n */\nfunction readConfig(env, spec) {\n const source = env ?? {};\n const out = {};\n const missing = [];\n const invalid = [];\n\n for (const entry of spec) {\n const raw = source[entry.key];\n let value = raw === undefined || raw === null || raw === '' ? entry.default : raw;\n\n if (value === undefined || value === null || value === '') {\n if (entry.required) missing.push(entry.key);\n out[entry.name] = entry.default ?? null;\n continue;\n }\n\n switch (entry.type) {\n case 'number': {\n const n = Number(value);\n if (!Number.isFinite(n)) invalid.push(`${entry.key} must be a number (got \"${value}\")`);\n value = n;\n break;\n }\n case 'boolean':\n value = value === true || String(value).trim().toLowerCase() === 'true' || String(value) === '1';\n break;\n case 'json':\n if (typeof value === 'string') {\n try {\n value = JSON.parse(value);\n } catch {\n invalid.push(`${entry.key} must be valid JSON (got \"${String(value).slice(0, 40)}\")`);\n value = entry.default;\n }\n }\n break;\n case 'list':\n value = String(value)\n .split(',')\n .map((v) => v.trim())\n .filter(Boolean);\n break;\n case 'url':\n value = String(value).trim().replace(/\\/+$/, '');\n if (!/^https?:\\/\\//i.test(value)) {\n invalid.push(`${entry.key} must start with https:// (got \"${value}\")`);\n }\n break;\n default:\n value = String(value).trim();\n }\n\n out[entry.name] = value;\n }\n\n const problems = [];\n if (missing.length) problems.push(`missing: ${missing.join(', ')}`);\n if (invalid.length) problems.push(invalid.join('; '));\n if (problems.length) {\n throw new Error(\n `DocuQuery configuration problem - ${problems.join(' | ')}. ` +\n `Set these in your n8n environment (see .env.example) and restart n8n.`,\n );\n }\n\n return out;\n}\n\n/**\n * n8n hides `$env` when N8N_BLOCK_ENV_ACCESS_IN_NODE is on. Detect it here so\n * the failure names the setting instead of surfacing as \"$env is not defined\".\n */\nfunction requireEnv(env) {\n if (!env || typeof env !== 'object') {\n throw new Error(\n 'This node cannot read environment variables. Set ' +\n 'N8N_BLOCK_ENV_ACCESS_IN_NODE=false on the n8n container and restart it.',\n );\n }\n return env;\n}\n\nconst INGESTION_CONFIG_SPEC = [\n { key: 'DRIVE_FOLDER_ID', name: 'driveFolderId', required: true },\n { key: 'PINECONE_INDEX_HOST', name: 'pineconeHost', required: true, type: 'url' },\n { key: 'PINECONE_NAMESPACE', name: 'pineconeNamespace', default: 'docuquery' },\n { key: 'PINECONE_DIMENSION', name: 'pineconeDimension', type: 'number', default: 1536 },\n { key: 'EMBEDDING_MODEL', name: 'embeddingModel', default: 'text-embedding-3-small' },\n { key: 'EMBED_BATCH_SIZE', name: 'embedBatchSize', type: 'number', default: 96 },\n { key: 'CHUNK_TOKENS', name: 'chunkTokens', type: 'number', default: 500 },\n { key: 'CHUNK_OVERLAP_TOKENS', name: 'overlapTokens', type: 'number', default: 50 },\n { key: 'AIRTABLE_BASE_ID', name: 'airtableBaseId', required: true },\n { key: 'AIRTABLE_DOCUMENTS_TABLE', name: 'documentsTable', default: 'Documents' },\n { key: 'MAX_INGEST_RETRIES', name: 'maxRetries', type: 'number', default: 3 },\n { key: 'DEPARTMENTS', name: 'departments', type: 'list', default: 'General' },\n { key: 'DEFAULT_DEPARTMENT', name: 'defaultDepartment', default: 'General' },\n { key: 'DEPARTMENT_FOLDER_MAP', name: 'departmentFolderMap', type: 'json', default: {} },\n];\n\nconst QUERY_CONFIG_SPEC = [\n { key: 'PINECONE_INDEX_HOST', name: 'pineconeHost', required: true, type: 'url' },\n { key: 'PINECONE_NAMESPACE', name: 'pineconeNamespace', default: 'docuquery' },\n { key: 'EMBEDDING_MODEL', name: 'embeddingModel', default: 'text-embedding-3-small' },\n { key: 'LLM_PROVIDER', name: 'llmProvider', default: 'openai' },\n { key: 'LLM_MODEL', name: 'llmModel', default: 'gpt-4o' },\n { key: 'LLM_MAX_TOKENS', name: 'llmMaxTokens', type: 'number', default: 900 },\n { key: 'LLM_TEMPERATURE', name: 'llmTemperature', type: 'number', default: 0 },\n { key: 'LLM_BASE_URL', name: 'llmBaseUrl', default: '' },\n { key: 'RETRIEVAL_TOP_K', name: 'topK', type: 'number', default: 8 },\n { key: 'RETRIEVAL_MIN_SCORE', name: 'minScore', type: 'number', default: 0.32 },\n { key: 'RETRIEVAL_MAX_PER_DOC', name: 'maxPerDoc', type: 'number', default: 4 },\n { key: 'RETRIEVAL_MAX_CONTEXT_TOKENS', name: 'maxContextTokens', type: 'number', default: 6000 },\n { key: 'DEPARTMENTS', name: 'departments', type: 'list', default: 'General' },\n { key: 'SLACK_SIGNING_SECRET', name: 'slackSigningSecret', default: '' },\n { key: 'SLACK_TEAM_ID', name: 'slackTeamId', default: '' },\n { key: 'SLACK_ALLOW_UNVERIFIED', name: 'slackAllowUnverified', type: 'boolean', default: false },\n { key: 'QUERY_API_TOKEN', name: 'queryApiToken', default: '' },\n];\n// --- end src/lib/config.js\n\n// --- inlined from src/lib/vectors.js --------------------------\n/**\n * Pinecone vector construction, batching, and stale-vector cleanup.\n *\n * Vector ids are deterministic: `{doc_id}#{chunk_index}`. Two consequences the\n * pipeline depends on:\n * - Re-indexing a document overwrites its vectors rather than duplicating\n * them, because the ids collide by design.\n * - When a revised document produces *fewer* chunks than before, the leftover\n * tail ids can be computed and deleted without listing the index. (Delete\n * by metadata filter is not supported on Pinecone serverless indexes, so\n * computing ids is not just an optimisation - it is the supported route.)\n */\n\nconst UPSERT_BATCH_SIZE = 100;\nconst DELETE_BATCH_SIZE = 1000;\nconst MAX_METADATA_TEXT = 8000;\n\nfunction vectorId(docId, chunkIndex) {\n return `${docId}#${chunkIndex}`;\n}\n\n/** Pinecone rejects null metadata values, so drop rather than send them. */\nfunction compact(metadata) {\n const out = {};\n for (const [key, value] of Object.entries(metadata)) {\n if (value === null || value === undefined || value === '') continue;\n if (Array.isArray(value) && !value.length) continue;\n out[key] = value;\n }\n return out;\n}\n\n/**\n * `upload_ts` exists alongside `upload_date` because Pinecone can only do\n * range comparisons ($gte/$lt) on numbers - a date string supports equality\n * filtering and nothing else.\n */\nfunction buildVector(doc, chunk, embedding) {\n const uploadDate = doc.upload_date ? String(doc.upload_date).slice(0, 10) : null;\n const uploadTs = uploadDate ? Math.floor(new Date(`${uploadDate}T00:00:00Z`).getTime() / 1000) : null;\n\n return {\n id: vectorId(doc.doc_id, chunk.chunk_index),\n values: embedding,\n metadata: compact({\n doc_id: doc.doc_id,\n filename: doc.filename,\n title: doc.title ?? doc.filename,\n department: doc.department,\n drive_url: doc.web_view_link,\n upload_date: uploadDate,\n upload_ts: Number.isFinite(uploadTs) ? uploadTs : null,\n chunk_index: chunk.chunk_index,\n chunk_count: doc.chunk_count,\n page_start: chunk.page_start,\n page_end: chunk.page_end,\n section: chunk.section,\n heading_path: chunk.heading_path,\n content_hash: doc.content_hash,\n text: String(chunk.text ?? '').slice(0, MAX_METADATA_TEXT),\n }),\n };\n}\n\nfunction batch(items, size) {\n const out = [];\n for (let i = 0; i < items.length; i += size) out.push(items.slice(i, i + size));\n return out;\n}\n\n/**\n * Ids left behind when a document shrinks. The `+ margin` tail covers the case\n * where the previous chunk count was never recorded (a run that died mid-way),\n * at the cost of a few no-op deletes - Pinecone ignores ids that do not exist.\n */\nfunction staleVectorIds(docId, newChunkCount, previousChunkCount = 0, margin = 25) {\n const upper = Math.max(Number(previousChunkCount) || 0, newChunkCount + margin);\n const ids = [];\n for (let i = newChunkCount; i < upper; i += 1) ids.push(vectorId(docId, i));\n return ids;\n}\n\nfunction assertDimensions(vectors, expected) {\n if (!expected) return;\n const actual = vectors[0]?.values?.length;\n if (actual && actual !== Number(expected)) {\n throw new Error(\n `Embedding dimension mismatch: the model returned ${actual}-dimensional vectors but ` +\n `PINECONE_DIMENSION is ${expected}. Recreate the index with dimension ${actual}, ` +\n `or switch EMBEDDING_MODEL back to one that emits ${expected} dimensions.`,\n );\n }\n}\n// --- end src/lib/vectors.js\n\n/**\n * Node: \"Build Stale Vector Purge\"\n *\n * A revised document that shrinks - a policy losing two sections - would\n * otherwise leave orphaned chunks in the index, and those orphans stay\n * retrievable and citable. They quote text the current document no longer\n * contains, which is worse than not answering.\n *\n * Ids are computed rather than listed because Pinecone serverless indexes do\n * not support delete-by-metadata-filter. Deleting an id that does not exist is\n * a no-op, so overshooting slightly is safe.\n *\n * This runs after the upsert, so the document is never briefly absent from the\n * index: ids 0..n-1 are overwritten first, then the leftover tail is removed.\n */\n\nconst cfg = readConfig(requireEnv($env), INGESTION_CONFIG_SPEC);\nconst doc = $('Chunk Document').first().json;\n\nconst ids = staleVectorIds(doc.doc_id, doc.chunk_count, doc.previous_chunk_count);\n\nif (!ids.length) {\n return [{ json: { ...doc, purge_batches: 0, stale_ids: [], has_stale: false } }];\n}\n\nconsole.log(\n `${doc.filename}: purging up to ${ids.length} stale vector id(s) ` +\n `(was ${doc.previous_chunk_count} chunks, now ${doc.chunk_count})`,\n);\n\nreturn batch(ids, DELETE_BATCH_SIZE).map((group, index) => ({\n json: {\n doc_id: doc.doc_id,\n filename: doc.filename,\n namespace: cfg.pineconeNamespace,\n batch_index: index,\n has_stale: true,\n stale_ids: group,\n },\n}));\n" }, "id": "0466168d-5aed-433d-a3d4-2cd643c936f1", "name": "Build Stale Vector Purge", "type": "n8n-nodes-base.code", "typeVersion": 2, "position": [ 5700, 285 ] }, { "parameters": { "conditions": { "options": { "caseSensitive": true, "leftValue": "", "typeValidation": "loose", "version": 2 }, "combinator": "and", "conditions": [ { "id": "a2d9368c-15f9-4774-a213-a1fd1faa083f", "leftValue": "={{ $json.has_stale }}", "rightValue": "", "operator": { "type": "boolean", "operation": "true", "singleValue": true } } ] }, "looseTypeValidation": true, "options": {} }, "id": "6459db9d-6e13-470a-a747-cc66ab677787", "name": "Has Stale Vectors?", "type": "n8n-nodes-base.if", "typeVersion": 2.2, "position": [ 6000, 285 ] }, { "parameters": { "options": { "timeout": 60000 }, "method": "POST", "url": "={{ $env.PINECONE_INDEX_HOST }}/vectors/delete", "authentication": "genericCredentialType", "genericAuthType": "httpHeaderAuth", "sendHeaders": true, "specifyHeaders": "keypair", "headerParameters": { "parameters": [ { "name": "X-Pinecone-API-Version", "value": "={{ $env.PINECONE_API_VERSION || \"2025-01\" }}" } ] }, "sendBody": true, "specifyBody": "json", "jsonBody": "={{ JSON.stringify({ namespace: $json.namespace, ids: $json.stale_ids }) }}" }, "id": "0513646a-4477-4c92-a1ba-befbba605a20", "name": "Pinecone: Delete Stale Vectors", "type": "n8n-nodes-base.httpRequest", "typeVersion": 4.2, "position": [ 6300, 190 ], "retryOnFail": true, "maxTries": 3, "waitBetweenTries": 3000 }, { "parameters": { "mode": "runOnceForAllItems", "jsCode": "// --- inlined from src/lib/config.js ---------------------------\n/**\n * Environment reading + validation shared by every workflow.\n *\n * Workflows fail fast with one readable message naming the missing variable,\n * rather than throwing \"undefined is not a string\" six nodes downstream.\n */\n\n/**\n * @param {object} env Usually n8n's `$env`.\n * @param {object[]} spec [{ key, name, required, default, type }]\n */\nfunction readConfig(env, spec) {\n const source = env ?? {};\n const out = {};\n const missing = [];\n const invalid = [];\n\n for (const entry of spec) {\n const raw = source[entry.key];\n let value = raw === undefined || raw === null || raw === '' ? entry.default : raw;\n\n if (value === undefined || value === null || value === '') {\n if (entry.required) missing.push(entry.key);\n out[entry.name] = entry.default ?? null;\n continue;\n }\n\n switch (entry.type) {\n case 'number': {\n const n = Number(value);\n if (!Number.isFinite(n)) invalid.push(`${entry.key} must be a number (got \"${value}\")`);\n value = n;\n break;\n }\n case 'boolean':\n value = value === true || String(value).trim().toLowerCase() === 'true' || String(value) === '1';\n break;\n case 'json':\n if (typeof value === 'string') {\n try {\n value = JSON.parse(value);\n } catch {\n invalid.push(`${entry.key} must be valid JSON (got \"${String(value).slice(0, 40)}\")`);\n value = entry.default;\n }\n }\n break;\n case 'list':\n value = String(value)\n .split(',')\n .map((v) => v.trim())\n .filter(Boolean);\n break;\n case 'url':\n value = String(value).trim().replace(/\\/+$/, '');\n if (!/^https?:\\/\\//i.test(value)) {\n invalid.push(`${entry.key} must start with https:// (got \"${value}\")`);\n }\n break;\n default:\n value = String(value).trim();\n }\n\n out[entry.name] = value;\n }\n\n const problems = [];\n if (missing.length) problems.push(`missing: ${missing.join(', ')}`);\n if (invalid.length) problems.push(invalid.join('; '));\n if (problems.length) {\n throw new Error(\n `DocuQuery configuration problem - ${problems.join(' | ')}. ` +\n `Set these in your n8n environment (see .env.example) and restart n8n.`,\n );\n }\n\n return out;\n}\n\n/**\n * n8n hides `$env` when N8N_BLOCK_ENV_ACCESS_IN_NODE is on. Detect it here so\n * the failure names the setting instead of surfacing as \"$env is not defined\".\n */\nfunction requireEnv(env) {\n if (!env || typeof env !== 'object') {\n throw new Error(\n 'This node cannot read environment variables. Set ' +\n 'N8N_BLOCK_ENV_ACCESS_IN_NODE=false on the n8n container and restart it.',\n );\n }\n return env;\n}\n\nconst INGESTION_CONFIG_SPEC = [\n { key: 'DRIVE_FOLDER_ID', name: 'driveFolderId', required: true },\n { key: 'PINECONE_INDEX_HOST', name: 'pineconeHost', required: true, type: 'url' },\n { key: 'PINECONE_NAMESPACE', name: 'pineconeNamespace', default: 'docuquery' },\n { key: 'PINECONE_DIMENSION', name: 'pineconeDimension', type: 'number', default: 1536 },\n { key: 'EMBEDDING_MODEL', name: 'embeddingModel', default: 'text-embedding-3-small' },\n { key: 'EMBED_BATCH_SIZE', name: 'embedBatchSize', type: 'number', default: 96 },\n { key: 'CHUNK_TOKENS', name: 'chunkTokens', type: 'number', default: 500 },\n { key: 'CHUNK_OVERLAP_TOKENS', name: 'overlapTokens', type: 'number', default: 50 },\n { key: 'AIRTABLE_BASE_ID', name: 'airtableBaseId', required: true },\n { key: 'AIRTABLE_DOCUMENTS_TABLE', name: 'documentsTable', default: 'Documents' },\n { key: 'MAX_INGEST_RETRIES', name: 'maxRetries', type: 'number', default: 3 },\n { key: 'DEPARTMENTS', name: 'departments', type: 'list', default: 'General' },\n { key: 'DEFAULT_DEPARTMENT', name: 'defaultDepartment', default: 'General' },\n { key: 'DEPARTMENT_FOLDER_MAP', name: 'departmentFolderMap', type: 'json', default: {} },\n];\n\nconst QUERY_CONFIG_SPEC = [\n { key: 'PINECONE_INDEX_HOST', name: 'pineconeHost', required: true, type: 'url' },\n { key: 'PINECONE_NAMESPACE', name: 'pineconeNamespace', default: 'docuquery' },\n { key: 'EMBEDDING_MODEL', name: 'embeddingModel', default: 'text-embedding-3-small' },\n { key: 'LLM_PROVIDER', name: 'llmProvider', default: 'openai' },\n { key: 'LLM_MODEL', name: 'llmModel', default: 'gpt-4o' },\n { key: 'LLM_MAX_TOKENS', name: 'llmMaxTokens', type: 'number', default: 900 },\n { key: 'LLM_TEMPERATURE', name: 'llmTemperature', type: 'number', default: 0 },\n { key: 'LLM_BASE_URL', name: 'llmBaseUrl', default: '' },\n { key: 'RETRIEVAL_TOP_K', name: 'topK', type: 'number', default: 8 },\n { key: 'RETRIEVAL_MIN_SCORE', name: 'minScore', type: 'number', default: 0.32 },\n { key: 'RETRIEVAL_MAX_PER_DOC', name: 'maxPerDoc', type: 'number', default: 4 },\n { key: 'RETRIEVAL_MAX_CONTEXT_TOKENS', name: 'maxContextTokens', type: 'number', default: 6000 },\n { key: 'DEPARTMENTS', name: 'departments', type: 'list', default: 'General' },\n { key: 'SLACK_SIGNING_SECRET', name: 'slackSigningSecret', default: '' },\n { key: 'SLACK_TEAM_ID', name: 'slackTeamId', default: '' },\n { key: 'SLACK_ALLOW_UNVERIFIED', name: 'slackAllowUnverified', type: 'boolean', default: false },\n { key: 'QUERY_API_TOKEN', name: 'queryApiToken', default: '' },\n];\n// --- end src/lib/config.js\n\n// --- inlined from src/lib/airtable.js -------------------------\n/**\n * Airtable helpers for the ingestion state table.\n *\n * The pipeline talks to Airtable over plain HTTP rather than through the\n * Airtable node, so that the record shape lives here in reviewable code instead\n * of inside a node's resource-mapper UI state.\n */\n\n/** Airtable formula strings are single-quoted; escape backslashes then quotes. */\nfunction escapeFormulaValue(value) {\n return String(value ?? '')\n .replace(/\\\\/g, '\\\\\\\\')\n .replace(/'/g, \"\\\\'\");\n}\n\nfunction lookupFormula(docIds) {\n const ids = [...new Set((docIds ?? []).filter(Boolean).map(String))];\n if (!ids.length) return { formula: 'FALSE()', doc_ids: [] };\n const clauses = ids.map((id) => `{doc_id} = '${escapeFormulaValue(id)}'`);\n return {\n formula: clauses.length === 1 ? clauses[0] : `OR(${clauses.join(', ')})`,\n doc_ids: ids,\n };\n}\n\n/** Airtable v2 responses nest under `fields`; flatten and keep the record id. */\nfunction flattenRecords(response) {\n const records = response?.records ?? (Array.isArray(response) ? response : []);\n return records.map((record) => ({\n record_id: record.id ?? null,\n ...(record.fields ?? record),\n }));\n}\n\nfunction indexByDocId(records) {\n const map = new Map();\n for (const record of records) {\n if (record.doc_id) map.set(String(record.doc_id), record);\n }\n return map;\n}\n\n/** Fields written when a document is successfully indexed. */\nfunction indexedFields(doc, nowIso) {\n return {\n doc_id: doc.doc_id,\n filename: doc.filename,\n drive_url: doc.web_view_link ?? null,\n department: doc.department ?? null,\n drive_md5: doc.drive_md5 ?? null,\n content_hash: doc.content_hash ?? null,\n status: 'Indexed',\n chunk_count: doc.chunk_count ?? 0,\n page_count: doc.page_count ?? 0,\n char_count: doc.char_count ?? 0,\n embedding_model: doc.embedding_model ?? null,\n index_namespace: doc.index_namespace ?? null,\n upload_date: doc.upload_date ? String(doc.upload_date).slice(0, 10) : null,\n modified_time: doc.modified_time ?? null,\n last_indexed: nowIso,\n last_checked: nowIso,\n retry_count: 0,\n error_notes: '',\n };\n}\n\n/** A document seen but deliberately not re-embedded. */\nfunction skippedFields(doc, nowIso) {\n return {\n doc_id: doc.doc_id,\n filename: doc.filename,\n drive_url: doc.web_view_link ?? null,\n department: doc.department ?? null,\n drive_md5: doc.drive_md5 ?? null,\n status: doc.existing?.status === 'Indexed' ? 'Indexed' : 'Skipped',\n last_checked: nowIso,\n };\n}\n\nfunction failedFields(doc, error, nowIso) {\n return {\n doc_id: doc.doc_id,\n filename: doc.filename ?? null,\n drive_url: doc.web_view_link ?? null,\n department: doc.department ?? null,\n drive_md5: doc.drive_md5 ?? null,\n status: 'Failed',\n retry_count: Number(doc.retry_count ?? 0) + 1,\n error_notes: String(error ?? 'unknown error').slice(0, 5000),\n last_checked: nowIso,\n };\n}\n\nfunction logFields({ docId, filename, event, detail, executionId, workflow }, nowIso) {\n return {\n timestamp: nowIso,\n doc_id: docId ?? null,\n filename: filename ?? null,\n event,\n detail: String(detail ?? '').slice(0, 5000),\n execution_id: executionId ?? null,\n workflow: workflow ?? null,\n };\n}\n\n/** Body for Airtable's upsert-on-PATCH. `typecast` lets new select options through. */\nfunction upsertBody(fieldsList, mergeOn = ['doc_id']) {\n return {\n performUpsert: { fieldsToMergeOn: mergeOn },\n typecast: true,\n records: fieldsList.map((fields) => ({ fields })),\n };\n}\n\nfunction createBody(fieldsList) {\n return { typecast: true, records: fieldsList.map((fields) => ({ fields })) };\n}\n// --- end src/lib/airtable.js\n\n/**\n * Node: \"Summarise Ingestion\"\n *\n * Collapses the per-batch fan-out back to one record per document and builds\n * the Airtable payload marking it indexed.\n */\n\nconst cfg = readConfig(requireEnv($env), INGESTION_CONFIG_SPEC);\nconst doc = $('Chunk Document').first().json;\n\nconst upserted = $('Pinecone: Upsert Vectors')\n .all()\n .reduce((total, item) => total + Number(item.json?.upsertedCount ?? 0), 0);\n\nif (upserted && upserted !== doc.chunk_count) {\n console.log(\n `WARNING: Pinecone reported ${upserted} upserts for ${doc.chunk_count} chunk(s) of ` +\n `\"${doc.filename}\".`,\n );\n}\n\nconst nowIso = new Date().toISOString();\nconst record = {\n ...doc,\n embedding_model: cfg.embeddingModel,\n index_namespace: cfg.pineconeNamespace,\n};\n\nreturn [\n {\n json: {\n doc_id: doc.doc_id,\n filename: doc.filename,\n department: doc.department,\n action: doc.action,\n chunk_count: doc.chunk_count,\n page_count: doc.page_count,\n vectors_upserted: upserted,\n content_hash: doc.content_hash,\n status: 'Indexed',\n fields: indexedFields(record, nowIso),\n log: logFields(\n {\n docId: doc.doc_id,\n filename: doc.filename,\n event: doc.action === 'reindex' ? 'Re-indexed' : 'Indexed',\n detail:\n `${doc.chunk_count} chunk(s) from ${doc.page_count} page(s); ` +\n `${upserted} vector(s) upserted to namespace \"${cfg.pineconeNamespace}\"; ` +\n `reason: ${doc.reason}`,\n executionId: $execution.id,\n workflow: $workflow.name,\n },\n nowIso,\n ),\n },\n },\n];\n" }, "id": "3e431c78-0224-476f-afd2-4d16d34fd0af", "name": "Summarise Ingestion", "type": "n8n-nodes-base.code", "typeVersion": 2, "position": [ 6600, 285 ] }, { "parameters": { "options": { "timeout": 30000 }, "method": "PATCH", "url": "=https://api.airtable.com/v0/{{ $env.AIRTABLE_BASE_ID }}/{{ encodeURIComponent($env.AIRTABLE_DOCUMENTS_TABLE || \"Documents\") }}", "authentication": "predefinedCredentialType", "nodeCredentialType": "airtableTokenApi", "sendBody": true, "specifyBody": "json", "jsonBody": "={{ JSON.stringify({ performUpsert: { fieldsToMergeOn: [\"doc_id\"] }, typecast: true, records: [{ fields: $json.fields }] }) }}" }, "id": "e3ea9e4f-f11b-41a1-a5ab-3eed0b42d3e3", "name": "Airtable: Upsert Document", "type": "n8n-nodes-base.httpRequest", "typeVersion": 4.2, "position": [ 6900, 285 ], "retryOnFail": true, "maxTries": 3, "waitBetweenTries": 2000 }, { "parameters": { "options": { "timeout": 30000 }, "method": "POST", "url": "=https://api.airtable.com/v0/{{ $env.AIRTABLE_BASE_ID }}/{{ encodeURIComponent($env.AIRTABLE_LOG_TABLE || \"Ingestion Log\") }}", "authentication": "predefinedCredentialType", "nodeCredentialType": "airtableTokenApi", "sendBody": true, "specifyBody": "json", "jsonBody": "={{ JSON.stringify({ typecast: true, records: [{ fields: $('Summarise Ingestion').first().json.log }] }) }}" }, "id": "abd67453-f23a-41bc-a43c-4ad67100e0c0", "name": "Airtable: Log Ingestion", "type": "n8n-nodes-base.httpRequest", "typeVersion": 4.2, "position": [ 7200, 285 ], "retryOnFail": true, "maxTries": 3, "waitBetweenTries": 2000 }, { "parameters": { "mode": "runOnceForAllItems", "jsCode": "// --- inlined from src/lib/airtable.js -------------------------\n/**\n * Airtable helpers for the ingestion state table.\n *\n * The pipeline talks to Airtable over plain HTTP rather than through the\n * Airtable node, so that the record shape lives here in reviewable code instead\n * of inside a node's resource-mapper UI state.\n */\n\n/** Airtable formula strings are single-quoted; escape backslashes then quotes. */\nfunction escapeFormulaValue(value) {\n return String(value ?? '')\n .replace(/\\\\/g, '\\\\\\\\')\n .replace(/'/g, \"\\\\'\");\n}\n\nfunction lookupFormula(docIds) {\n const ids = [...new Set((docIds ?? []).filter(Boolean).map(String))];\n if (!ids.length) return { formula: 'FALSE()', doc_ids: [] };\n const clauses = ids.map((id) => `{doc_id} = '${escapeFormulaValue(id)}'`);\n return {\n formula: clauses.length === 1 ? clauses[0] : `OR(${clauses.join(', ')})`,\n doc_ids: ids,\n };\n}\n\n/** Airtable v2 responses nest under `fields`; flatten and keep the record id. */\nfunction flattenRecords(response) {\n const records = response?.records ?? (Array.isArray(response) ? response : []);\n return records.map((record) => ({\n record_id: record.id ?? null,\n ...(record.fields ?? record),\n }));\n}\n\nfunction indexByDocId(records) {\n const map = new Map();\n for (const record of records) {\n if (record.doc_id) map.set(String(record.doc_id), record);\n }\n return map;\n}\n\n/** Fields written when a document is successfully indexed. */\nfunction indexedFields(doc, nowIso) {\n return {\n doc_id: doc.doc_id,\n filename: doc.filename,\n drive_url: doc.web_view_link ?? null,\n department: doc.department ?? null,\n drive_md5: doc.drive_md5 ?? null,\n content_hash: doc.content_hash ?? null,\n status: 'Indexed',\n chunk_count: doc.chunk_count ?? 0,\n page_count: doc.page_count ?? 0,\n char_count: doc.char_count ?? 0,\n embedding_model: doc.embedding_model ?? null,\n index_namespace: doc.index_namespace ?? null,\n upload_date: doc.upload_date ? String(doc.upload_date).slice(0, 10) : null,\n modified_time: doc.modified_time ?? null,\n last_indexed: nowIso,\n last_checked: nowIso,\n retry_count: 0,\n error_notes: '',\n };\n}\n\n/** A document seen but deliberately not re-embedded. */\nfunction skippedFields(doc, nowIso) {\n return {\n doc_id: doc.doc_id,\n filename: doc.filename,\n drive_url: doc.web_view_link ?? null,\n department: doc.department ?? null,\n drive_md5: doc.drive_md5 ?? null,\n status: doc.existing?.status === 'Indexed' ? 'Indexed' : 'Skipped',\n last_checked: nowIso,\n };\n}\n\nfunction failedFields(doc, error, nowIso) {\n return {\n doc_id: doc.doc_id,\n filename: doc.filename ?? null,\n drive_url: doc.web_view_link ?? null,\n department: doc.department ?? null,\n drive_md5: doc.drive_md5 ?? null,\n status: 'Failed',\n retry_count: Number(doc.retry_count ?? 0) + 1,\n error_notes: String(error ?? 'unknown error').slice(0, 5000),\n last_checked: nowIso,\n };\n}\n\nfunction logFields({ docId, filename, event, detail, executionId, workflow }, nowIso) {\n return {\n timestamp: nowIso,\n doc_id: docId ?? null,\n filename: filename ?? null,\n event,\n detail: String(detail ?? '').slice(0, 5000),\n execution_id: executionId ?? null,\n workflow: workflow ?? null,\n };\n}\n\n/** Body for Airtable's upsert-on-PATCH. `typecast` lets new select options through. */\nfunction upsertBody(fieldsList, mergeOn = ['doc_id']) {\n return {\n performUpsert: { fieldsToMergeOn: mergeOn },\n typecast: true,\n records: fieldsList.map((fields) => ({ fields })),\n };\n}\n\nfunction createBody(fieldsList) {\n return { typecast: true, records: fieldsList.map((fields) => ({ fields })) };\n}\n// --- end src/lib/airtable.js\n\n/**\n * Node: \"Build Skip Record\"\n *\n * Reached by two paths: a document whose Drive checksum was unchanged (never\n * downloaded), and one whose extracted text hashed identically despite new\n * bytes. Both update `last_checked` so the ops view can distinguish \"still\n * current\" from \"not looked at since March\".\n */\n\nconst doc = $input.first().json;\nconst nowIso = new Date().toISOString();\n\nconst contentIdentical = doc.content_hash && !doc.content_changed;\nconst detail = contentIdentical\n ? `Drive checksum changed but extracted text is identical (${doc.content_hash}); not re-embedded`\n : doc.reason;\n\nreturn [\n {\n json: {\n doc_id: doc.doc_id,\n filename: doc.filename,\n status: 'skipped',\n reason: detail,\n // Only the Documents row is touched. Skips are the common case on a\n // sweep, and a log row per unchanged file is noise that costs API calls.\n fields: skippedFields(doc, nowIso),\n },\n },\n];\n" }, "id": "bfc4b7b7-826a-4756-a103-5020d91fb734", "name": "Build Skip Record", "type": "n8n-nodes-base.code", "typeVersion": 2, "position": [ 3900, 760 ], "notes": "Reached by both duplicate checks - unchanged checksum, or unchanged text." }, { "parameters": { "options": { "timeout": 30000 }, "method": "PATCH", "url": "=https://api.airtable.com/v0/{{ $env.AIRTABLE_BASE_ID }}/{{ encodeURIComponent($env.AIRTABLE_DOCUMENTS_TABLE || \"Documents\") }}", "authentication": "predefinedCredentialType", "nodeCredentialType": "airtableTokenApi", "sendBody": true, "specifyBody": "json", "jsonBody": "={{ JSON.stringify({ performUpsert: { fieldsToMergeOn: [\"doc_id\"] }, typecast: true, records: [{ fields: $json.fields }] }) }}" }, "id": "00c599ca-ba62-403d-a5f5-07cc2a170ab2", "name": "Airtable: Record Skip", "type": "n8n-nodes-base.httpRequest", "typeVersion": 4.2, "position": [ 4200, 760 ], "notes": "Updates last_checked only. Skips are the common case and do not need a log row.", "retryOnFail": true, "maxTries": 3, "waitBetweenTries": 2000 }, { "parameters": { "mode": "runOnceForAllItems", "jsCode": "// --- inlined from src/lib/airtable.js -------------------------\n/**\n * Airtable helpers for the ingestion state table.\n *\n * The pipeline talks to Airtable over plain HTTP rather than through the\n * Airtable node, so that the record shape lives here in reviewable code instead\n * of inside a node's resource-mapper UI state.\n */\n\n/** Airtable formula strings are single-quoted; escape backslashes then quotes. */\nfunction escapeFormulaValue(value) {\n return String(value ?? '')\n .replace(/\\\\/g, '\\\\\\\\')\n .replace(/'/g, \"\\\\'\");\n}\n\nfunction lookupFormula(docIds) {\n const ids = [...new Set((docIds ?? []).filter(Boolean).map(String))];\n if (!ids.length) return { formula: 'FALSE()', doc_ids: [] };\n const clauses = ids.map((id) => `{doc_id} = '${escapeFormulaValue(id)}'`);\n return {\n formula: clauses.length === 1 ? clauses[0] : `OR(${clauses.join(', ')})`,\n doc_ids: ids,\n };\n}\n\n/** Airtable v2 responses nest under `fields`; flatten and keep the record id. */\nfunction flattenRecords(response) {\n const records = response?.records ?? (Array.isArray(response) ? response : []);\n return records.map((record) => ({\n record_id: record.id ?? null,\n ...(record.fields ?? record),\n }));\n}\n\nfunction indexByDocId(records) {\n const map = new Map();\n for (const record of records) {\n if (record.doc_id) map.set(String(record.doc_id), record);\n }\n return map;\n}\n\n/** Fields written when a document is successfully indexed. */\nfunction indexedFields(doc, nowIso) {\n return {\n doc_id: doc.doc_id,\n filename: doc.filename,\n drive_url: doc.web_view_link ?? null,\n department: doc.department ?? null,\n drive_md5: doc.drive_md5 ?? null,\n content_hash: doc.content_hash ?? null,\n status: 'Indexed',\n chunk_count: doc.chunk_count ?? 0,\n page_count: doc.page_count ?? 0,\n char_count: doc.char_count ?? 0,\n embedding_model: doc.embedding_model ?? null,\n index_namespace: doc.index_namespace ?? null,\n upload_date: doc.upload_date ? String(doc.upload_date).slice(0, 10) : null,\n modified_time: doc.modified_time ?? null,\n last_indexed: nowIso,\n last_checked: nowIso,\n retry_count: 0,\n error_notes: '',\n };\n}\n\n/** A document seen but deliberately not re-embedded. */\nfunction skippedFields(doc, nowIso) {\n return {\n doc_id: doc.doc_id,\n filename: doc.filename,\n drive_url: doc.web_view_link ?? null,\n department: doc.department ?? null,\n drive_md5: doc.drive_md5 ?? null,\n status: doc.existing?.status === 'Indexed' ? 'Indexed' : 'Skipped',\n last_checked: nowIso,\n };\n}\n\nfunction failedFields(doc, error, nowIso) {\n return {\n doc_id: doc.doc_id,\n filename: doc.filename ?? null,\n drive_url: doc.web_view_link ?? null,\n department: doc.department ?? null,\n drive_md5: doc.drive_md5 ?? null,\n status: 'Failed',\n retry_count: Number(doc.retry_count ?? 0) + 1,\n error_notes: String(error ?? 'unknown error').slice(0, 5000),\n last_checked: nowIso,\n };\n}\n\nfunction logFields({ docId, filename, event, detail, executionId, workflow }, nowIso) {\n return {\n timestamp: nowIso,\n doc_id: docId ?? null,\n filename: filename ?? null,\n event,\n detail: String(detail ?? '').slice(0, 5000),\n execution_id: executionId ?? null,\n workflow: workflow ?? null,\n };\n}\n\n/** Body for Airtable's upsert-on-PATCH. `typecast` lets new select options through. */\nfunction upsertBody(fieldsList, mergeOn = ['doc_id']) {\n return {\n performUpsert: { fieldsToMergeOn: mergeOn },\n typecast: true,\n records: fieldsList.map((fields) => ({ fields })),\n };\n}\n\nfunction createBody(fieldsList) {\n return { typecast: true, records: fieldsList.map((fields) => ({ fields })) };\n}\n// --- end src/lib/airtable.js\n\n/**\n * Node: \"Build Failure Record\"\n *\n * Sits on the error output of the download / extract / embed / upsert nodes.\n * Those nodes are configured to continue on error rather than abort, so one\n * unreadable PDF marks itself Failed and the loop moves on to the next\n * document instead of stranding the rest of the queue.\n */\n\nconst item = $input.first();\nconst doc = $('Loop Over Documents').first().json;\n\nconst error = item.json?.error ?? {};\nconst message =\n (typeof error === 'string' ? error : error.message ?? error.description) ??\n item.json?.message ??\n 'unknown error';\n\nconst nowIso = new Date().toISOString();\nconst detail = `${message}${error.httpCode ? ` (HTTP ${error.httpCode})` : ''}`;\n\nconsole.log(`FAILED ${doc.filename}: ${detail}`);\n\nreturn [\n {\n json: {\n doc_id: doc.doc_id,\n filename: doc.filename,\n status: 'failed',\n error: detail,\n retry_count: Number(doc.retry_count ?? 0) + 1,\n fields: failedFields(doc, detail, nowIso),\n log: logFields(\n {\n docId: doc.doc_id,\n filename: doc.filename,\n event: 'Failed',\n detail,\n executionId: $execution.id,\n workflow: $workflow.name,\n },\n nowIso,\n ),\n },\n },\n];\n" }, "id": "66246475-2354-41b7-aa79-3ce6a41c6b4a", "name": "Build Failure Record", "type": "n8n-nodes-base.code", "typeVersion": 2, "position": [ 3900, 1045 ], "notes": "Error output of every external call in the loop lands here." }, { "parameters": { "options": { "timeout": 30000 }, "method": "PATCH", "url": "=https://api.airtable.com/v0/{{ $env.AIRTABLE_BASE_ID }}/{{ encodeURIComponent($env.AIRTABLE_DOCUMENTS_TABLE || \"Documents\") }}", "authentication": "predefinedCredentialType", "nodeCredentialType": "airtableTokenApi", "sendBody": true, "specifyBody": "json", "jsonBody": "={{ JSON.stringify({ performUpsert: { fieldsToMergeOn: [\"doc_id\"] }, typecast: true, records: [{ fields: $json.fields }] }) }}" }, "id": "860cef40-54d3-4af9-a86b-7c423e318b5f", "name": "Airtable: Mark Failed", "type": "n8n-nodes-base.httpRequest", "typeVersion": 4.2, "position": [ 4200, 1045 ], "retryOnFail": true, "maxTries": 3, "waitBetweenTries": 2000 }, { "parameters": { "options": { "timeout": 30000 }, "method": "POST", "url": "=https://api.airtable.com/v0/{{ $env.AIRTABLE_BASE_ID }}/{{ encodeURIComponent($env.AIRTABLE_LOG_TABLE || \"Ingestion Log\") }}", "authentication": "predefinedCredentialType", "nodeCredentialType": "airtableTokenApi", "sendBody": true, "specifyBody": "json", "jsonBody": "={{ JSON.stringify({ typecast: true, records: [{ fields: $('Build Failure Record').first().json.log }] }) }}" }, "id": "b4cd8cfc-3ee8-4678-a537-4b9d1de46ef4", "name": "Airtable: Log Failure", "type": "n8n-nodes-base.httpRequest", "typeVersion": 4.2, "position": [ 4500, 1045 ], "retryOnFail": true, "maxTries": 3, "waitBetweenTries": 2000 }, { "parameters": { "content": "## Entry points\n\nFour ways in, one pipeline:\n- **File Created / File Updated** - Drive polling, every 5 min\n- **Backfill** - run manually to index a folder that already has files\n- **Nightly Sweep** - reconciliation; also retries earlier failures\n\nAll four converge on `Normalize File Event`.", "width": 520, "height": 240, "color": 4 }, "id": "f8f2a5e1-4840-45f9-a754-da75ead16b97", "name": "Note: Entry Points", "type": "n8n-nodes-base.stickyNote", "typeVersion": 1, "position": [ -60, -260 ] }, { "parameters": { "content": "## Why a file gets skipped\n\n**Tier 1 - `Decide Action`**: Drive md5 unchanged, so do not even download.\nDrive triggers fire on renames and sharing changes, not just content.\n\n**Tier 2 - `Content Changed?`**: bytes changed but extracted text hashes the same.\nA re-export or re-compression costs nothing to skip.\n\nEmbeddings are the expensive step; both checks sit in front of it.", "width": 560, "height": 280, "color": 5 }, "id": "16ef63c0-ff30-49dc-adad-9ab2081996cb", "name": "Note: Duplicate Protection", "type": "n8n-nodes-base.stickyNote", "typeVersion": 1, "position": [ 1980, -300 ] }, { "parameters": { "content": "## Failures do not stop the queue\n\nDownload, extraction, embedding and upsert all use n8n error outputs\n(the red branch) rather than aborting.\n\nA corrupt or scanned PDF marks its own Airtable row `Failed` with the\nreason, and the loop continues to the next document.\n\n`retry_count` is what stops it retrying forever - see MAX_INGEST_RETRIES.", "width": 560, "height": 260, "color": 3 }, "id": "221a8b29-85a5-4360-ae77-829b4b54c615", "name": "Note: Failure Isolation", "type": "n8n-nodes-base.stickyNote", "typeVersion": 1, "position": [ 3660, 1180 ] }, { "parameters": { "content": "## Stale vector cleanup\n\nVector ids are `{doc_id}#{chunk_index}`, so re-indexing overwrites in place.\n\nWhen a revised document produces *fewer* chunks, the leftover tail would\nstay retrievable and citable - quoting text the document no longer contains.\n\nThose ids are computed and deleted. Pinecone serverless has no\ndelete-by-metadata-filter, so computing them is the supported route.", "width": 580, "height": 300, "color": 6 }, "id": "84c538c3-5644-4aac-a9db-f0d0b1956802", "name": "Note: Vector Hygiene", "type": "n8n-nodes-base.stickyNote", "typeVersion": 1, "position": [ 5700, -320 ] } ], "connections": { "Drive Trigger: File Created": { "main": [ [ { "node": "Validate Config", "type": "main", "index": 0 } ] ] }, "Drive Trigger: File Updated": { "main": [ [ { "node": "Validate Config", "type": "main", "index": 0 } ] ] }, "Manual Trigger: Backfill Folder": { "main": [ [ { "node": "Validate Config", "type": "main", "index": 0 } ] ] }, "Schedule: Nightly Sweep": { "main": [ [ { "node": "Validate Config", "type": "main", "index": 0 } ] ] }, "Validate Config": { "main": [ [ { "node": "Has File Id?", "type": "main", "index": 0 } ] ] }, "Has File Id?": { "main": [ [ { "node": "Drive: Get File Metadata", "type": "main", "index": 0 } ], [ { "node": "Drive: List PDFs In Folder", "type": "main", "index": 0 } ] ] }, "Drive: Get File Metadata": { "main": [ [ { "node": "Normalize File Event", "type": "main", "index": 0 } ] ] }, "Drive: List PDFs In Folder": { "main": [ [ { "node": "Normalize File Event", "type": "main", "index": 0 } ] ] }, "Normalize File Event": { "main": [ [ { "node": "Build Airtable Lookup", "type": "main", "index": 0 } ] ] }, "Build Airtable Lookup": { "main": [ [ { "node": "Airtable: Lookup Documents", "type": "main", "index": 0 } ] ] }, "Airtable: Lookup Documents": { "main": [ [ { "node": "Decide Action", "type": "main", "index": 0 } ] ] }, "Decide Action": { "main": [ [ { "node": "Loop Over Documents", "type": "main", "index": 0 } ] ] }, "Loop Over Documents": { "main": [ [ { "node": "Run Summary", "type": "main", "index": 0 } ], [ { "node": "Needs Processing?", "type": "main", "index": 0 } ] ] }, "Run Summary": { "main": [ [ { "node": "Notify?", "type": "main", "index": 0 } ] ] }, "Notify?": { "main": [ [ { "node": "Slack: Post Run Summary", "type": "main", "index": 0 } ], [ { "node": "Ingestion Complete", "type": "main", "index": 0 } ] ] }, "Slack: Post Run Summary": { "main": [ [ { "node": "Ingestion Complete", "type": "main", "index": 0 } ] ] }, "Needs Processing?": { "main": [ [ { "node": "Drive: Download File", "type": "main", "index": 0 } ], [ { "node": "Build Skip Record", "type": "main", "index": 0 } ] ] }, "Drive: Download File": { "main": [ [ { "node": "Extract PDF Text", "type": "main", "index": 0 } ], [ { "node": "Build Failure Record", "type": "main", "index": 0 } ] ] }, "Extract PDF Text": { "main": [ [ { "node": "Chunk Document", "type": "main", "index": 0 } ], [ { "node": "Build Failure Record", "type": "main", "index": 0 } ] ] }, "Chunk Document": { "main": [ [ { "node": "Content Changed?", "type": "main", "index": 0 } ], [ { "node": "Build Failure Record", "type": "main", "index": 0 } ] ] }, "Content Changed?": { "main": [ [ { "node": "Airtable: Mark Indexing", "type": "main", "index": 0 } ], [ { "node": "Build Skip Record", "type": "main", "index": 0 } ] ] }, "Airtable: Mark Indexing": { "main": [ [ { "node": "Batch Chunks For Embedding", "type": "main", "index": 0 } ] ] }, "Batch Chunks For Embedding": { "main": [ [ { "node": "OpenAI: Create Embeddings", "type": "main", "index": 0 } ] ] }, "OpenAI: Create Embeddings": { "main": [ [ { "node": "Build Pinecone Vectors", "type": "main", "index": 0 } ], [ { "node": "Build Failure Record", "type": "main", "index": 0 } ] ] }, "Build Pinecone Vectors": { "main": [ [ { "node": "Pinecone: Upsert Vectors", "type": "main", "index": 0 } ] ] }, "Pinecone: Upsert Vectors": { "main": [ [ { "node": "Build Stale Vector Purge", "type": "main", "index": 0 } ], [ { "node": "Build Failure Record", "type": "main", "index": 0 } ] ] }, "Build Stale Vector Purge": { "main": [ [ { "node": "Has Stale Vectors?", "type": "main", "index": 0 } ] ] }, "Has Stale Vectors?": { "main": [ [ { "node": "Pinecone: Delete Stale Vectors", "type": "main", "index": 0 } ], [ { "node": "Summarise Ingestion", "type": "main", "index": 0 } ] ] }, "Pinecone: Delete Stale Vectors": { "main": [ [ { "node": "Summarise Ingestion", "type": "main", "index": 0 } ] ] }, "Summarise Ingestion": { "main": [ [ { "node": "Airtable: Upsert Document", "type": "main", "index": 0 } ] ] }, "Airtable: Upsert Document": { "main": [ [ { "node": "Airtable: Log Ingestion", "type": "main", "index": 0 } ] ] }, "Airtable: Log Ingestion": { "main": [ [ { "node": "Loop Over Documents", "type": "main", "index": 0 } ] ] }, "Build Skip Record": { "main": [ [ { "node": "Airtable: Record Skip", "type": "main", "index": 0 } ] ] }, "Airtable: Record Skip": { "main": [ [ { "node": "Loop Over Documents", "type": "main", "index": 0 } ] ] }, "Build Failure Record": { "main": [ [ { "node": "Airtable: Mark Failed", "type": "main", "index": 0 } ] ] }, "Airtable: Mark Failed": { "main": [ [ { "node": "Airtable: Log Failure", "type": "main", "index": 0 } ] ] }, "Airtable: Log Failure": { "main": [ [ { "node": "Loop Over Documents", "type": "main", "index": 0 } ] ] } }, "settings": { "executionOrder": "v1" }, "meta": { "templateCredsSetupCompleted": false } }