Bewertung der Relevanz von KI-Agentenantworten mit OpenAI und Kosinus-Ähnlichkeit

• •Möglicherweise sind Ziel-API-Anmeldedaten erforderlich
• •OpenAI API Key

{
  "meta": {
    "instanceId": "408f9fb9940c3cb18ffdef0e0150fe342d6e655c3a9fac21f0f644e8bedabcd9",
    "templateCredsSetupCompleted": true
  },
  "nodes": [
    {
      "id": "49b274c7-1ff2-43ba-acb9-349a8665b513",
      "name": "OpenAI Chat Model1",
      "type": "@n8n/n8n-nodes-langchain.lmChatOpenAi",
      "position": [
        148,
        180
      ],
      "parameters": {
        "model": {
          "__rl": true,
          "mode": "list",
          "value": "gpt-4.1-mini",
          "cachedResultName": "gpt-4.1-mini"
        },
        "options": {}
      },
      "credentials": {
        "openAiApi": {
          "id": "8gccIjcuf3gvaoEr",
          "name": "OpenAi account"
        }
      },
      "typeVersion": 1.2
    },
    {
      "id": "77f0c130-085f-4fa5-b74e-66fd084949b7",
      "name": "Beim Abrufen eines Datensatzzeile",
      "type": "n8n-nodes-base.evaluationTrigger",
      "position": [
        -380,
        -140
      ],
      "parameters": {
        "sheetName": {
          "__rl": true,
          "mode": "list",
          "value": 2015506025,
          "cachedResultUrl": "https://docs.google.com/spreadsheets/d/1YOnu2JJjlxd787AuYcg-wKbkjyjyZFgASYVV0jsij5Y/edit#gid=2015506025",
          "cachedResultName": "Relevance"
        },
        "documentId": {
          "__rl": true,
          "mode": "list",
          "value": "1YOnu2JJjlxd787AuYcg-wKbkjyjyZFgASYVV0jsij5Y",
          "cachedResultUrl": "https://docs.google.com/spreadsheets/d/1YOnu2JJjlxd787AuYcg-wKbkjyjyZFgASYVV0jsij5Y/edit?usp=drivesdk",
          "cachedResultName": "96. Evaluations Test"
        }
      },
      "credentials": {
        "googleSheetsOAuth2Api": {
          "id": "XHvC7jIRR8A2TlUl",
          "name": "Google Sheets account"
        }
      },
      "typeVersion": 4.6
    },
    {
      "id": "63d8ff44-5c4b-4cf6-b5b2-1f22de0de495",
      "name": "Eingabe neu zuordnen",
      "type": "n8n-nodes-base.set",
      "position": [
        -160,
        -140
      ],
      "parameters": {
        "options": {},
        "assignments": {
          "assignments": [
            {
              "id": "00924b90-278f-49f5-80f2-c297df0fcc97",
              "name": "chatInput",
              "type": "string",
              "value": "={{ $json.input }}"
            }
          ]
        }
      },
      "typeVersion": 3.4
    },
    {
      "id": "d8ac1b2b-f4ee-4af1-92f5-d194832c5c76",
      "name": "Evaluation",
      "type": "n8n-nodes-base.evaluation",
      "position": [
        436,
        -40
      ],
      "parameters": {
        "operation": "checkIfEvaluating"
      },
      "typeVersion": 4.6
    },
    {
      "id": "1d8d9c97-0c90-4878-95fa-0dc6d3ad82c6",
      "name": "Eingabefelder setzen",
      "type": "n8n-nodes-base.set",
      "position": [
        656,
        -140
      ],
      "parameters": {
        "options": {},
        "assignments": {
          "assignments": [
            {
              "id": "d58952c1-d346-4fbf-881e-d5c04b6781a5",
              "name": "question",
              "type": "string",
              "value": "={{ $('When fetching a dataset row').first().json.input }}"
            },
            {
              "id": "0f10a3d0-cf6e-4715-9ded-2cee54aa62ec",
              "name": "answer",
              "type": "string",
              "value": "={{ $json.output }}"
            }
          ]
        }
      },
      "typeVersion": 3.4
    },
    {
      "id": "d4ad7c5e-f1cf-463b-b9b3-a45ff5597199",
      "name": "Keine Operation, nichts tun",
      "type": "n8n-nodes-base.noOp",
      "position": [
        656,
        60
      ],
      "parameters": {},
      "typeVersion": 1
    },
    {
      "id": "1aea189f-3e7d-45d2-9802-d44b1dce4797",
      "name": "KI-Agent",
      "type": "@n8n/n8n-nodes-langchain.agent",
      "position": [
        60,
        -40
      ],
      "parameters": {
        "options": {}
      },
      "typeVersion": 2
    },
    {
      "id": "54596db1-4ba7-414c-88cf-250f528d6c21",
      "name": "Bei Empfang einer Chat-Nachricht",
      "type": "@n8n/n8n-nodes-langchain.chatTrigger",
      "position": [
        -160,
        60
      ],
      "webhookId": "ba1fadeb-b566-469a-97b3-3159a99f1805",
      "parameters": {
        "options": {}
      },
      "typeVersion": 1.1
    },
    {
      "id": "4184f202-48f5-4e58-a0e8-0227a23553f3",
      "name": "Ausgabe aktualisieren",
      "type": "n8n-nodes-base.evaluation",
      "position": [
        2132,
        -140
      ],
      "parameters": {
        "outputs": {
          "values": [
            {
              "outputName": "output",
              "outputValue": "={{ $('Set Input Fields').first().json.answer }}"
            },
            {
              "outputName": "score",
              "outputValue": "={{ $json.score }}"
            }
          ]
        },
        "sheetName": {
          "__rl": true,
          "mode": "list",
          "value": 2015506025,
          "cachedResultUrl": "https://docs.google.com/spreadsheets/d/1YOnu2JJjlxd787AuYcg-wKbkjyjyZFgASYVV0jsij5Y/edit#gid=2015506025",
          "cachedResultName": "Relevance"
        },
        "documentId": {
          "__rl": true,
          "mode": "list",
          "value": "1YOnu2JJjlxd787AuYcg-wKbkjyjyZFgASYVV0jsij5Y",
          "cachedResultUrl": "https://docs.google.com/spreadsheets/d/1YOnu2JJjlxd787AuYcg-wKbkjyjyZFgASYVV0jsij5Y/edit?usp=drivesdk",
          "cachedResultName": "96. Evaluations Test"
        }
      },
      "credentials": {
        "googleSheetsOAuth2Api": {
          "id": "XHvC7jIRR8A2TlUl",
          "name": "Google Sheets account"
        }
      },
      "typeVersion": 4.6
    },
    {
      "id": "fe7b41d2-e382-4f59-8b0c-06361062581d",
      "name": "Metriken aktualisieren",
      "type": "n8n-nodes-base.evaluation",
      "position": [
        2352,
        -140
      ],
      "parameters": {
        "metrics": {
          "assignments": [
            {
              "id": "1fd7759c-f4ef-4eda-87ad-9d9563b63e99",
              "name": "score",
              "type": "number",
              "value": "={{ $json.score }}"
            }
          ]
        },
        "operation": "setMetrics"
      },
      "typeVersion": 4.6
    },
    {
      "id": "04f714cd-99ce-45a9-bc43-351fb6ea14d4",
      "name": "OpenAI Chat Model",
      "type": "@n8n/n8n-nodes-langchain.lmChatOpenAi",
      "position": [
        920,
        80
      ],
      "parameters": {
        "model": {
          "__rl": true,
          "mode": "list",
          "value": "gpt-4.1-mini",
          "cachedResultName": "gpt-4.1-mini"
        },
        "options": {}
      },
      "credentials": {
        "openAiApi": {
          "id": "8gccIjcuf3gvaoEr",
          "name": "OpenAi account"
        }
      },
      "typeVersion": 1.2
    },
    {
      "id": "d821b0ec-93c9-46e1-ab03-280aeb9b87b1",
      "name": "Strukturierter Ausgabe-Parser",
      "type": "@n8n/n8n-nodes-langchain.outputParserStructured",
      "position": [
        1040,
        80
      ],
      "parameters": {
        "jsonSchemaExample": "{\n  \"question\": \"Where was Albert Einstein born?\",\n  \"noncommittal\": 0\n}"
      },
      "typeVersion": 1.2
    },
    {
      "id": "be975ef4-c50b-4bf9-b871-cc8d479661eb",
      "name": "Antwortrelevanz",
      "type": "@n8n/n8n-nodes-langchain.chainLlm",
      "position": [
        900,
        -140
      ],
      "parameters": {
        "text": "={{ $json.answer }}",
        "batching": {},
        "messages": {
          "messageValues": [
            {
              "message": "=Generate a question for the given answer and Identify if answer is noncommittal. Give noncommittal as 1 if the answer is noncommittal and 0 if the answer is committal. A noncommittal answer is one that is evasive, vague, or ambiguous. For example, \"I don't know\" or \"I'm not sure\" are noncommittal answers\""
            }
          ]
        },
        "promptType": "define",
        "hasOutputParser": true
      },
      "typeVersion": 1.7
    },
    {
      "id": "8e3df14a-7754-4257-8209-a24cfa30dbbc",
      "name": "Kurznotiz1",
      "type": "n8n-nodes-base.stickyNote",
      "position": [
        -452,
        -400
      ],
      "parameters": {
        "color": 7,
        "width": 840,
        "height": 720,
        "content": "## 1. Setup Your AI Workflow to Use Evaluations\n[Learn more about the Evaluations Trigger](https://docs.n8n.io/integrations/builtin/?utm_source=n8n_app&utm_medium=node_settings_modal-credential_link&utm_campaign=n8n-nodes-base.evaluationTrigger)\n\nThe Evaluations Trigger is a separate execution which does not affect your production workflow in any way. It is manually triggered and automatically pulled datasets from the assigned Google Sheet."
      },
      "typeVersion": 1
    },
    {
      "id": "eeeac9d0-39c6-4d17-8d93-9c60b6c612f8",
      "name": "Kurznotiz",
      "type": "n8n-nodes-base.stickyNote",
      "position": [
        820,
        -400
      ],
      "parameters": {
        "color": 7,
        "width": 1200,
        "height": 720,
        "content": "## 2. Answer Relevance: How relevant is the agent response to the user question?\n[Learn more about the Evaluations Trigger](https://docs.n8n.io/integrations/builtin/?utm_source=n8n_app&utm_medium=node_settings_modal-credential_link&utm_campaign=n8n-nodes-base.evaluationTrigger)\n\nFor this evaluation, we analyse the agent's answer and ask another AI to generate a question for it - kinda like the game show [Jeopardy!](https://en.wikipedia.org/wiki/Jeopardy!).\nWe do this in order to compare the generated question with the original question for the test. If the questions are similar, then relevancy is high."
      },
      "typeVersion": 1
    },
    {
      "id": "c86f66e2-9997-4ca6-9ec6-cfa185d404f2",
      "name": "Kurznotiz3",
      "type": "n8n-nodes-base.stickyNote",
      "position": [
        -920,
        -340
      ],
      "parameters": {
        "width": 420,
        "height": 720,
        "content": "## Try It Out!\n### This n8n template demonstrates how to calculate the evaluation metric \"Relevance\" which in this scenario, measures the relevance of the agent's response to the user's question.\n\nThe scoring approach is adapted from [https://github.com/explodinggradients/ragas/blob/main/ragas/src/ragas/metrics/_answer_relevance.py](https://github.com/explodinggradients/ragas/blob/main/ragas/src/ragas/metrics/_answer_relevance.py)\n\n### How it works\n* This evaluation works best for Q&A agents.\n* For our scoring, we analyse the agent's response and ask another AI to generate a question from it. This generated question is then compared to the original question using cosine similarity.\n* A high score indicates relevance and the agent's successful ability to answer the question whereas a low score means agent may have added too much irrelevant info, went off script or hallucinated.\n\n### Requirements\n* n8n version 1.94+\n* Check out this Google Sheet for a sample data [https://docs.google.com/spreadsheets/d/1YOnu2JJjlxd787AuYcg-wKbkjyjyZFgASYVV0jsij5Y/edit?usp=sharing](https://docs.google.com/spreadsheets/d/1YOnu2JJjlxd787AuYcg-wKbkjyjyZFgASYVV0jsij5Y/edit?usp=sharing)\n\n\n### Need Help?\nJoin the [Discord](https://discord.com/invite/XPKeKXeB7d) or ask in the [Forum](https://community.n8n.io/)!\n\nHappy Hacking!"
      },
      "typeVersion": 1
    },
    {
      "id": "3bdd3254-7cdf-455a-ae63-9639069993bc",
      "name": "Embeddings abrufen",
      "type": "n8n-nodes-base.httpRequest",
      "position": [
        1460,
        -140
      ],
      "parameters": {
        "url": "https://api.openai.com/v1/embeddings",
        "method": "POST",
        "options": {},
        "sendBody": true,
        "authentication": "predefinedCredentialType",
        "bodyParameters": {
          "parameters": [
            {
              "name": "input",
              "value": "={{ $json.data }}"
            },
            {
              "name": "model",
              "value": "text-embedding-3-small"
            },
            {
              "name": "encoding_format",
              "value": "float"
            }
          ]
        },
        "nodeCredentialType": "openAiApi"
      },
      "credentials": {
        "openAiApi": {
          "id": "8gccIjcuf3gvaoEr",
          "name": "OpenAi account"
        }
      },
      "typeVersion": 4.2
    },
    {
      "id": "9e3bcdee-5f24-45e6-a85f-6dcfacb58d5d",
      "name": "Ähnlichkeitswert berechnen",
      "type": "n8n-nodes-base.code",
      "position": [
        1660,
        -140
      ],
      "parameters": {
        "jsCode": "const [vectorsA, vectorsB] = $input.all().map(item => item.json.data[0].embedding);\nconst score = cosineSimilarity(vectorsA, vectorsB);\n\nreturn { json: { similarityScore: score } };\n\nfunction cosineSimilarity(a, b) {  \n  let dotProduct = normA = normB = 0;\n  for (let i = 0; i < a.length; i++) {\n    dotProduct += a[i] * b[i];\n    normA += a[i] ** 2;\n    normB += b[i] ** 2;\n  }\n  return dotProduct / (Math.sqrt(normA) * Math.sqrt(normB));\n}"
      },
      "typeVersion": 2
    },
    {
      "id": "69c30814-9725-426d-9572-9f92bd81dea5",
      "name": "Fragen zu Elementen",
      "type": "n8n-nodes-base.code",
      "position": [
        1260,
        -140
      ],
      "parameters": {
        "jsCode": "return [\n  { json: { data: $('Set Input Fields').first().json.question } },\n  { json: { data: $input.first().json.output.question } }\n]"
      },
      "typeVersion": 2
    },
    {
      "id": "83be23ff-6857-4265-b1d0-f4b682c66903",
      "name": "Relevanzwert berechnen",
      "type": "n8n-nodes-base.set",
      "position": [
        1860,
        -140
      ],
      "parameters": {
        "options": {},
        "assignments": {
          "assignments": [
            {
              "id": "c5abdf27-cb9e-4969-b669-25c6fbd135f1",
              "name": "score",
              "type": "number",
              "value": "={{ $json.similarityScore * Math.abs(!$('Answer Relevance').first().json.output.noncommittal) }}"
            }
          ]
        }
      },
      "typeVersion": 3.4
    }
  ],
  "pinData": {},
  "connections": {
    "1aea189f-3e7d-45d2-9802-d44b1dce4797": {
      "main": [
        [
          {
            "node": "d8ac1b2b-f4ee-4af1-92f5-d194832c5c76",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "d8ac1b2b-f4ee-4af1-92f5-d194832c5c76": {
      "main": [
        [
          {
            "node": "1d8d9c97-0c90-4878-95fa-0dc6d3ad82c6",
            "type": "main",
            "index": 0
          }
        ],
        [
          {
            "node": "d4ad7c5e-f1cf-463b-b9b3-a45ff5597199",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "63d8ff44-5c4b-4cf6-b5b2-1f22de0de495": {
      "main": [
        [
          {
            "node": "1aea189f-3e7d-45d2-9802-d44b1dce4797",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "4184f202-48f5-4e58-a0e8-0227a23553f3": {
      "main": [
        [
          {
            "node": "fe7b41d2-e382-4f59-8b0c-06361062581d",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "3bdd3254-7cdf-455a-ae63-9639069993bc": {
      "main": [
        [
          {
            "node": "9e3bcdee-5f24-45e6-a85f-6dcfacb58d5d",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "be975ef4-c50b-4bf9-b871-cc8d479661eb": {
      "main": [
        [
          {
            "node": "69c30814-9725-426d-9572-9f92bd81dea5",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "1d8d9c97-0c90-4878-95fa-0dc6d3ad82c6": {
      "main": [
        [
          {
            "node": "be975ef4-c50b-4bf9-b871-cc8d479661eb",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "04f714cd-99ce-45a9-bc43-351fb6ea14d4": {
      "ai_languageModel": [
        [
          {
            "node": "be975ef4-c50b-4bf9-b871-cc8d479661eb",
            "type": "ai_languageModel",
            "index": 0
          }
        ]
      ]
    },
    "49b274c7-1ff2-43ba-acb9-349a8665b513": {
      "ai_languageModel": [
        [
          {
            "node": "1aea189f-3e7d-45d2-9802-d44b1dce4797",
            "type": "ai_languageModel",
            "index": 0
          }
        ]
      ]
    },
    "69c30814-9725-426d-9572-9f92bd81dea5": {
      "main": [
        [
          {
            "node": "3bdd3254-7cdf-455a-ae63-9639069993bc",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "d821b0ec-93c9-46e1-ab03-280aeb9b87b1": {
      "ai_outputParser": [
        [
          {
            "node": "be975ef4-c50b-4bf9-b871-cc8d479661eb",
            "type": "ai_outputParser",
            "index": 0
          }
        ]
      ]
    },
    "83be23ff-6857-4265-b1d0-f4b682c66903": {
      "main": [
        [
          {
            "node": "4184f202-48f5-4e58-a0e8-0227a23553f3",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "9e3bcdee-5f24-45e6-a85f-6dcfacb58d5d": {
      "main": [
        [
          {
            "node": "83be23ff-6857-4265-b1d0-f4b682c66903",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "54596db1-4ba7-414c-88cf-250f528d6c21": {
      "main": [
        [
          {
            "node": "1aea189f-3e7d-45d2-9802-d44b1dce4797",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "77f0c130-085f-4fa5-b74e-66fd084949b7": {
      "main": [
        [
          {
            "node": "63d8ff44-5c4b-4cf6-b5b2-1f22de0de495",
            "type": "main",
            "index": 0
          }
        ]
      ]
    }
  }
}