평가 지표 예시：RAG 문서 관련성

• •Google Sheets API 인증 정보
• •OpenAI API Key

{
  "meta": {
    "instanceId": "bf40384a063e00f3b983f4f9bada22b57a8231a04c0fb48d363e26d7b0f2b7e7",
    "templateCredsSetupCompleted": true
  },
  "nodes": [
    {
      "id": "86d6a658-f056-4434-9720-dfd3ca514cb3",
      "name": "스티키 노트1",
      "type": "n8n-nodes-base.stickyNote",
      "position": [
        420,
        20
      ],
      "parameters": {
        "color": 7,
        "width": 300,
        "height": 180,
        "content": "Check whether the documents returned a relevant to the question"
      },
      "typeVersion": 1
    },
    {
      "id": "90aac144-164f-4961-89f8-4b8402177202",
      "name": "스티키 노트3",
      "type": "n8n-nodes-base.stickyNote",
      "position": [
        -1220,
        -40
      ],
      "parameters": {
        "width": 200,
        "height": 500,
        "content": "## How it works\nThis template shows how to calculate a workflow evaluation metric: **retrieved document relevance** (i.e. whether the information retrieved from the vector store is relevant to the question).\n\nThe workflow takes a question and checks whether the information retrieved to answer it is relevant.\n\nYou can find more information on workflow evaluation [here](https://docs.n8n.io/advanced-ai/evaluations/overview), and other metric examples [here](https://docs.n8n.io/advanced-ai/evaluations/metric-based-evaluations/#2-calculate-metrics)."
      },
      "typeVersion": 1
    },
    {
      "id": "f3970d8c-b205-4348-b375-d919d2f56f94",
      "name": "스티키 노트4",
      "type": "n8n-nodes-base.stickyNote",
      "position": [
        -960,
        280
      ],
      "parameters": {
        "color": 7,
        "width": 220,
        "height": 220,
        "content": "Read in [this test dataset](https://docs.google.com/spreadsheets/d/1uuPS5cHtSNZ6HNLOi75A2m8nVWZrdBZ_Ivf58osDAS8/edit?gid=630157309#gid=630157309) of questions"
      },
      "typeVersion": 1
    },
    {
      "id": "677b90c8-aa6f-4cc5-88df-d8e635b0bde2",
      "name": "데이터셋 행 가져올 때",
      "type": "n8n-nodes-base.evaluationTrigger",
      "position": [
        -900,
        340
      ],
      "parameters": {
        "sheetName": {
          "__rl": true,
          "mode": "url",
          "value": "https://docs.google.com/spreadsheets/d/1uuPS5cHtSNZ6HNLOi75A2m8nVWZrdBZ_Ivf58osDAS8/edit?gid=630157309#gid=630157309"
        },
        "documentId": {
          "__rl": true,
          "mode": "url",
          "value": "https://docs.google.com/spreadsheets/d/1uuPS5cHtSNZ6HNLOi75A2m8nVWZrdBZ_Ivf58osDAS8/edit?gid=630157309#gid=630157309"
        }
      },
      "credentials": {
        "googleSheetsOAuth2Api": {
          "id": "bpr2LoSELMlxpwnN",
          "name": "Google Sheets account David"
        }
      },
      "typeVersion": 4.6
    },
    {
      "id": "c381a3c3-12b6-44fe-a2ed-6adf6f997534",
      "name": "평가 중?",
      "type": "n8n-nodes-base.evaluation",
      "position": [
        -60,
        200
      ],
      "parameters": {
        "operation": "checkIfEvaluating"
      },
      "typeVersion": 4.6
    },
    {
      "id": "e1cf0122-c668-41d2-9316-a173d425d897",
      "name": "채팅 메시지 수신 시",
      "type": "@n8n/n8n-nodes-langchain.chatTrigger",
      "position": [
        -900,
        100
      ],
      "webhookId": "d53bc846-db11-446c-8436-9d25dc265663",
      "parameters": {
        "options": {}
      },
      "typeVersion": 1.1
    },
    {
      "id": "b521aa40-c8da-4485-b834-50a129c27d3c",
      "name": "채팅 형식 맞추기",
      "type": "n8n-nodes-base.set",
      "position": [
        -680,
        340
      ],
      "parameters": {
        "options": {},
        "assignments": {
          "assignments": [
            {
              "id": "93f89095-7918-45ad-aa74-a0bbcf0d5788",
              "name": "chatInput",
              "type": "string",
              "value": "={{ $json.question }}"
            }
          ]
        }
      },
      "typeVersion": 3.4
    },
    {
      "id": "d0170cbd-ba01-45db-a405-aec943bf1608",
      "name": "스티키 노트",
      "type": "n8n-nodes-base.stickyNote",
      "position": [
        -80,
        100
      ],
      "parameters": {
        "color": 7,
        "width": 150,
        "height": 260,
        "content": "Only calculate metrics if we're evaluating, to reduce costs"
      },
      "typeVersion": 1
    },
    {
      "id": "a21f98f0-25eb-44cc-a04a-33181d7e5288",
      "name": "채팅 응답 반환",
      "type": "n8n-nodes-base.noOp",
      "position": [
        220,
        340
      ],
      "parameters": {},
      "typeVersion": 1
    },
    {
      "id": "fd8628a1-9f31-4f42-88df-b87eda7216ee",
      "name": "지표 설정",
      "type": "n8n-nodes-base.evaluation",
      "position": [
        820,
        80
      ],
      "parameters": {
        "metrics": {
          "assignments": [
            {
              "id": "230589eb-34c8-4d10-9296-4a78d673077a",
              "name": "similarity",
              "type": "number",
              "value": "={{ $json.message.content.score }}"
            }
          ]
        },
        "operation": "setMetrics"
      },
      "typeVersion": 4.6
    },
    {
      "id": "eefe3a5a-722a-4cce-b84b-65c0520a225f",
      "name": "데이터셋 가져오기",
      "type": "n8n-nodes-base.googleSheets",
      "position": [
        -680,
        -620
      ],
      "parameters": {
        "options": {},
        "sheetName": {
          "__rl": true,
          "mode": "url",
          "value": "https://docs.google.com/spreadsheets/d/1uuPS5cHtSNZ6HNLOi75A2m8nVWZrdBZ_Ivf58osDAS8/edit?gid=630157309#gid=630157309"
        },
        "documentId": {
          "__rl": true,
          "mode": "url",
          "value": "https://docs.google.com/spreadsheets/d/1uuPS5cHtSNZ6HNLOi75A2m8nVWZrdBZ_Ivf58osDAS8/edit?gid=630157309#gid=630157309"
        }
      },
      "credentials": {
        "googleSheetsOAuth2Api": {
          "id": "bpr2LoSELMlxpwnN",
          "name": "Google Sheets account David"
        }
      },
      "typeVersion": 4.5
    },
    {
      "id": "ea83d68c-f499-4864-918c-5ccaccf555a5",
      "name": "중복 제거",
      "type": "n8n-nodes-base.removeDuplicates",
      "position": [
        -460,
        -620
      ],
      "parameters": {
        "compare": "selectedFields",
        "options": {},
        "fieldsToCompare": "document_id"
      },
      "typeVersion": 2
    },
    {
      "id": "6bf59dc8-fe59-49cf-971f-e22dac5d0500",
      "name": "심플 벡터 스토어",
      "type": "@n8n/n8n-nodes-langchain.vectorStoreInMemory",
      "position": [
        -240,
        -620
      ],
      "parameters": {
        "mode": "insert",
        "memoryKey": "rag_evaluation_demo",
        "clearStore": true
      },
      "typeVersion": 1.1
    },
    {
      "id": "66dd59b0-b2e4-43c9-b1f5-7f5a65568f5e",
      "name": "임베딩 OpenAI",
      "type": "@n8n/n8n-nodes-langchain.embeddingsOpenAi",
      "position": [
        -300,
        -400
      ],
      "parameters": {
        "options": {}
      },
      "credentials": {
        "openAiApi": {
          "id": "Ag9qPAsY7lpIGkvC",
          "name": "JPs n8n openAI key"
        }
      },
      "typeVersion": 1.2
    },
    {
      "id": "213b230d-0f40-4e71-b0bf-4783274e2cdb",
      "name": "기본 데이터 로더",
      "type": "@n8n/n8n-nodes-langchain.documentDefaultDataLoader",
      "position": [
        -100,
        -380
      ],
      "parameters": {
        "options": {},
        "jsonData": "={{ $json.document_text }}",
        "jsonMode": "expressionData"
      },
      "typeVersion": 1
    },
    {
      "id": "64ec7e8c-63f9-4198-9e8d-61345c564c13",
      "name": "재귀 문자 텍스트 분할기",
      "type": "@n8n/n8n-nodes-langchain.textSplitterRecursiveCharacterTextSplitter",
      "position": [
        -20,
        -220
      ],
      "parameters": {
        "options": {}
      },
      "typeVersion": 1
    },
    {
      "id": "c561f2ce-ef05-4f42-9b6e-6fabd12d6d23",
      "name": "스티키 노트2",
      "type": "n8n-nodes-base.stickyNote",
      "position": [
        -980,
        -700
      ],
      "parameters": {
        "color": 7,
        "width": 1180,
        "height": 620,
        "content": "### Setup: Populate vector DB\nRun this once before running the main workflow. It inserts the documents from the dataset in the vector store, so they can be queried by the agent below"
      },
      "typeVersion": 1
    },
    {
      "id": "4a7de6bd-17ac-40df-9b27-2d8f6fdbeb07",
      "name": "'워크플로 실행' 클릭 시",
      "type": "n8n-nodes-base.manualTrigger",
      "position": [
        -900,
        -620
      ],
      "parameters": {},
      "typeVersion": 1
    },
    {
      "id": "a0f6211f-6202-462c-8fa3-52e10ad89b5d",
      "name": "문서 관련성 지표 계산",
      "type": "@n8n/n8n-nodes-langchain.openAi",
      "position": [
        440,
        80
      ],
      "parameters": {
        "modelId": {
          "__rl": true,
          "mode": "list",
          "value": "gpt-4o-mini",
          "cachedResultName": "GPT-4O-MINI"
        },
        "options": {},
        "messages": {
          "values": [
            {
              "role": "system",
              "content": "=You are a teacher evaluating the relevance of student responses to questions.\n\nYou will be given a QUESTION and one or more FACTS provided by a student. Your task is to determine whether these FACTS are relevant to answering the QUESTION.\n\nGrading criteria:\n(1) Evaluate whether the provided FACTS contain information that is relevant to answering the QUESTION\n(2) A FACT is relevant if it contains keywords or concepts semantically related to the QUESTION\n(3) A FACT is relevant if it provides context, examples, or explanations that address any aspect of the QUESTION\n(4) A FACT may contain some unrelated information and still be considered relevant if it also contains information that satisfies criteria (2) or (3)\n\nScoring:\n- Score 1: At least one of the provided FACTS is relevant to the QUESTION\n- Score 0: All provided FACTS are completely unrelated to the QUESTION\n\nEvaluation process:\n1. Identify the key concepts and requirements in the QUESTION\n2. Examine each provided FACT for keywords or concepts related to these key elements\n3. Consider semantic relationships, not just exact keyword matches\n4. Determine if any FACT contributes useful information toward answering the QUESTION\n5. Provide your reasoning in a detailed, step-by-step manner first\n6. Then create a one-sentence summary of your reasoning\n7. Avoid stating your conclusion at the beginning of your extended reasoning\n\nOutput format:\n{\n  \"extended_reasoning\": \"<detailed step-by-step analysis of how you evaluated the relevance>\",\n  \"reasoning_summary\": \"<one sentence summary of your conclusion>\",\n  \"score\": <number: either 0 or 1>\n}\n\nExamples:\n\nQUESTION: \"What factors contributed to the fall of the Roman Empire?\"\nFACTS: \"The Roman Empire fell due to economic problems, military failures, and political corruption. Germanic tribes invaded from the north while the empire was weakened.\"\nEVALUATION: This would receive a score of 1 because the facts directly address causes of the Roman Empire's fall.\n\nQUESTION: \"What factors contributed to the fall of the Roman Empire?\"\nFACTS: \"The Pythagorean theorem states that a² + b² = c² in a right triangle. Jupiter is the largest planet in our solar system.\"\nEVALUATION: This would receive a score of 0 because neither fact relates to the Roman Empire or its fall."
            },
            {
              "content": "=QUESTION: {{ $('When fetching a dataset row').item.json.question }}\n\nFACTS: \n{{ $json.documents }}"
            }
          ]
        },
        "jsonOutput": true
      },
      "credentials": {
        "openAiApi": {
          "id": "Ag9qPAsY7lpIGkvC",
          "name": "JPs n8n openAI key"
        }
      },
      "typeVersion": 1.8
    },
    {
      "id": "b01d3427-4907-4715-8c15-669500012f89",
      "name": "AI 에이전트",
      "type": "@n8n/n8n-nodes-langchain.agent",
      "position": [
        -420,
        200
      ],
      "parameters": {
        "options": {
          "systemMessage": "You are a helpful assistant. Answer the user's questions using information from your vector knowledge base only.",
          "returnIntermediateSteps": true
        }
      },
      "typeVersion": 1.9
    },
    {
      "id": "97e2db38-0114-449f-bb41-0ab490fc7327",
      "name": "OpenAI 채팅 모델",
      "type": "@n8n/n8n-nodes-langchain.lmChatOpenAi",
      "position": [
        -480,
        440
      ],
      "parameters": {
        "model": {
          "__rl": true,
          "mode": "list",
          "value": "gpt-4o-mini"
        },
        "options": {}
      },
      "credentials": {
        "openAiApi": {
          "id": "Ag9qPAsY7lpIGkvC",
          "name": "JPs n8n openAI key"
        }
      },
      "typeVersion": 1.2
    },
    {
      "id": "5b3679ab-b0a6-4461-9276-89eddb16f699",
      "name": "문서 추출",
      "type": "n8n-nodes-base.set",
      "position": [
        220,
        80
      ],
      "parameters": {
        "options": {},
        "assignments": {
          "assignments": [
            {
              "id": "2d771c52-6637-4af0-9ff7-c87368041bfd",
              "name": "documents",
              "type": "string",
              "value": "={{ $json.intermediateSteps.filter(x => x.action.tool == 'vector_knowledge_base')[0].observation }}"
            }
          ]
        }
      },
      "typeVersion": 3.4
    },
    {
      "id": "0e4c4512-7b9f-4881-8ac7-4c7c24a4a446",
      "name": "심플 벡터 스토어1",
      "type": "@n8n/n8n-nodes-langchain.vectorStoreInMemory",
      "position": [
        -260,
        440
      ],
      "parameters": {
        "mode": "retrieve-as-tool",
        "toolName": "vector_knowledge_base",
        "memoryKey": "rag_evaluation_demo",
        "toolDescription": "Source of all knowledge to work with"
      },
      "typeVersion": 1.1
    },
    {
      "id": "edde6c75-dea5-46cc-937e-68e4a925559b",
      "name": "임베딩 OpenAI1",
      "type": "@n8n/n8n-nodes-langchain.embeddingsOpenAi",
      "position": [
        -220,
        620
      ],
      "parameters": {
        "options": {}
      },
      "credentials": {
        "openAiApi": {
          "id": "Ag9qPAsY7lpIGkvC",
          "name": "JPs n8n openAI key"
        }
      },
      "typeVersion": 1.2
    },
    {
      "id": "e23289e7-a769-4aae-82e6-b7f9e6d903f0",
      "name": "스티키 노트6",
      "type": "n8n-nodes-base.stickyNote",
      "position": [
        -980,
        -40
      ],
      "parameters": {
        "color": 7,
        "width": 2000,
        "height": 820,
        "content": "### Main workflow"
      },
      "typeVersion": 1
    },
    {
      "id": "c90baefc-22bd-4782-a8eb-236bfb1d2751",
      "name": "스티키 노트5",
      "type": "n8n-nodes-base.stickyNote",
      "position": [
        -440,
        120
      ],
      "parameters": {
        "color": 7,
        "width": 300,
        "height": 200,
        "content": "Make sure to enable 'Return intermediate steps' in the agent, to get the list of executed tools"
      },
      "typeVersion": 1
    }
  ],
  "pinData": {},
  "connections": {
    "b01d3427-4907-4715-8c15-669500012f89": {
      "main": [
        [
          {
            "node": "c381a3c3-12b6-44fe-a2ed-6adf6f997534",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "c381a3c3-12b6-44fe-a2ed-6adf6f997534": {
      "main": [
        [
          {
            "node": "5b3679ab-b0a6-4461-9276-89eddb16f699",
            "type": "main",
            "index": 0
          }
        ],
        [
          {
            "node": "a21f98f0-25eb-44cc-a04a-33181d7e5288",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "eefe3a5a-722a-4cce-b84b-65c0520a225f": {
      "main": [
        [
          {
            "node": "ea83d68c-f499-4864-918c-5ccaccf555a5",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "66dd59b0-b2e4-43c9-b1f5-7f5a65568f5e": {
      "ai_embedding": [
        [
          {
            "node": "6bf59dc8-fe59-49cf-971f-e22dac5d0500",
            "type": "ai_embedding",
            "index": 0
          }
        ]
      ]
    },
    "5b3679ab-b0a6-4461-9276-89eddb16f699": {
      "main": [
        [
          {
            "node": "a0f6211f-6202-462c-8fa3-52e10ad89b5d",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "b521aa40-c8da-4485-b834-50a129c27d3c": {
      "main": [
        [
          {
            "node": "b01d3427-4907-4715-8c15-669500012f89",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "97e2db38-0114-449f-bb41-0ab490fc7327": {
      "ai_languageModel": [
        [
          {
            "node": "b01d3427-4907-4715-8c15-669500012f89",
            "type": "ai_languageModel",
            "index": 0
          }
        ]
      ]
    },
    "ea83d68c-f499-4864-918c-5ccaccf555a5": {
      "main": [
        [
          {
            "node": "6bf59dc8-fe59-49cf-971f-e22dac5d0500",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "edde6c75-dea5-46cc-937e-68e4a925559b": {
      "ai_embedding": [
        [
          {
            "node": "0e4c4512-7b9f-4881-8ac7-4c7c24a4a446",
            "type": "ai_embedding",
            "index": 0
          }
        ]
      ]
    },
    "213b230d-0f40-4e71-b0bf-4783274e2cdb": {
      "ai_document": [
        [
          {
            "node": "6bf59dc8-fe59-49cf-971f-e22dac5d0500",
            "type": "ai_document",
            "index": 0
          }
        ]
      ]
    },
    "0e4c4512-7b9f-4881-8ac7-4c7c24a4a446": {
      "ai_tool": [
        [
          {
            "node": "b01d3427-4907-4715-8c15-669500012f89",
            "type": "ai_tool",
            "index": 0
          }
        ]
      ]
    },
    "e1cf0122-c668-41d2-9316-a173d425d897": {
      "main": [
        [
          {
            "node": "b01d3427-4907-4715-8c15-669500012f89",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "677b90c8-aa6f-4cc5-88df-d8e635b0bde2": {
      "main": [
        [
          {
            "node": "b521aa40-c8da-4485-b834-50a129c27d3c",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "a0f6211f-6202-462c-8fa3-52e10ad89b5d": {
      "main": [
        [
          {
            "node": "fd8628a1-9f31-4f42-88df-b87eda7216ee",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "64ec7e8c-63f9-4198-9e8d-61345c564c13": {
      "ai_textSplitter": [
        [
          {
            "node": "213b230d-0f40-4e71-b0bf-4783274e2cdb",
            "type": "ai_textSplitter",
            "index": 0
          }
        ]
      ]
    },
    "4a7de6bd-17ac-40df-9b27-2d8f6fdbeb07": {
      "main": [
        [
          {
            "node": "eefe3a5a-722a-4cce-b84b-65c0520a225f",
            "type": "main",
            "index": 0
          }
        ]
      ]
    }
  }
}