From 32bb0cc06ae0288ff8c4670139bd4c760a4c06bb Mon Sep 17 00:00:00 2001 From: oscarvik Date: Thu, 22 Apr 2021 15:25:52 +0200 Subject: [PATCH 01/12] modified model to handle fake news datasets. Included notebook to evaluate the models classification performance --- .gitignore | 1 + FakeNewsDetection.ipynb | 1648 +++++++++++++++++++++++++++++++++ run_fn_detection.py | 194 ++++ trainer.py | 4 +- utils/gsn_argparse.py | 1 + utils/label_encode_dataset.py | 8 +- 6 files changed, 1848 insertions(+), 8 deletions(-) create mode 100644 FakeNewsDetection.ipynb create mode 100644 run_fn_detection.py diff --git a/.gitignore b/.gitignore index d68c297..3cce0b1 100644 --- a/.gitignore +++ b/.gitignore @@ -141,6 +141,7 @@ data/* #Block embeddings embeddings/* +fn_embeddings/* # misc .DS_Store diff --git a/FakeNewsDetection.ipynb b/FakeNewsDetection.ipynb new file mode 100644 index 0000000..8f84352 --- /dev/null +++ b/FakeNewsDetection.ipynb @@ -0,0 +1,1648 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Detecting Fake News using Graph Neural Networksin Knowledge Graphs \n", + "This notebook will be used to valuate the classification prerformance of our model. Be sure to have trained a model using the specific fake news dataset to ensure all entities have been \"observed\" and assigned an embedding. " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# LIAR\n", + "## Step 1: Import triple extracted dataset" + ] + }, + { + "cell_type": "code", + "execution_count": 62, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/people/person/employment_history./business/employment_tenure/title\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
text_idheadrelationtaillabelfb_headfb_tail
09524.jsonScott Walkerpeople/person/employment_history./business/emp...governorFalse/m/04tzmt/m/05pd99q
\n", + "
" + ], + "text/plain": [ + " text_id head relation \\\n", + "0 9524.json Scott Walker people/person/employment_history./business/emp... \n", + "\n", + " tail label fb_head fb_tail \n", + "0 governor False /m/04tzmt /m/05pd99q " + ] + }, + "execution_count": 62, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import pandas as pd\n", + "#load test data\n", + "LIAR = pd.read_csv(\n", + " \"./data/LIAR/test.csv\",\n", + " sep=\";\",\n", + " header=0,\n", + " engine=\"python\",\n", + " usecols = ['text_id', 'head', 'relation', 'tail', 'fb_head', 'fb_tail', 'label']\n", + "\n", + " )\n", + "\n", + "\n", + "heads = LIAR[\"fb_head\"]\n", + "tails = LIAR[\"fb_tail\"]\n", + "relations = LIAR[\"relation\"]\n", + "\n", + "relations = [\"/\" + rel for rel in relations]\n", + "\n", + "print(relations[0])\n", + "\n", + "\n", + "LIAR.head(1)" + ] + }, + { + "cell_type": "code", + "execution_count": 63, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "no. unique entities (original): 14951\n", + "no. unique entities in dataset: 155\n", + "no. unseen entities in dataset: 108\n", + "no. unique entities (updated): 15059\n" + ] + } + ], + "source": [ + "from os import path\n", + "import numpy as np\n", + "import torch\n", + "#construct entity encoder with original and new entity ids\n", + "entity_id = pd.read_csv('data/FB15k/entities.txt', sep='\\t', header=None, names=['entity', 'id'], engine='python')\n", + "model_entities = entity_id['entity'].values\n", + "print('no. unique entities (original):', len(model_entities))\n", + "\n", + "# add unseen entities from fake news dataset\n", + "dataset_entities = np.concatenate([np.array(heads), np.array(tails)])\n", + "dataset_entities = np.unique(dataset_entities)\n", + "print('no. unique entities in dataset:', len(dataset_entities))\n", + "new_entities = np.array([ent for ent in dataset_entities if not ent in model_entities])\n", + "print('no. unseen entities in dataset:', len(new_entities))\n", + "\n", + "all_entities = np.concatenate((model_entities, new_entities), axis=0)\n", + "print('no. unique entities (updated):', len(all_entities))\n" + ] + }, + { + "cell_type": "code", + "execution_count": 64, + "metadata": {}, + "outputs": [], + "source": [ + "from sklearn.preprocessing import LabelEncoder\n", + "\n", + "# fit encoder\n", + "le_entity = LabelEncoder()\n", + "le_entity.classes_ = np.load(path.join('fn_embeddings','le_entity_classes_LIAR.npy'), allow_pickle=True)\n", + "#load relation encoder\n", + "le_relation = LabelEncoder()\n", + "le_relation.classes_ = np.load(path.join('fn_embeddings','le_relation_classes_LIAR.npy'), allow_pickle=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 65, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "le_relation test: [1033]\n", + "[1033]\n" + ] + } + ], + "source": [ + "test_rel = '/american_football/football_team/current_roster./sports/sports_team_roster/position'\n", + "test_rel2 = '/people/person/employment_history./business/employment_tenure/title'\n", + "print('le_relation test: ', le_relation.transform([test_rel2]))\n", + "\n", + "print(le_relation.transform([relations[3]]))" + ] + }, + { + "cell_type": "code", + "execution_count": 66, + "metadata": {}, + "outputs": [], + "source": [ + "#create torch_geometric Data object\n", + "#edge_index = torch.tensor([le_heads, le_tails], dtype=torch.long)\n", + "#edge_attributes = torch.tensor(le_relation.transform(relations), dtype=torch.long)\n", + "#dataset = Data(x=le_entity.transform(all_entities), edge_type=edge_attributes, edge_index=edge_index)\n" + ] + }, + { + "cell_type": "code", + "execution_count": 67, + "metadata": {}, + "outputs": [], + "source": [ + "le_heads = le_entity.transform(heads)\n", + "le_tails = le_entity.transform(tails)\n", + "edge_index = torch.tensor([le_heads, le_tails], dtype=torch.long)\n", + "\n", + "le_relations = le_relation.transform(relations)\n", + "edge_type = torch.tensor(le_relations, dtype=torch.long)" + ] + }, + { + "cell_type": "code", + "execution_count": 68, + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "no. true: 79\n", + "no. false: 122\n" + ] + }, + { + "data": { + "text/plain": [ + "tensor([0, 0, 1, 0, 0, 1, 1, 0, 1, 1, 1, 0, 0, 0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0,\n", + " 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0,\n", + " 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 1, 0, 0, 1, 0, 1, 0, 0, 0,\n", + " 0, 0, 1, 1, 1, 0, 0, 0, 1, 1, 1, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0,\n", + " 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 1,\n", + " 0, 0, 1, 0, 1, 0, 0, 0, 1, 0, 0, 1, 1, 0, 0, 0, 1, 1, 1, 0, 0, 1, 1, 0,\n", + " 0, 0, 0, 0, 0, 1, 1, 0, 1, 1, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 1, 1, 1, 0,\n", + " 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 1, 1,\n", + " 1, 0, 0, 0, 0, 0, 1, 0, 0])" + ] + }, + "execution_count": 68, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# create and 'encode' lables\n", + "raw_labels = np.array(LIAR[\"label\"])\n", + "false = 0\n", + "true = 0\n", + "for l in raw_labels:\n", + " if l == False:\n", + " false += 1\n", + " else:\n", + " true += 1\n", + "print('no. true:', true)\n", + "print('no. false:', false)\n", + "labels = torch.tensor([0 if l == False else 1 for l in raw_labels], dtype=torch.int64)\n", + "labels" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 2: Classify using Multi-Relational GraphStar model\n", + "Use model pretrained with the entities of this specific fake news dataset." + ] + }, + { + "cell_type": "code", + "execution_count": 69, + "metadata": {}, + "outputs": [], + "source": [ + "# use env37test kernel\n", + "import torch\n", + "model = torch.load(\"output/FB15k_237_CPU_60epochs.pkl\", map_location=torch.device('cpu'))\n", + "\n", + "#model" + ] + }, + { + "cell_type": "code", + "execution_count": 70, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "torch.Size([15167, 256])" + ] + }, + "execution_count": 70, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "z = model.z\n", + "z.size()" + ] + }, + { + "cell_type": "code", + "execution_count": 71, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "ap: 0.40816418350539074 , auc: 0.49927370823822376\n" + ] + } + ], + "source": [ + "\n", + "pred = model.lp_score(z, edge_index, edge_type)\n", + "lp_auc, lp_ap = model.lp_test(pred, labels)\n", + "\n", + "print('ap:', lp_ap, ', auc:', lp_auc)" + ] + }, + { + "cell_type": "code", + "execution_count": 72, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 72, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAUoAAAEGCAYAAAADs9wSAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAZ20lEQVR4nO3deZhddX3H8fdnJjOZkJWExcieEHZKhAgRHiNbEagVrYoKWkBaRCiLFAT7ULTYKqKIoAWbYmsUioqigEBQEcoiAiHsCZg0GAiEJQtkD5mZb/84Z8hlnLn3zM2999x75/N6nvPM2e4535tJvvkt5/c7igjMzKx/LXkHYGZW75wozcxKcKI0MyvBidLMrAQnSjOzEobkHUCltasjhml43mHYAEzae3XeIdgAPfLE+iURsWW5n3//IcNj6bKurPe6IyKOLPdeldB0iXKYhjO14+i8w7ABuP2OP+Qdgg1Q6/j5Czfl80uXdfHQHdtnvNe8LTblXpXQdInSzOpfAN105x1GZk6UZlZzQbAhslW964ETpZnlwiVKM7MigqCrgYZPO1GaWS66caI0M+tXAF1OlGZmxblEaWZWRAAb3EZpZta/IFz1NjMrKqCrcfKkE6WZ1V4yMqdxOFGaWQ5EF8o7iMycKM2s5pLOHCdKM7N+Jc9ROlGamRXV7RKlmVn/XKI0MyshEF0N9CYaJ0ozy4Wr3mZmRQTizWjNO4zMnCjNrOaSB85d9TYzK8qdOWZmRUSIrnCJ0sysqG6XKM3M+pd05jRO+mmcSM2sabgzx8wsg64Geo6ycVK6mTWNnpE5WZZSJP2XpFclPVWwb6yk30ial/7cPN0vSVdKmi/pCUn7ZonXidLMctEdLZmWDH4AHNlr3wXAnRExCbgz3QY4CpiULqcAV2e5gROlmdVcMilGZUqUEXEPsKzX7mOAGen6DOBDBft/GIk/AGMkjS91D7dRmlnNBWJD9iGMW0iaVbA9PSKml/jM1hGxOF1/Gdg6Xd8GeKHgvEXpvsUU4URpZjUXwUAeOF8SEVPKv1eEpE16lZmr3maWA9GdcSnTKz1V6vTnq+n+F4HtCs7bNt1XlBOlmdVckJQosyxluhk4IV0/AbipYP/fpr3fU4E3Cqro/XLV28xyUamJeyVdDxxM0pa5CPgScAnwU0knAwuBY9PTbwOOBuYDa4CTstzDidLMai5QxSbujYhP9nPosD7ODeD0gd7DidLMai55XW3jpJ/GidTMmog8H6WZWTEBWUfd1AUnSjPLhUuUZmZFRMglSjOzYpLOHL+F0cysCL8zx8ysqKQzx22UZmZFVWpkTi04UZpZzVVyZE4tOFGaWS78cjEzsyIiYEO3E6WZWb+SqrcTpZlZUR6ZYxU1fGQnZ1+ygB12WUsEXH7+BJ55dGTeYQ16l31+Ox787SjGbNHJ9LueBeCeW0bzo8vewQvzOrjytj+yyz5r3zp/wZwOrjx/O1avbKGlBb5z2x9p79ikNxQ0LD8elJJ0JvA5YHZEHN/H8YOBcyPiA9WKoVmcetFCZv3vGP7t9F0Y0tbN0I7uvEMy4IiPL+ODJy3hG2dt/9a+HXdbx0XX/Ikrz9/ubed2dcKlZ+zAeVcuZOKe61ixrJXWtsGZJBOuevc4DTg8IhZV8R5Nb7ORney1/0ouO28CAJ0bWujc0Dh/wZrZ3lNX8/IL7W/bt/2k9X2e+8j/jmSn3dcycc91AIwa21X1+OrdJrwPp+aqkiglfQ+YANwu6VqSd+p2AGuBkyLi2V7nvw+4It0MYFpErJR0HskU7kOBX0TEl6oRbz17x7breWPZEM65dAETdl/DvKeG872Ld2D92sYZJ2uwaEEHEvzTJyfwxtIhvO+Y1zn29FdLf7BJJb3ejfN3uCpFk4g4FXgJOAS4GnhvRLwLuAj4ah8fORc4PSImA+8F1ko6ApgE7A9MBvaTNK2v+0k6RdIsSbPepO//0RtV65Bg5z1Xc+t1W/MPf70369a0cOypL+Udlg1QVyc89dBwzv/uQi775Tx+P3M0j947Iu+wctPzwHmWpR7Uog43GrhB0lPA5cCefZxzP/CttF1zTER0Akeky6PAbGA3ksT5ZyJiekRMiYgp7QytxnfIzZLF7Sx5uZ1nH0/+Ud03cyw777Um56hsoLYcv4G9p65m9LguOjYL3n3oCuY/OSzvsHJV5dfVVlQtEuVXgLsiYi/gr0mq4G8TEZcAfwcMA+6XtBsg4GsRMTlddo6I79cg3rqyfEk7ry0eyjY7Jb2nkw9cwfPzBvc/sEa038Er+dPcDtatEV2d8MQDI9h+l+aq/QxET693o5Qoa/F40Gg2vmD8xL5OkDQxIp4EnpT0bpLS4x3AVyRdFxGrJG0DbIiIQdewc/WXd+AL3/4/2tq6Wfx8B5d/YULeIRnwtc/twBMPjOCNZUM4fr89+PQ/vszIzbu46sJteGPpEP750xOYuOdavnr9AkaO6eJvPvsaZxy9CxLsf+gKDjh8Rd5fIVfu9X67S4EZki4Ebu3nnLMlHQJ0A08Dt0fEekm7Aw9IAlgFfAoYdIlywdzhnHXMXnmHYb188eqFfe4/6Kg3+tx/2EeWc9hHllczpIYRITqdKCEidkxXlwC7FBy6MD1+N3B3un5GP9e4go294WbWROqlWp2FR+aYWc15ZI6ZWQZOlGZmRXjiXjOzDOrlGcksnCjNrOYioLOBJu5tnEjNrKlU8oFzSZ+X9LSkpyRdL6lD0k6SHpQ0X9JPJLWXvlLfnCjNrOYqOdY7HYxyJjAlHQHYCnwC+DpweUTsDCwHTi43XidKM8tFhDItGQ0BhkkaAmwGLAYOBX6WHp9BMotZWZwozSwXlZoUIyJeBL4JPE+SIN8AHgFeTyfYAVgEbFNurE6UZlZzEQNqo9yiZxrFdDml8FqSNgeOAXYC3gkMB46sZLzu9TazHIiu7L3eSyJiSpHjhwPPRcRrAJJuBA4CxkgakpYqt2Xj5DwD5hKlmeWigm2UzwNTJW2mZAadw4A5wF3AR9NzTgBuKjdWJ0ozq7lKzkcZEQ+SdNrMBp4kyWvTgfOBcyTNB8YBZc9n66q3mdVeJO2UFbtc8j6t3u/UWkDyKplN5kRpZrnwEEYzsyJiYJ05uXOiNLNcVLLqXW1OlGaWiwGMusmdE6WZ1VyEE6WZWUmeuNfMrAS3UZqZFRGIbvd6m5kV10AFSidKM8uBO3PMzDJooCKlE6WZ5aIpSpSSvkORnB8RZ1YlIjNregF0dzdBogRm1SwKMxtcAmiGEmVEzCjclrRZRKypfkhmNhg00nOUJR9kkvQeSXOAZ9LtfSRdVfXIzKy5RcalDmR54vPbwPuBpQAR8TgwrYoxmVnTy/YaiHrp8MnU6x0RLySvonhLV3XCMbNBo05Ki1lkSZQvSDoQCEltwFnA3OqGZWZNLSAaqNc7S9X7VOB0kpeHvwRMTrfNzDaBMi75K1mijIglwPE1iMXMBpMGqnpn6fWeIOkWSa9JelXSTZIm1CI4M2tiTdbr/T/AT4HxwDuBG4DrqxmUmTW5ngfOsyx1IEui3CwifhQRnelyLdBR7cDMrLlFZFvqQbGx3mPT1dslXQD8mOT/gY8Dt9UgNjNrZg3U612sM+cRksTY820+W3AsgC9WKygza36qk9JiFsXGeu9Uy0DMbBCpo46aLDKNzJG0F7AHBW2TEfHDagVlZs2ufjpqsiiZKCV9CTiYJFHeBhwF3Ac4UZpZ+RqoRJml1/ujwGHAyxFxErAPMLqqUZlZ8+vOuNSBLIlybUR0A52SRgGvAttVNywza2oVfo5S0hhJP5P0jKS56fSQYyX9RtK89Ofm5YabJVHOkjQG+E+SnvDZwAPl3tDMDJJe7yxLRlcAMyNiN5Ja71zgAuDOiJgE3JlulyXLWO/T0tXvSZoJjIqIJ8q9oZkZULE2SkmjSebIPREgIt4E3pR0DEn/CsAM4G7g/HLuUeyB832LHYuI2eXc0MxsgLaQVPgOr+kRMb1geyfgNeC/Je1DUvM9C9g6Ihan57wMbF1uAMVKlJcVORbAoeXetJoigu516/IOwwbgXQ9/Iu8QbMD+dZOvMIBq9ZKImFLk+BBgX+CMiHhQ0hX0qmZHREjlP+Je7IHzQ8q9qJlZUUElhzAuAhZFxIPp9s9IEuUrksZHxGJJ40k6osuSpTPHzKzyKjTNWkS8TPImhl3TXYcBc4CbgRPSfScAN5UbaqaROWZmlVbhsd5nANdJagcWACeRFAR/KulkYCFwbLkXd6I0s3xUMFFGxGNAX+2Yh1Xi+llmOJekT0m6KN3eXtL+lbi5mQ1iTTbD+VXAe4BPptsrgX+vWkRm1vSyPmxeL1OxZal6HxAR+0p6FCAilqftAGZm5WuSiXt7bJDUSloIlrQldTNU3cwaVb2UFrPIUvW+EvgFsJWkfyOZYu2rVY3KzJpfA7VRZhnrfZ2kR0h6jwR8KCLmVj0yM2teddT+mEWWiXu3B9YAtxTui4jnqxmYmTW5ZkqUwK1sfMlYB8kA9GeBPasYl5k1OTVQT0eWqvfehdvprEKn9XO6mVnTGfDInIiYLemAagRjZoNIM1W9JZ1TsNlCMp3RS1WLyMyaX7N15gAjC9Y7Sdosf16dcMxs0GiWRJk+aD4yIs6tUTxmNlg0Q6KUNCQiOiUdVMuAzKz5iebp9X6IpD3yMUk3AzcAq3sORsSNVY7NzJpVE7ZRdgBLSd6R0/M8ZQBOlGZWviZJlFulPd5PsTFB9migr2hmdamBskixRNkKjODtCbJHA31FM6tHzVL1XhwRF9csEjMbXJokUTbOrJpm1liieXq9K/JSHjOzPjVDiTIiltUyEDMbXJqljdLMrHqcKM3Miqij1zxk4URpZjUnXPU2MyvJidLMrBQnSjOzEpwozcyKaLDZg1ryDsDMBqnIuGQkqVXSo5J+lW7vJOlBSfMl/URSe7mhOlGaWS7UnW0ZgLOAuQXbXwcuj4idgeXAyeXG6kRpZrlQZFsyXUvaFvgr4Jp0WyRz6P4sPWUG8KFyY3WiNLPay1rtzl71/jbwBaCnDDoOeD0iOtPtRcA25YbrRGlm+cieKLeQNKtgOaXwMpI+ALwaEY9UK1T3eptZzQ1wZM6SiJhS5PhBwAclHU3y6ppRwBXAmJ6XJALbAi+WG69LlGaWC3VHpqWUiPhiRGwbETsCnwB+FxHHA3cBH01POwG4qdxYnSjNrPYq30bZl/OBcyTNJ2mz/H65F3LV28xyUY0HziPibuDudH0BsH8lrutEaWb5aKCROU6UZpaLRhrC6ERpZvlwojQzK6KJ3sJoZlYVnuHczCyLaJxM6URpZrlwidIqpm1oN5fdOJ+29qB1SHDvrWP40TffkXdY1kvrovWM+uZLG7df3sDq47Zg7QfHMuxXyxh22+vQAuunjGD1iVvlF2i98FsY/5ykMcBxEXFVLe7XTDasF1/42ETWrWmldUjwrV/O5+HfjeSZ2cPzDs0KdG07lOXf3indCMZ9Zj7rp46k7YnVDH1wFcuu2BHaWtDrnUWvM5g0UmdOrYYwjgFO671Tkku0JYl1a1oBGNIWtLZFIzXtDErtT6yh6x3tdG/VxrCZr7P6I+OgLfmnFmP8V75HFSburZpa/dYuASZKegzYAKwjmXF4N0lHAL+KiL0AJJ0LjIiIL0uaCPw7sCWwBvj7iHimRjHXjZaW4Lt3/JF37vgmt/xgHM8+6tJkPRt67wrWTxsFQOtLb9I+Zw0jrn2NaBerTtqKzknDco6wDgQN1ZlTqxLlBcD/RcRk4DxgX+CsiNilxOemA2dExH7AuUCfVXdJp/TMVbeB9RUMuz50d4vT/nJXjt9vD3advIYddl2bd0jWnw3B0IdWse6gkQCoK9CqLpZ/YwdWnbgVoy99qaESRDVVcobzasurHvBQRDxX7ARJI4ADgRuSWd0BGNrXuRExnSSpMkpj6+SPtvJWr2jl8d+P4N2HrGThsy6V1KP22avonDj0rSp217g21k8dCRKduwyDFtCKLmK0q+CN1JmT1zRrqwvWO3vF0ZH+bCGZyn1ywbJ7zSKsE6PHdjJ8VBcA7R3d7DttFS/M7yjxKctLxz0rWPfeUW9trz9gBO1PrgGg9cU3YUMQo1rzCq9u9Dxw7hLl260ERvZz7BVgK0njgFXAB4CZEbFC0nOSPhYRN6QvC/qLiHi8RjHXhbFbb+DcK56npQVaWuCeW0bz4G9Hlf6g1d66btofX83K0zY+vrXu8DGM+s5ixp6xgBgiVpw9HjbWkAavyDYpb72oSaKMiKWS7pf0FLCWJDn2HNsg6WLgIZKp2gs7a44HrpZ0IdAG/BgYVInyubnDOP2IXfMOw7LoaGHJtb2a3dvEinPemU889a5x8mTt2igj4rgix64Eruxj/3PAkdWMy8zyUS/V6izcomxmtReAq95mZiU0Tp50ojSzfLjqbWZWgnu9zcyK8exBZmbFJQ+cN06mdKI0s3zUycxAWThRmlkuXKI0MyvGbZRmZqV4rLeZWWmuepuZFRH185qHLJwozSwfDVSizGviXjMb7CLjUoKk7STdJWmOpKclnZXuHyvpN5LmpT83LzdUJ0ozy4W6uzMtGXQC/xgRewBTgdMl7UHyrq47I2IScGe6XRYnSjOrvSB54DzLUupSEYsjYna6vhKYC2wDHAPMSE+bAXyo3HDdRmlmNSdiIA+cbyFpVsH29PSFgn9+XWlH4F3Ag8DWEbE4PfQysHWZ4TpRmllOsifKJRExpdRJ6Ztbfw6cnb5zq+BWEVL5E7u56m1m+YjItmQgqY0kSV4XETemu1+RND49Ph54tdxQnSjNrPYq2EaZvqH1+8DciPhWwaGbgRPS9ROAm8oN11VvM8tFxh7tLA4CPg08KemxdN8/AZcAP5V0MrAQOLbcGzhRmlkOslerS14p4j6SKS77clgl7uFEaWa1FzTUyBwnSjPLh8d6m5kV54l7zcxKcaI0MysiAroap+7tRGlm+XCJ0sysBCdKM7MiAvA7c8zMigkIt1GamfUvcGeOmVlJbqM0MyvBidLMrJjKTYpRC06UZlZ7AVRumrWqc6I0s3y4RGlmVoyHMJqZFRcQfo7SzKwEj8wxMyvBbZRmZkVEuNfbzKwklyjNzIoJoqsr7yAyc6I0s9rzNGtmZhn48SAzs/4FEC5RmpkVEZ6418yspEbqzFE0UBd9FpJeAxbmHUeVbAEsyTsIy6yZf187RMSW5X5Y0kySP58slkTEkeXeqxKaLlE2M0mzImJK3nFYNv59NY+WvAMwM6t3TpRmZiU4UTaW6XkHYAPi31eTcBulmVkJLlGamZXgRGlmVoITZc4knSlprqTr+jl+sKRf1Tou65+kMZJOyzsOqx0nyvydBvxlRByfdyCW2RiS39vbSPJItyblRJkjSd8DJgC3Szpf0gOSHpX0e0m79nH++yQ9li6PShqZ7j9P0sOSnpD0L7X+HoPQJcDE9PfwsKR7Jd0MzJG0o6Snek6UdK6kL6frEyXNlPRI+pndcorfBsj/A+YoIk6VdCRwCPAmcFlEdEo6HPgq8JFeHzkXOD0i7pc0Algn6QhgErA/IOBmSdMi4p7afZNB5wJgr4iYLOlg4NZ0+zlJOxb53HTg1IiYJ+kA4Crg0GoHa5vOibJ+jAZmSJpEMgtVWx/n3A98K23PvDEiFqWJ8gjg0fScESSJ04mydh6KiOeKnZD+x3YgcIOknt1Dqx2YVYYTZf34CnBXRHw4LZXc3fuEiLhE0q3A0cD9kt5PUor8WkT8Ry2DtbdZXbDeydubtDrSny3A6xExuVZBWeW4jbJ+jAZeTNdP7OsESRMj4smI+DrwMLAbcAfwmbTEgqRtJG1Vg3gHs5XAyH6OvQJsJWmcpKHABwAiYgXwnKSPASixT02itU3mEmX9uJSk6n0hSZtXX86WdAjQDTwN3B4R6yXtDjyQVulWAZ8CXq1BzINSRCyVdH/aabOWJDn2HNsg6WLgIZL/+J4p+OjxwNXp77gN+DHweO0it3J5CKOZWQmuepuZleBEaWZWghOlmVkJTpRmZiU4UZqZleBEOQhJ6krHKT8l6QZJm23CtX4g6aPp+jWS9ihy7sGSDizjHn+S9Gdv7Otvf69zVg3wXl+WdO5AY7Tm5kQ5OK2NiMkRsRfJGPNTCw+WOwtORPxdRMwpcsrBJMP4zBqKE6XdC+yclvYKZ8FplfSNglmJPgtvjSj5rqRnJf0WeGsUkKS7JU1J14+UNFvS45LuTIdlngp8Pi3NvlfSlpJ+nt7jYUkHpZ8dJ+nXkp6WdA3JMM2iJP0ynZXnaUmn9Dp2ebr/Tklbpvs8k49l5pE5g1hacjwKmJnu2peNs+CcArwREe9Oh+LdL+nXwLuAXYE9gK2BOcB/9brulsB/AtPSa42NiGVKppVbFRHfTM/7H+DyiLhP0vYkwzF3B74E3BcRF0v6K+DkDF/nM+k9hgEPS/p5RCwFhgOzIuLzki5Kr/0PeCYfGwAnysFpmKTH0vV7ge+TVIkLZ8E5AviLnvZHkrHok4BpwPUR0QW8JOl3fVx/KnBPz7UiYlk/cRwO7FEwm86odMz6NOBv0s/eKml5hu90pqQPp+vbpbEuJRnu+ZN0/7XAjZ7JxwbKiXJwWtt7Fps0YRTOgiPgjIi4o9d5R1cwjhZgakSs6yOWzNI5IQ8H3hMRayTdzcZZe3oLPJOPDZDbKK0/dwCfk9QGIGkXScNJ5rn8eNqGOZ5k0uHe/gBMk7RT+tmx6f7es+78GjijZ0PS5HT1HuC4dN9RwOYlYh0NLE+T5G4kJdoeLUBPqfg4kiq9Z/KxAXGitP5cQ9L+ODudJec/SGogvwDmpcd+CDzQ+4MR8RpwCkk193E2Vn1vAT7c05kDnAlMSTuL5rCx9/1fSBLt0yRV8OdLxDoTGCJpLslrGv5QcGw1sH/6HQ4FLk73Hw+cnMb3NHBMhj8TG6Q8e5CZWQkuUZqZleBEaWZWghOlmVkJTpRmZiU4UZqZleBEaWZWghOlmVkJ/w8NHwhk+Mt1pgAAAABJRU5ErkJggg==\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "import matplotlib.pyplot as plt\n", + "from sklearn.metrics import precision_score, confusion_matrix, ConfusionMatrixDisplay\n", + "\n", + "pred_classes = torch.tensor([0 if p < 0 else 1 for p in pred], dtype=torch.int64)\n", + "\n", + "cm = confusion_matrix(labels, pred_classes)\n", + "\n", + "disp = ConfusionMatrixDisplay(confusion_matrix=cm, display_labels=['false', 'true']) #usikker på rekkefølgen her\n", + "disp.plot()" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
text_idheadrelationtaillabelfb_headfb_tailpredicted_label
502615.jsonLee Fisherpeople/person/employment_history./business/emp...Lieutenant governorFalse/m/034q8v/m/0fkzqTrue
51592.jsonUnited States Senateorganization/organization/headquarters./locati...United StatesFalse/m/07t58/m/09c7w0False
5212745.jsonBarack Obamapeople/person/employment_history./business/emp...President of the United StatesTrue/m/02mjmr/m/060d2True
536386.jsonJon Runyanpeople/person/employment_history./business/emp...Member of CongressFalse/m/04q2pt/m/01gkgkTrue
548676.jsonRick Scottpeople/person/employment_history./business/emp...governorFalse/m/0btx2g/m/05pd99qTrue
558853.jsonBarack Obamapeople/person/employment_history./business/emp...President of the United StatesFalse/m/02mjmr/m/060d2True
5610208.jsonBarack Obamapeople/person/employment_history./business/emp...President of the United StatesFalse/m/02mjmr/m/060d2True
5711409.jsonCharlie Dentpeople/person/employment_history./business/emp...RepFalse/m/04fcm3/m/07r8g3True
5812994.jsonBill Clintonpeople/person/employment_history./business/emp...SecretaryTrue/m/0157m/m/01fcnxTrue
599891.jsonLorne Michaelspeople/person/employment_history./business/emp...Executive producerTrue/m/0p_2rExecutive producerTrue
\n", + "
" + ], + "text/plain": [ + " text_id head \\\n", + "50 2615.json Lee Fisher \n", + "51 592.json United States Senate \n", + "52 12745.json Barack Obama \n", + "53 6386.json Jon Runyan \n", + "54 8676.json Rick Scott \n", + "55 8853.json Barack Obama \n", + "56 10208.json Barack Obama \n", + "57 11409.json Charlie Dent \n", + "58 12994.json Bill Clinton \n", + "59 9891.json Lorne Michaels \n", + "\n", + " relation \\\n", + "50 people/person/employment_history./business/emp... \n", + "51 organization/organization/headquarters./locati... \n", + "52 people/person/employment_history./business/emp... \n", + "53 people/person/employment_history./business/emp... \n", + "54 people/person/employment_history./business/emp... \n", + "55 people/person/employment_history./business/emp... \n", + "56 people/person/employment_history./business/emp... \n", + "57 people/person/employment_history./business/emp... \n", + "58 people/person/employment_history./business/emp... \n", + "59 people/person/employment_history./business/emp... \n", + "\n", + " tail label fb_head fb_tail \\\n", + "50 Lieutenant governor False /m/034q8v /m/0fkzq \n", + "51 United States False /m/07t58 /m/09c7w0 \n", + "52 President of the United States True /m/02mjmr /m/060d2 \n", + "53 Member of Congress False /m/04q2pt /m/01gkgk \n", + "54 governor False /m/0btx2g /m/05pd99q \n", + "55 President of the United States False /m/02mjmr /m/060d2 \n", + "56 President of the United States False /m/02mjmr /m/060d2 \n", + "57 Rep False /m/04fcm3 /m/07r8g3 \n", + "58 Secretary True /m/0157m /m/01fcnx \n", + "59 Executive producer True /m/0p_2r Executive producer \n", + "\n", + " predicted_label \n", + "50 True \n", + "51 False \n", + "52 True \n", + "53 True \n", + "54 True \n", + "55 True \n", + "56 True \n", + "57 True \n", + "58 True \n", + "59 True " + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "predicted_raw_labels = [False if p == 0 else True for p in pred_classes]\n", + "predicted_raw_labels\n", + "LIAR['predicted_label'] = predicted_raw_labels\n", + "\n", + "LIAR[50:60]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 3: Dataset exploration\n", + "We suspect some issues with classification performance stem from the quality of the LIAR RDF dataset. To futher understand the limitations of our model, we will try to explore what properties of the dataset make classification hard. \n", + "\n", + "### Conflicting labels\n", + "Identical triples may have been extracted from both true and false documents, meaning they will exist in our dataset with conflicting labels:" + ] + }, + { + "cell_type": "code", + "execution_count": 73, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "no. triples: 201\n" + ] + }, + { + "data": { + "text/plain": [ + "array([['/m/04tzmt',\n", + " '/people/person/employment_history./business/employment_tenure/title',\n", + " '/m/05pd99q']], dtype=object)" + ] + }, + "execution_count": 73, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# create array of triples [[h,r,t], [h,r,t], [h,r,t], ...]\n", + "triples = np.vstack((heads, relations, tails)).T\n", + "print('no. triples: ', len(triples))\n", + "triples[:1]" + ] + }, + { + "cell_type": "code", + "execution_count": 74, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "no. unique triples: 134\n" + ] + } + ], + "source": [ + "print('no. unique triples:', len(set([triple[0] + triple[1] + triple[2] for triple in triples])))" + ] + }, + { + "cell_type": "code", + "execution_count": 75, + "metadata": {}, + "outputs": [], + "source": [ + "triple_dict = {}\n", + "conflicting_triples = []\n", + "for i in range(len(triples)):\n", + " triple = tuple(triples[i])\n", + " if triple in triple_dict: \n", + " if triple_dict[triple] != labels[i]:\n", + " conflicting_triples.append(triple)\n", + " else:\n", + " triple_dict[triple] = labels[i]\n", + "conflicting_triples = set(conflicting_triples)" + ] + }, + { + "cell_type": "code", + "execution_count": 76, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/m/0f8t6k /people/person/employment_history./business/employment_tenure/title /m/05pd99q\n", + "/m/0271_s /people/person/employment_history./business/employment_tenure/title /m/05pd99q\n", + "/m/02mjmr /people/person/employment_history./business/employment_tenure/title /m/048zv9l\n", + "/m/0157m /people/person/employment_history./business/employment_tenure/title /m/060d2\n", + "/m/04tzmt /people/person/employment_history./business/employment_tenure/title /m/05pd99q\n", + "/m/02mjmr /people/person/employment_history./business/employment_tenure/title /m/060d2\n", + "/m/01fwm3 /organization/role/leaders./organization/leadership/person /m/07t58\n", + "\n", + "no. uniqe triples associated with both labels: 7\n" + ] + } + ], + "source": [ + "for triple in conflicting_triples:\n", + " print(triple[0], triple[1], triple[2])\n", + "\n", + "print('\\nno. uniqe triples associated with both labels:', len(conflicting_triples))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### option 1: removing triples \n", + "remove all triples included in label conflict" + ] + }, + { + "cell_type": "code", + "execution_count": 77, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "no. triples in cleaned dataset: 143\n" + ] + } + ], + "source": [ + "cleaned_triples = []\n", + "cleaned_labels = []\n", + "for i in range(len(triples)):\n", + " triple = triples[i]\n", + " triple = tuple(triple)\n", + " if triple not in conflicting_triples:\n", + " cleaned_triples.append(triple)\n", + " cleaned_labels.append(labels[i])\n", + "print('no. triples in cleaned dataset:', len(cleaned_triples))\n", + "\n", + "cleaned_triples = np.array(cleaned_triples)\n", + "cleaned_labels = torch.tensor(cleaned_labels, dtype=torch.int64)" + ] + }, + { + "cell_type": "code", + "execution_count": 78, + "metadata": {}, + "outputs": [], + "source": [ + "t = np.vsplit(cleaned_triples.T, 3)\n", + "\n", + "cleaned_heads = t[0].flatten()\n", + "cleaned_relations = t[1].flatten()\n", + "cleaned_tails = t[2].flatten()\n", + "\n", + "\n", + "cleaned_le_heads = le_entity.transform(cleaned_heads)\n", + "cleaned_le_tails = le_entity.transform(cleaned_tails)\n", + "cleaned_edge_index = torch.tensor([cleaned_le_heads, cleaned_le_tails], dtype=torch.long)\n", + "\n", + "cleaned_le_relations = le_relation.transform(cleaned_relations)\n", + "cleaned_edge_type = torch.tensor(cleaned_le_relations, dtype=torch.long)" + ] + }, + { + "cell_type": "code", + "execution_count": 80, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "ap: 0.40124318478274923 , auc: 0.47045454545454546\n" + ] + } + ], + "source": [ + "cleaned_pred = model.lp_score(z, cleaned_edge_index, cleaned_edge_type)\n", + "lp_auc, lp_ap = model.lp_test(cleaned_pred, cleaned_labels)\n", + "\n", + "print('ap:', lp_ap, ', auc:', lp_auc)" + ] + }, + { + "cell_type": "code", + "execution_count": 81, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor([ 15.7264, 20.5741, 13.8190, 14.3914, 7.0685, 15.9150, 6.6974,\n", + " 15.0100, 13.5818, 13.0623, -10.5703, 15.8250, 15.0079, 15.0079,\n", + " 21.1873, 15.0079, 13.4696, 6.5500, 15.8451, 6.5895, 15.8620,\n", + " 9.0786, 15.7282, 13.8719, 16.3676, 16.3170, 16.1365, 0.1115,\n", + " 14.6679, 20.6207, 15.8791, 15.8339, -2.6281, 13.5036, 20.5372,\n", + " 15.7612, 15.7553, 11.8745, 27.8678, 14.4203, 15.8517, 15.8877,\n", + " 14.9755, 4.3363, 17.0142, 15.8395, 15.7869, 7.7396, 15.8315,\n", + " 11.5955, 15.8912, 15.8711, 26.0412, 7.2123, 0.4710, 14.4164,\n", + " 15.8731, 15.8581, 16.4693, 6.4843, 6.3987, 15.8082, 15.7058,\n", + " 6.4677, 7.9754, 7.9754, -1.7726, 6.0192, -1.7726, -1.7726,\n", + " 14.2982, 7.8923, 7.8923, 15.7562, 15.7310, 14.8623, 13.4865,\n", + " 15.6548, 15.6548, 3.4196, 28.1789, 15.8985, 20.5741, 6.7694,\n", + " 14.5086, 15.8689, 15.7915, 6.3969, 27.6602, 15.8466, 15.8466,\n", + " 10.2443, 16.4087, 16.4087, -1.1832, 5.2127, 16.3929, 15.3436,\n", + " 15.8921, 16.5208, 15.8947, 15.7964, 15.6279, 20.5090, 15.8354,\n", + " 6.4793, 11.1237, 14.5073, -2.6281, 12.3574, 16.4336, 15.7818,\n", + " 15.0437, 16.5219, 15.8464, 15.7287, 16.6142, 15.9446, 16.5642,\n", + " 16.3135, 16.3135, 15.7272, 10.5168, 14.3288, 4.3319, 15.6679,\n", + " 14.4164, 15.6818, 15.6818, 15.7869, 15.8637, 16.4949, 16.0811,\n", + " 13.8916, 16.0811, 16.5949, 15.8468, 15.8237, -1.3501, 15.8692,\n", + " -1.4290, 15.8763, 2.9085], grad_fn=)" + ] + }, + "execution_count": 81, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "cleaned_pred" + ] + }, + { + "cell_type": "code", + "execution_count": 82, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n", + " 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n", + " 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 0, 1, 1,\n", + " 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1,\n", + " 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n", + " 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 1, 1])" + ] + }, + "execution_count": 82, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "cleaned_pred_classes = torch.tensor([0 if p < 0 else 1 for p in cleaned_pred], dtype=torch.int64)\n", + "cleaned_pred_classes" + ] + }, + { + "cell_type": "code", + "execution_count": 83, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "0.3880597014925373" + ] + }, + "execution_count": 83, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "precision_score(cleaned_labels, cleaned_pred_classes)" + ] + }, + { + "cell_type": "code", + "execution_count": 84, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "array([[ 6, 82],\n", + " [ 3, 52]])" + ] + }, + "execution_count": 84, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "cleaned_cm = confusion_matrix(cleaned_labels, cleaned_pred_classes)\n", + "cleaned_cm" + ] + }, + { + "cell_type": "code", + "execution_count": 85, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 85, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAUQAAAEGCAYAAAAdeuyhAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAbyUlEQVR4nO3de5hdVX3/8fdnJvc7SSBGQLnFYJpKwIgoFbkLagVbpFWwqdIiBUVUFPDnIyq2olUp1qKm0BorICBgUBrQRngESoFAQLnFIOFqLiQkkDuZme/vj72GnExm5uyTnLP3mZnPi2c/2de1v8PAN2vttfdaigjMzAxayg7AzKxZOCGamSVOiGZmiROimVnihGhmlgwqO4B6G6JhMbxlVNlhWA2mTF9XdghWo/t/u3llROy6o9e/64iRserF9rz3ujUijtvRe9Wi3yXE4S2jOGTU+8oOw2ow79Y7yg7BatQ6+Ymnd+b6VS+2c++tr8t5r8UTd+Zeteh3CdHMml8AHXSUHcZ2/AzRzAoXBFuiPdeSh6RPSXpE0sOSrpY0TNLeku6R9ISkayQNqVaOE6KZlaIj5z/VSNodOBuYGRHTgVbgr4GvA5dExH7AauC0amU5IZpZ4YKgPfItOQ0ChksaBIwAlgJHAj9Nx+cAJ1YrxAnRzErRQeRagImSFlQsp1eWExHPA98EniFLhC8B9wNrIqItnfYcsHu1mNypYmaFC6Cd3LW/lRExs6eDknYBTgD2BtYA1wE79JqOE6KZlaIjf0Ks5mhgSUS8ACDpBuBQYJykQamWuAfwfLWC3GQ2s8IFsCUi15LDM8AhkkZIEnAU8ChwG3BSOmcWMLdaQU6IZla4IGjPuVQtK+Iess6TB4DfkeW12cB5wKclPQFMAK6oVpabzGZWvID2Oo5NHREXAhd22f0kcHAt5Tghmlnhsi9Vmo8TopmVQLSjsoPYjhOimRUu61RxQjQzS+8hOiGamQHQ4RqimZlriGZmrwpEexO+Bu2EaGalcJPZzIyshvhKtJYdxnacEM2scNmL2W4ym5kB7lQxMwMgQrSHa4hmZgB0uIZoZtbZqdJ86af5IjKzfs+dKmZmFdr9HqKZmb9UMTPbRkcT9jI3X0Rm1u9lgzu05FqqkTRV0oMVy8uSzpE0XtKvJC1Of+5SrSwnRDMrXCC2RGuupWpZEYsiYkZEzADeDGwAbgTOB+ZHxBRgftrulROimRUuAtqjJddSo6OAP0TE02ST189J++cAJ1a72M8QzawEquXF7ImSFlRsz46I2T2c+9fA1Wl9UkQsTevLgEnVbuSEaGaFC6il9rcyImZWO0nSEOB9wAXb3S8iJFWd+NQJ0cxK0YDXbo4HHoiI5Wl7uaTJEbFU0mRgRbUC/AzRzAoXiI7It9Tgg2xtLgPcBMxK67OAudUKcA3RzAqXTUNav/QjaSRwDPCxit0XA9dKOg14Gji5WjlOiGZWgvpOVB8R64EJXfatIut1zs0J0cwKFzTnlypOiGZWCo+YbWZGNmK2a4hmZnR2qnjWPTMzwHOqmJkBnZ0qfoZoZgY05EuVneaEaGaF6/xSpdk4IZpZKTzJlJkZ2XiIWzqcEM3MUpPZCdHMDPCXKraDRo5u45yvLub1b9hABFzy+Sk8/uCYssOyLm6YvSvzrhqPBHvvv4nPXPIM3/7Mnix+aAStg4OpMzbwyW88y6DBZUdavmZ97aZhdVZJZ0t6TNKVPRw/XNIvGnX//uSM//ckC+7YhdOPfzNnnXAgz/5hRNkhWRcrlw7mZ1dM5Lvzfs/s2xbR3gG3z92FI/9iNZff8Tg/+PUiXtnUwryrJlQvbEDImsx5liI1soZ4JnB0RDzXwHv0eyNGtTH9LS/xrfOnANC2pYW2Lc337MWgvU1s3tTCoMHtbN7YwoRJW3jz4WtfPT71wA2sXOrqYaca5lQpTEMSoqTvA/sA8yT9mGy2q2HARuAjEbGoy/nvBC5NmwEcFhFrJX2WbFDHocCNEXFhI+JtZq/ZYxMvvTiYT39tMfvsv57Fj4zi+/+4D5s3Nt93oAPZxMlbOOkfVvDht0xj6LDgoHe+vE0ybNsC83+6C2dc9HyJUTaPrJe5+f4bbkhVIyLOAP4IHAF8D3hHRBwIfBH4p24uORc4K82r+g5go6RjgSnAwcAM4M2SDuvufpJOl7RA0oJXYlO9f5xStQ4K9pu2jpuvnszH338gmza2cPLprnQ3m7VrWrn71rHMuedRrlr4MJs2tDL/+q3zov/rBXsy/ZD1/Olb15cYZfNo0BQCO62IttdY4DpJDwOXAH/SzTl3Ad+WdDYwLiLagGPTshB4ANifLEFuJyJmR8TMiJg5RMMa8TOUZuWyoaxcNpRFvx0NwJ23TGS/aetKjsq6WnjHKF6z5yuMm9DOoMFw6LvX8OiCkQD8+FuTeGnVID72JdcOK3WkqUirLUUqIiFeBNwWEdOBPydrOm8jIi4G/g4YDtwlaX9AwNciYkZa9ouIKwqIt6msXjmEF5YNZfe9NwAw421reMadKk1nt9238NgDI9i0QUTAg3eO5nX7bWLeleNZcPsYLrjsKVr86PdVnb3MzVZDLOK1m7FA51+Nf9vdCZL2jYjfAb+T9Bay2uCtwEWSroyIdZJ2B7ZERNWpBPub7120D5/75u8ZPLiDpc8O45IL3lB2SNbF/gdt4B3veYmz3jU1e8wxfSPHn7qKE/Z7E5P2eIVz/jz7nR367jWc+unlVUobGOrZgyxpHHA5MJ0s334UWARcA+wFPAWcHBGreyuniIT4DWCOpC8AN/dwzjmSjgA6gEeAeRGxWdIbgbslAawDTiXH3Kr9zZOPj+KTfzmj7DCsir/57DL+5rPLttk379mHSoqmuUWItvq+UnMpcEtEnJQmrB8BfB6YHxEXSzofOB84r7dCGpYQI2KvtLoSqKzSfCEdvx24Pa1/oocyLmVr77OZ9SP1ag5LGgscRmqBRsQrwCuSTgAOT6fNIcs35SREM7Oe1PilykRJCyq2Z0fE7IrtvYEXgP+UdABwP/BJYFJELE3nLAMmVbuRE6KZlaKGhLgyImb2cnwQcBDwiYi4R9KlZM3jV0VESIpqN3K/l5kVrs7vIT4HPBcR96Ttn5IlyOWSJgOkP6v2Pzghmlkp6vUeYkQsA56VNDXtOgp4FLgJmJX2zQLmVivLTWYzK1wEtNV3gNhPAFemHuYngY+QVfiulXQa8DTZZ8C9ckI0s1LU86XriHgQ6O4541G1lOOEaGaF8yRTZmYVwgnRzCwzYMZDNDPrTURzTiHghGhmJRDtnobUzCzjZ4hmZjTvrHtOiGZWvMieIzYbJ0QzK4V7mc3MyF7MdqeKmVniJrOZWeJeZjMzstqhE6KZWeLXbszMEj9DNDMjDf/lXmYzs0wTVhCdEM2sBO5UMTOrUMcqoqSngLVAO9AWETMljQeuAfYCngJOjojVvZXTfI14MxsQIpRrqcERETGjYg7n84H5ETEFmE+XuZq702MNUdK/0ksOj4iza4nUzKxTAB0dDW8ynwAcntbnALcD5/V2QW9N5gV1CcnMrKsA8tf+JkqqzEezI2J2NyX+UlIAP0jHJ0XE0nR8GTCp2o16TIgRMadyW9KIiNiQK3wzsypqeA9xZUUzuCd/FhHPS9oN+JWkx7e9V0RKlr2q+gxR0tskPQo8nrYPkHRZtevMzHoVOZc8RUU8n/5cAdwIHAwslzQZIP25olo5eTpV/gV4F7Aq3fAh4LB8YZqZdSdfh0qeThVJIyWN7lwHjgUeBm4CZqXTZgFzq5WV67WbiHhW2iaw9jzXmZn1qH6v3UwCbkw5ahBwVUTcIuk+4FpJpwFPAydXKyhPQnxW0tuBkDQY+CTw2A6HbmYWEHXqZY6IJ4EDutm/CjiqlrLyNJnPAM4Cdgf+CMxI22ZmO0E5l+JUrSFGxErglAJiMbOBpAk/Zs7Ty7yPpJ9LekHSCklzJe1TRHBm1o/VsZe5XvI0ma8CrgUmA68FrgOubmRQZtbPdb6YnWcpUJ6EOCIi/isi2tLyY2BYowMzs/4tIt9SpN6+ZR6fVudJOh/4CVle/yvgvwuIzcz6s8Z/y1yz3jpV7idLgJ1Rf6ziWAAXNCooM+v/qn9IV7zevmXeu8hAzGwAKaHDJI9cX6pImg5Mo+LZYUT8qFFBmVl/V3yHSR5VE6KkC8nGFJtG9uzweOBOwAnRzHZcE9YQ8/Qyn0T2+cuyiPgI2ScyYxsalZn1fx05lwLlaTJvjIgOSW2SxpANobNng+Mys/6stgFiC5MnIS6QNA74d7Ke53XA3Y0Mysz6vz7Vy9wpIs5Mq9+XdAswJiJ+29iwzKzf60sJUdJBvR2LiAcaE5KZWTl6qyF+q5djARxZ51jqIjo66Fi7tuwwrAZvvOvDZYdgNbtwp0voU03miDiiyEDMbAAJ+tyne2ZmjdOXaohmZo3UjE3mPC9mm5nVX50HiJXUKmmhpF+k7b0l3SPpCUnXSBpSrYw8I2ZL0qmSvpi2Xyfp4Pxhmpl1o/4jZnedAO/rwCURsR+wGjitWgF5aoiXAW8DPpi21wL/VlOYZmYVFPmXXOVJewDvAS5P2yJ7E+an6ZQ5wInVysnzDPGtEXGQpIUAEbE6T9XTzKxX+XuZJ0paULE9OyJmdznnX4DPAaPT9gRgTUS0pe3nyGYO7VWehLhFUiup8ippVwr/5NrM+psaOlVWRsTMHsuR3gusiIj7JR2+MzHlSYjfAW4EdpP0j2Sj33xhZ25qZlbH124OBd4n6d1kY7aOAS4FxkkalGqJewDPVysoz7fMV0q6n2wIMAEnRsRjVS4zM+tZDc8HqxYVcQFpSpNUQzw3Ik6RdB1ZBe4nwCxgbrWy8vQyvw7YAPwcuAlYn/aZme24xs/LfB7waUlPkD1TvKLaBXmazDezdbKpYcDewCLgT3Y8TjMb6NSAnoiIuB24Pa0/CdT0imCeJvOfVm6nUXDO7OF0M7M+q+ZP9yLiAUlvbUQwZjaANOGne3kmmfp0xWYLcBDwx4ZFZGb9Xx07VeopTw1xdMV6G9kzxesbE46ZDRh9LSGmF7JHR8S5BcVjZgNFX0qInS80Sjq0yIDMrP8Tjell3lm91RDvJXte+KCkm4DrgPWdByPihgbHZmb9VR9+hjgMWEU2ckTn+4gBOCGa2Y7rYwlxt9TD/DBbE2GnJvxRzKxPacIs0ltCbAVGsW0i7NSEP4qZ9SV9rcm8NCK+UlgkZjaw9LGE2HxzBJpZ/xB9r5f5qMKiMLOBpy/VECPixSIDMbOBpa89QzQzaxwnRDMz6jH4a0M4IZpZ4YSbzGZmr2rGhJhnonozs/qr05wqkoZJulfSQ5IekfTltH9vSfdIekLSNXnmk3dCNLNy1G+Sqc3AkRFxADADOE7SIcDXgUsiYj9gNXBatYKcEM2seGm0mzxL1aIy69Lm4LQE2YA0P0375wAnVivLCdHMypG/hjhR0oKK5fSuRUlqlfQgsAL4FfAHYE2apB7gOWD3aiG5U8XMSlHDp3srI2JmbydERDswQ9I44EZg/x2JyQnRzErRiF7miFgj6TbgbcC4zpH/gT2A56td7yazmRUvb3M5Xy/zrqlmiKThwDHAY8BtwEnptFnA3GpluYZoZuWoXw1xMjAnTYrXAlwbEb+Q9CjwE0lfBRYCV1QryAnRzApXzy9VIuK3wIHd7H8SOLiWspwQzawU6mi+T1WcEM2seB7cwcxsq2b8ltkJ0czK4YRoZpZxDdHMrJMTopkZfXLWPTOzhvCI2WZmlaL5MqITopmVwjVEq9ngoR1864YnGDwkaB0U3HHzOP7rm68pOyzrxmvPfJyOYS3QIqJVLP/6foz70VKG37+WGCTaJg1h1Vl7ECNbyw61fAP5xew0EsWHIuKyIu7Xn2zZLD73gX3ZtKGV1kHBt3/2BPf9ejSPPzCy7NCsGyu+tA8dY7b+b7XpgFGsOeU10CrG/XgpY29cwZpTJ5cYYfNoxk6Voob/Ggec2XWnJNdQqxKbNmQ1ikGDg9bB0YyPXqwHmw4YDa0CYPOUEbSu2lJyRM1DHfmWIhWVkC4G9k1DfG8BNpFN+rK/pGOBX0TEdABJ5wKjIuJLkvYF/g3YFdgA/H1EPF5QzE2jpSX47q2/57V7vcLPfziBRQtdO2xWu311CQBrj5nA+mPGb3Ns1G2rWf/2sWWE1XyCAd2pcj4wPSJmSDocuDltL5G0Vy/XzQbOiIjFkt4KXEY2ccw20hwLpwMMY0SdQy9fR4c485ipjBzTzoVXLOH1Uzfy9KLhZYdlXSy/aF/aJwym5aU2drtoCW27D2XztOwvrzHXryBaxIZ3jCs3yCbiTpWt7o2IJb2dIGkU8HbgOkmdu4d2d25EzCZLnozR+Cb811wf619u5aH/HcVbjljrhNiE2icMBqBj7CA2HjyGIU9sYPO0kYy8bTXD73+ZFRfuA1v/W7Ym/D+1rCkE1lest3WJY1j6s4Vs1qwZFcsbC4uwSYwd38bIMe0ADBnWwUGHrePZJ4ZVucqKpk0daGP7q+vDHlrHlj2HMWzhWsbMfYEXztuLGOoZOzp1vphdj2lI66moGuJaYHQPx5YDu0maAKwD3gvcEhEvS1oi6QMRcZ2yauKbIuKhgmJuCuMnbeHcS5+hpQVaWuA3Px/LPf8zpuywrIuWl9rY9Z+fzjbagw1/No5NB45m8scXobZgt4uyBtHmN4xg9elVZ8Ps/yIG7gCxEbFK0l2SHgY2kiXBzmNbJH0FuJdsVqzKTpNTgO9J+gLZ5NM/AQZUQlzy2HDOOnZq2WFYFe2ThrDsm1O227/0u/7d9aj58mFxzxAj4kO9HPsO8J1u9i8BjmtkXGZWjno1hyXtCfwImESWZmdHxKWSxgPXAHsBTwEnR8Tq3sryQw0zK14AHZFvqa4N+ExETAMOAc6SNI3s7Zb5ETEFmJ+2e+WEaGblqNO8zBGxNCIeSOtryeZk3h04AZiTTpsDnFitLH8pYmalqKHJPFHSgort2elVu+3LzN5rPhC4B5gUEUvToWVkTepeOSGaWSlq6GVeGREzq5aXvbt8PXBOekvl1WMREVL1FOwms5kVL29zOWfOlDSYLBleGRE3pN3LJU1OxycDK6qV44RoZoXLXsyOXEvVsrKq4BXAYxHx7YpDNwGz0vosYG61stxkNrNy1G8km0OBDwO/SwPIAHyebFCZayWdBjwNnFytICdEMytFntpfHhFxJ1mlsztH1VKWE6KZFW8gj5htZratAfwts5nZdgbwALFmZlt5onozswquIZqZJc2XD50Qzawc6mi+NrMTopkVL6jni9l144RoZoUT+T7LK5oTopmVwwnRzCxxQjQzw88QzcwquZfZzAyAcJPZzAxIo904IZqZZZqvxeyEaGbl8HuIZmadmjAhepIpMyteBLR35FuqkPQfklZIerhi33hJv5K0OP25S56wnBDNrBwR+Zbqfggc12Xf+cD8iJgCzE/bVTkhmlk56pQQI+I3wItddp8AzEnrc4AT84TkZ4hmVrwAGjunyqSIWJrWlwGT8lzkhGhmJQiI3O/dTJS0oGJ7dkTMzn2niJCUK/s6IZpZ8YJcHSbJyoiYWeMdlkuaHBFLJU0GVuS5yM8Qzawc9etU6c5NwKy0PguYm+ciJ0QzK0edEqKkq4G7gamSnpN0GnAxcIykxcDRabsqN5nNrAT1G9whIj7Yw6Gjai3LCdHMiheAh/8yM0ua8NM9J0QzK0HU0stcGCdEMyteQOR/D7EwTohmVo7GfqmyQ5wQzawcfoZoZkaWDN3LbGaWuIZoZgYQRHt72UFsxwnRzIrX+OG/dogTopmVw6/dmJmlaZldQzQzI41k4xqimRlAU3aqKJqw63tnSHoBeLrsOBpkIrCy7CAst/78+3p9ROy6oxdLuoXs308eKyOi66x6DdHvEmJ/JmnBDgylbiXx76vv8YjZZmaJE6KZWeKE2LfknnrRmoJ/X32MnyGamSWuIZqZJU6IZmaJE2LJJJ0t6TFJV/Zw/HBJvyg6LuuZpHGSziw7Dqs/J8TynQkcExGnlB2I5TaO7Pe2DUn+8quPc0IskaTvA/sA8ySdJ+luSQsl/a+kqd2c/05JD6ZloaTRaf9nJd0n6beSvlz0zzEAXQzsm34P90m6Q9JNwKOS9pL0cOeJks6V9KW0vq+kWyTdn67Zv6T4rQf+G61EEXGGpOOAI4BXgG9FRJuko4F/Av6yyyXnAmdFxF2SRgGbJB0LTAEOBgTcJOmwiPhNcT/JgHM+MD0iZkg6HLg5bS+RtFcv180GzoiIxZLeClwGHNnoYC0/J8TmMRaYI2kK2ehIg7s55y7g2+l54w0R8VxKiMcCC9M5o8gSpBNice6NiCW9nZD+Ans7cJ2kzt1DGx2Y1cYJsXlcBNwWEe9PtYzbu54QERdLuhl4N3CXpHeR1Qq/FhE/KDJY28b6ivU2tn0UNSz92QKsiYgZRQVltfMzxOYxFng+rf9tdydI2jcifhcRXwfuA/YHbgU+mmogSNpd0m4FxDuQrQVG93BsObCbpAmShgLvBYiIl4Elkj4AoMwBhURrubmG2Dy+QdZk/gLZM6nunCPpCKADeASYFxGbJb0RuDs1xdYBpwIrCoh5QIqIVZLuSp0nG8mSYOexLZK+AtxL9hfc4xWXngJ8L/2OBwM/AR4qLnKrxp/umZklbjKbmSVOiGZmiROimVnihGhmljghmpklTogDkKT29B3uw5KukzRiJ8r6oaST0vrlkqb1cu7hkt6+A/d4StJ2M7T1tL/LOetqvNeXJJ1ba4zWPzghDkwbI2JGREwn+4b6jMqDOzpqS0T8XUQ82ssph5N9vmbWlJwQ7Q5gv1R7qxy1pVXSP1eMovMxePULi+9KWiTpf4BXv4qRdLukmWn9OEkPSHpI0vz0OeIZwKdS7fQdknaVdH26x32SDk3XTpD0S0mPSLqc7PPEXkn6WRpF5hFJp3c5dknaP1/SrmmfR56x7fhLlQEs1QSPB25Juw5i66gtpwMvRcRb0idod0n6JXAgMBWYBkwCHgX+o0u5uwL/DhyWyhofES8qG+5sXUR8M513FXBJRNwp6XVknyG+EbgQuDMiviLpPcBpOX6cj6Z7DAfuk3R9RKwCRgILIuJTkr6Yyv44HnnGuuGEODANl/RgWr8DuIKsKVs5asuxwJs6nw+SfWs9BTgMuDoi2oE/Svp1N+UfAvyms6yIeLGHOI4GplWM/jImfZN9GPAX6dqbJa3O8TOdLen9aX3PFOsqss8cr0n7fwzc4JFnrCdOiAPTxq6jrqTEUDlqi4BPRMStXc57dx3jaAEOiYhN3cSSWxqT8GjgbRGxQdLtbB1lpqvAI89YD/wM0XpyK/APkgYDSHqDpJFk4yz+VXrGOJlscNuu/g84TNLe6drxaX/XUWJ+CXyic0PSjLT6G+BDad/xwC5VYh0LrE7JcH+yGmqnFqCzlvshsqa4R56xbjkhWk8uJ3s++EAa1eUHZC2KG4HF6diPgLu7XhgRLwCnkzVPH2Jrk/XnwPs7O1WAs4GZqdPmUbb2dn+ZLKE+QtZ0fqZKrLcAgyQ9Rja8//9VHFsPHJx+hiOBr6T9pwCnpfgeAU7I8e/E+jmPdmNmlriGaGaWOCGamSVOiGZmiROimVnihGhmljghmpklTohmZsn/B0CxSBRcHg0HAAAAAElFTkSuQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "disp = ConfusionMatrixDisplay(confusion_matrix=cleaned_cm, display_labels=['false', 'true']) #usikker på rekkefølgen her\n", + "disp.plot()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### option 2: Manually change triple label of conflicting " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Dataset similarities to fb15k\n", + "Many entities from the fake news datasets have not been seen in the original FB15k knowledge graph. How does this affetc the performance of the classification?" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "no. unique entities in fake news dataset: 155\n", + "no. new entities in fake news dataset: 108\n" + ] + } + ], + "source": [ + "#\n", + "print('no. unique entities in fake news dataset:', len(dataset_entities))\n", + "print('no. new entities in fake news dataset:', len(new_entities))\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We observe that only 30% of entities in the fake news dataset are in the original FB15k knowledge graph.." + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "no. triples with known entities: 61\n" + ] + } + ], + "source": [ + "known_triples = []\n", + "known_labels = []\n", + "for i in range(len(triples)):\n", + " triple = triples[i]\n", + " if triple[0] not in new_entities and triple[2] not in new_entities:\n", + " known_triples.append(tuple(triple))\n", + " known_labels.append(labels[i])\n", + "print('no. triples with known entities:', len(known_triples))\n", + "\n", + "known_triples = np.array(known_triples)\n", + "known_labels = torch.tensor(known_labels, dtype=torch.int64)" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "metadata": {}, + "outputs": [], + "source": [ + "kt = np.vsplit(known_triples.T, 3)\n", + "\n", + "known_heads = kt[0].flatten()\n", + "known_relations = kt[1].flatten()\n", + "known_tails = kt[2].flatten()\n", + "\n", + "\n", + "known_le_heads = le_entity.transform(known_heads)\n", + "known_le_tails = le_entity.transform(known_tails)\n", + "known_edge_index = torch.tensor([known_le_heads, known_le_tails], dtype=torch.long)\n", + "\n", + "known_le_relations = le_relation.transform(known_relations)\n", + "known_edge_type = torch.tensor(known_le_relations, dtype=torch.long)" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor([[ 137, 4792, 466, 4792, 4792, 4792, 4792, 10187, 4792, 4792,\n", + " 10782, 4792, 4792, 10190, 4792, 4792, 4792, 4792, 466, 4792,\n", + " 4792, 4792, 11775, 4792, 4792, 4792, 4792, 4792, 4792, 4792,\n", + " 6589, 4792, 4792, 4792, 4792, 4792, 4792, 4792, 11187, 4792,\n", + " 4792, 5627, 4792, 4792, 4792, 4792, 4792, 4792, 4792, 4792,\n", + " 4792, 4792, 4792, 4792, 10807, 7185, 10807, 11999, 4792, 4792,\n", + " 4792],\n", + " [10338, 9034, 9034, 9034, 9034, 9034, 819, 10807, 9034, 9034,\n", + " 9034, 9034, 9034, 10807, 9034, 9034, 9034, 9034, 9034, 4583,\n", + " 9034, 9034, 11945, 9034, 9034, 9034, 9034, 9034, 9034, 6589,\n", + " 6817, 6589, 6589, 9034, 9034, 9034, 9034, 9034, 10807, 9034,\n", + " 9034, 11226, 9034, 11944, 9034, 9034, 9034, 9034, 9034, 9034,\n", + " 9034, 9034, 9034, 9034, 7185, 9034, 7185, 9034, 9034, 9034,\n", + " 10807]])" + ] + }, + "execution_count": 28, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "known_edge_index" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "0.3673469387755102" + ] + }, + "execution_count": 29, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "known_pred = model.lp_score(z, known_edge_index, known_edge_type)\n", + "known_pred_classes = torch.tensor([0 if p < 0 else 1 for p in known_pred], dtype=torch.int64)\n", + "\n", + "precision_score(known_labels, known_pred_classes)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "How about performance on triples with only unseen entities:" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "no. triples with known entities: 76\n" + ] + } + ], + "source": [ + "unknown_triples = []\n", + "unknown_labels = []\n", + "for i in range(len(triples)):\n", + " triple = triples[i]\n", + " if triple[0] in new_entities and triple[2] in new_entities:\n", + " unknown_triples.append(tuple(triple))\n", + " unknown_labels.append(labels[i])\n", + "print('no. triples with known entities:', len(unknown_triples))\n", + "\n", + "unknown_triples = np.array(unknown_triples)\n", + "unknown_labels = torch.tensor(unknown_labels, dtype=torch.int64)" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "metadata": {}, + "outputs": [], + "source": [ + "ukt = np.vsplit(unknown_triples.T, 3)\n", + "\n", + "unknown_heads = ukt[0].flatten()\n", + "unknown_relations = ukt[1].flatten()\n", + "unknown_tails = ukt[2].flatten()\n", + "\n", + "\n", + "unknown_le_heads = le_entity.transform(unknown_heads)\n", + "unknown_le_tails = le_entity.transform(unknown_tails)\n", + "unknown_edge_index = torch.tensor([unknown_le_heads, unknown_le_tails], dtype=torch.long)\n", + "\n", + "unknown_le_relations = le_relation.transform(unknown_relations)\n", + "unknown_edge_type = torch.tensor(unknown_le_relations, dtype=torch.long)" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "0.42105263157894735" + ] + }, + "execution_count": 32, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "unknown_pred = model.lp_score(z, unknown_edge_index, unknown_edge_type)\n", + "unknown_pred_classes = torch.tensor([0 if p < 0 else 1 for p in unknown_pred], dtype=torch.int64)\n", + "\n", + "precision_score(unknown_labels, unknown_pred_classes)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Fake News Net\n", + "## Step 1: Import triple extracted dataset" + ] + }, + { + "cell_type": "code", + "execution_count": 237, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
headrelationtailfb_headfb_taillabel
text_id
415William H. Sewardorganization/role/leaders./organization/leader...Republican Party/m/0k_2z/m/085srzTrue
415John Tylerpeople/person/employment_history./business/emp...President of the United States/m/042dk/m/02mjmrTrue
415Donald Trumppeople/person/employment_history./business/emp...President of the United States/m/0cqt90/m/02mjmrTrue
415Hillary Clintonpeople/person/employment_history./business/emp...President of the United States/m/0d06m5/m/02mjmrTrue
416Donald Trumppeople/person/employment_history./business/emp...King/m/0cqt90/m/03w9bnrTrue
\n", + "
" + ], + "text/plain": [ + " head relation \\\n", + "text_id \n", + "415 William H. Seward organization/role/leaders./organization/leader... \n", + "415 John Tyler people/person/employment_history./business/emp... \n", + "415 Donald Trump people/person/employment_history./business/emp... \n", + "415 Hillary Clinton people/person/employment_history./business/emp... \n", + "416 Donald Trump people/person/employment_history./business/emp... \n", + "\n", + " tail fb_head fb_tail label \n", + "text_id \n", + "415 Republican Party /m/0k_2z /m/085srz True \n", + "415 President of the United States /m/042dk /m/02mjmr True \n", + "415 President of the United States /m/0cqt90 /m/02mjmr True \n", + "415 President of the United States /m/0d06m5 /m/02mjmr True \n", + "416 King /m/0cqt90 /m/03w9bnr True " + ] + }, + "execution_count": 237, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "#load test data\n", + "FNN = pd.read_csv(\n", + " \"./data/FakeNewsNet/train.csv\",\n", + " sep=\",\",\n", + " header=0,\n", + " engine=\"python\",\n", + " index_col=0\n", + " )\n", + "\n", + "FNN = FNN.set_index('text_id')\n", + "\n", + "FNN.tail(5)" + ] + }, + { + "cell_type": "code", + "execution_count": 238, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'/organization/role/leaders./organization/leadership/person'" + ] + }, + "execution_count": 238, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "heads = FNN[\"fb_head\"]\n", + "tails = FNN[\"fb_tail\"]\n", + "relations = FNN[\"relation\"]\n", + "\n", + "relations = [\"/\" + rel for rel in relations]\n", + "\n", + "relations[0]" + ] + }, + { + "cell_type": "code", + "execution_count": 239, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "no. unique entities (original): 14951\n", + "no. unique entities in dataset: 682\n", + "no. unseen entities in dataset: 542\n", + "no. unique entities (updated): 15493\n", + "True\n" + ] + } + ], + "source": [ + "#construct entity encoder with original and new entity ids\n", + "entity_id = pd.read_csv('data/FB15k/entities.txt', sep='\\t', header=None, names=['entity', 'id'], engine='python')\n", + "model_entities = entity_id['entity'].values\n", + "print('no. unique entities (original):', len(model_entities))\n", + "\n", + "# add unseen entities from fake news dataset\n", + "dataset_entities = np.concatenate([np.array(heads), np.array(tails)])\n", + "dataset_entities = np.unique(dataset_entities)\n", + "print('no. unique entities in dataset:', len(dataset_entities))\n", + "new_entities = np.array([ent for ent in dataset_entities if not ent in model_entities])\n", + "print('no. unseen entities in dataset:', len(new_entities))\n", + "\n", + "all_entities = np.concatenate((model_entities, new_entities), axis=0)\n", + "print('no. unique entities (updated):', len(all_entities))\n", + "\n", + "print('/m/01fnkt' in new_entities)" + ] + }, + { + "cell_type": "code", + "execution_count": 240, + "metadata": {}, + "outputs": [], + "source": [ + "# fit encoder\n", + "le_entity_fnn = LabelEncoder()\n", + "le_entity_fnn.classes_ = np.load(path.join('fn_embeddings','le_entity_classes.npy'), allow_pickle=True)\n", + "#load relation encoder\n", + "le_relation_fnn = LabelEncoder()\n", + "le_relation_fnn.classes_ = np.load(path.join('fn_embeddings','le_relation_classes.npy'), allow_pickle=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 241, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "array([1577])" + ] + }, + "execution_count": 241, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "le_entity_fnn.transform(['/m/01fnkt'])" + ] + }, + { + "cell_type": "code", + "execution_count": 242, + "metadata": {}, + "outputs": [], + "source": [ + "le_heads = le_entity_fnn.transform(heads)\n", + "le_tails = le_entity_fnn.transform(tails)\n", + "edge_index = torch.tensor([le_heads, le_tails], dtype=torch.long)\n", + "\n", + "le_relations = le_relation_fnn.transform(relations)\n", + "edge_type = torch.tensor(le_relations, dtype=torch.long)" + ] + }, + { + "cell_type": "code", + "execution_count": 243, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "no. true: 589\n", + "no. false: 411\n" + ] + } + ], + "source": [ + "# create and 'encode' lables\n", + "raw_labels = np.array(FNN[\"label\"])\n", + "false = 0\n", + "true = 0\n", + "for l in raw_labels:\n", + " if l == False:\n", + " false += 1\n", + " else:\n", + " true += 1\n", + "print('no. true:', true)\n", + "print('no. false:', false)\n", + "labels = torch.tensor([0 if l == False else 1 for l in raw_labels], dtype=torch.int64)" + ] + }, + { + "cell_type": "code", + "execution_count": 252, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "torch.Size([16035, 256])" + ] + }, + "execution_count": 252, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "model = torch.load(\"output/FB15k_237_CPU_20epochs.pkl\", map_location=torch.device('cpu'))\n", + "\n", + "model.z.shape" + ] + }, + { + "cell_type": "code", + "execution_count": 253, + "metadata": {}, + "outputs": [], + "source": [ + "pred = model.lp_score(model.z, edge_index, edge_type)\n", + "pred_classes = torch.tensor([0 if p < 0 else 1 for p in pred], dtype=torch.int64)\n", + "\n", + "cm = confusion_matrix(labels, pred_classes)" + ] + }, + { + "cell_type": "code", + "execution_count": 254, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 254, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAUoAAAEGCAYAAAADs9wSAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAd9klEQVR4nO3deZgdVbnv8e8vnREyNJkwJGAYIjEiBi4yXpFAgICeEzkqKHhEwRtGAREUEQVBFL0K6lXAQEBQkEnQAGEeHgaBECAEkphDlCEDJmQeSdLd7/2jVpNOp3vv3Un33r13/z7PU09XrVq16900eXutWrWqFBGYmVnzOpU6ADOz9s6J0swsDydKM7M8nCjNzPJwojQzy6NzqQNobV3VLbqzbanDsBZYt3OPUodgLbT+zfmLImLAlh5/5KhtY/GS2oLqvjRt3UMRMWZLz9UaKi5Rdmdb9tNhpQ7DWmD2j/cqdQjWQm+d8P23t+b4xUtqmfzQTgXVrRr0Rv+tOVdrqLhEaWbtXwB11JU6jII5UZpZ0QXBhiis690eOFGaWUm4RWlmlkMQ1JbR9GknSjMriTqcKM3MmhVArROlmVlublGameUQwAZfozQza14Q7nqbmeUUUFs+edKJ0syKL5uZUz6cKM2sBEQtKnUQBXOiNLOiywZznCjNzJqV3UfpRGlmllOdW5RmZs1zi9LMLI9A1JbRm2icKM2sJNz1NjPLIRDro6rUYRTMidLMii674dxdbzOznDyYY2aWQ4SoDbcozcxyqnOL0sysedlgTvmkn/Jp+5pZxagfzClkKZSkKkmvSLovbe8s6QVJsyXdLqlrKu+Wtmen/UPzfbYTpZmVRG2ooKUFzgZmNtj+GXBVROwGLAVOTuUnA0tT+VWpXk5OlGZWdPUzcwpZCiFpCPAZ4Pq0LeBQ4K5U5Sbgc2l9bNom7T8s1W9W+VwkMLOKUlf4qHd/SVMabI+PiPGN6vwK+A7QK233A5ZFRE3angsMTuuDgTkAEVEjaXmqv6i5AJwozazosodiFJwoF0XEPs3tlPRZYGFEvCTpkK2PbnNOlGZWdIHY0HpTGA8C/lPS0UB3oDfwa6BaUufUqhwCzEv15wE7AnMldQb6AItzncDXKM2s6CKgNjoVtOT/rPheRAyJiKHAl4DHI+IE4AngC6naicDf0vrEtE3a/3hE7nfnOlGaWQmIugKXrfBd4FxJs8muQU5I5ROAfqn8XOCCfB/krreZFV1Am0xhjIgngSfT+r+AfZuo8z7wxZZ8rhOlmZWEH9xrZpZDID+418wsl+x1teWTfsonUjOrIPLzKM3McglaNDOn5Jwozawk3KI0M8shQm5Rmpnlkg3m+C2MZmY5+J05ZmY5ZYM5vkZpZpaTZ+aYmeXgmTlmZgVoyYvDSs2J0syKLgI21DlRmpk1K+t6O1GameXkmTnWYude+Q77jV7JskWdOeXQ3QHoVV3Dhde+zfZD1rNgblcuP+XDrFqe/cr2PGAVp146j86dg+VLOnP+53crZfgdktbXMfjHb6CaOqiF1ftWs+Tzg+i8cB0f+t1bdFpZw7qdt2HBaR+Gzp3o9dRi+v95PjXbdQFg+eH9WTGqf4m/RWn49qBE0lnAacDL6f0VjfcfApwXEZ9tqxjKycO392Xijf05/9dzPig79syFvPJMT+747fYce+YCjjtzIRMu34Fte9dy5k/n8v0TduG9eV3p029DCSPvuKKLmHfhbkT3KqgJhlz2P6z+RG+qJy1k2ZiBrDpgOwbc8A69n1zMitEDAFi5fzWLTtyxxJG3B+XV9W7LSE8HDm8qSdrmXn+hJyuXbvp364AjV/DoHX0BePSOvhwwZgUAo45ZyrOT+vDevK4ALF/cpbjBWkbKkiSg2oCa7P1U28xYyap9qwFY+al+9HxpeakibNeK8M6cVtMmLUpJ1wK7AA9I+hPwObLXSK4Fvh4RsxrV/zTZ6yUha5UfHBErJZ0PHAt0A+6JiIvbIt72arv+G1iyMEuCSxZ2Zrv+WctxyC7rqOoS/Pyu2WzTs46/Xt+fR+/qW8pQO666YMeLZtFlwTqWH96fDdt3o3abKqjK/oHX9O1C1dKNLf6ek5fR4x+r2PCh7iz6ymBq+nUtVeQllY16d/C53hFxqqQxwChgPfDLiKiRNBr4CfD5RoecB5wREc9K6gm8L+kIYBjZy4EETJR0cEQ81fh8ksYB4wC6s01bfKV2QES6plPVORj28bV899hd6NYj+NXEN5j58rbM+1e3EsfYAXUSc34ynE6ra/jQr96k6/z3m626eq8+rDxgO+jSid6PLWLg799m/oXDihhs++EbzjfXB7hJ0jCy1mJT/cRngSsl3QLcHRFzU6I8Angl1elJljg3S5QRMR4YD9BbfXO+n7ecLF3Uhb4Ds1Zl34EbWLY4+3W9924XViztzLq1VaxbC6+90JNdRqx1oiyhum07s3ZET7q/sZqqNbVQG1AlOi/ZQG0avKnrtfGf24pR/eh327xShdsutJdudSGKcTX1MuCJiNgD+A+yLvgmIuIK4BtAD+BZScPJWpE/jYiRadktIiY0PraSPf9wb0YfuwSA0ccu4bmHegPw3IN9+NgnV9OpKujWo47he63hnTecJIut04oNdFpdA2Qj4Nu8tpL1g7uzdkQvek5eBkCvpxezau8+AJt0wbd9aTkbdtjsn0KHUT/qXcjSHhSrRVn/p/NrTVWQtGtEvAa8JumTwHDgIeAySbdExCpJg4ENEbGwCDEX3QVXv82eB6yiT98a/jRlBn/85fbc/tuBfP/atxnzpSUsnJfdHgQwZ3Z3pjzZi2sfm0XUiQdv7cvbs3qU+Bt0PJ2X1bD979+GuoCAVftVs2avPqwf3J0P/fYt+t45n/VDt2HFIf0AqH74PbZ5eTlUQe22nVmQfp8dVWuNekvqTtbT7EaW0+6KiIsl/QH4NFA/mva1iJgqSWRjIkcDa1L5y7nOUYxE+XOyrvdFwP3N1DlH0iigDpgOPBAR6yR9FHgu+16sAr4CVGSivOL0pv/RXHDcrk2W33XNQO66ZmBbhmR5rN+pB3MuH75Zec3Absy9dPfNyhcftwOLj9uhGKG1exGipvVuD1oHHJoaVF2AZyQ9kPadHxF3Nap/FNllvGHAfsA16Wez2ixRRsTQtLoI+EiDXRel/U8CT6b1bzbzGb9m42i4mVWQ1upWR0SQNaQgGwPpQta7b85Y4OZ03POSqiUNioh3mzugfO74NLOK0cJrlP0lTWmwjGv8eZKqJE0l63E+EhEvpF2XS5om6SpJ9RfyBwNzGhw+N5U1y1MYzawkWtCiXBQR++SqEBG1wEhJ1cA9kvYAvgf8G+hKdlfMd4FLtyRWtyjNrOjq76Ns7VHviFgGPAGMiYh3I7MOuJHsnmzIBpcbziMdwsYB5yY5UZpZSbTWFEZJA1JLEkk9gMOBf0galMpENjvw9XTIROCryuwPLM91fRLc9TazEoiAmtZ7cO8gsjtrqsgaf3dExH2SHpc0gOye7KnAqan+JLJbg2aT3R709XwncKI0s5JoxVHvacBeTZQf2kz9AM5oyTmcKM2s6DzX28ysAOFEaWaWWzk9FMOJ0syKLsKvgjAzy0PU+nW1Zma5+RqlmVkOfgujmVk+kV2nLBdOlGZWEh71NjPLITyYY2aWn7veZmZ5eNTbzCyHCCdKM7O8fHuQmVkevkZpZpZDIOo86m1mllsZNSidKM2sBDyYY2ZWgDJqUjpRmllJVESLUtL/I0fOj4iz2iQiM6t4AdTVVUCiBKYULQoz61gCqIQWZUTc1HBb0jYRsabtQzKzjqCc7qPMeyOTpAMkzQD+kbY/IenqNo/MzCpbFLjkIam7pMmSXpU0XdKPUvnOkl6QNFvS7ZK6pvJuaXt22j803zkKuePzV8CRwGKAiHgVOLiA48zMmiEiClsKsA44NCI+AYwExkjaH/gZcFVE7AYsBU5O9U8Glqbyq1K9nAq6NT4i5jQqqi3kODOzZrVSizIyq9Jml7QEcChwVyq/CfhcWh+btkn7D5OUMyMXkijnSDoQCEldJJ0HzCzgODOzpgVEnQpagP6SpjRYxjX+OElVkqYCC4FHgH8CyyKiJlWZCwxO64OBOQBp/3KgX65wC7mP8lTg1+nD5wMPAWcUcJyZWQ4Fj3ovioh9clWIiFpgpKRq4B5g+NbFtqm8iTIiFgEntOZJzczaYmZORCyT9ARwAFAtqXNqNQ4B5qVq84AdgbmSOgN9SGMwzSlk1HsXSfdKek/SQkl/k7TLVn0bM7PWG/UekFqSSOoBHE52efAJ4Aup2onA39L6xLRN2v94RO6blQq5RnkrcAcwCNgBuBP4cwHHmZk1rf6G80KW/AYBT0iaBrwIPBIR9wHfBc6VNJvsGuSEVH8C0C+VnwtckO8EhVyj3CYi/thg+0+Szi8kejOz5rTWDecRMQ3Yq4nyfwH7NlH+PvDFlpwj11zvvmn1AUkXALeR/R04DpjUkpOYmW2mQuZ6v0SWGOu/zSkN9gXwvbYKyswqn8poCmOuud47FzMQM+tAChyoaS8Keh6lpD2AEUD3+rKIuLmtgjKzSlfwQE27kDdRSroYOIQsUU4CjgKeAZwozWzLlVGLspDbg74AHAb8OyK+DnyC7AZNM7MtV1fg0g4U0vVeGxF1kmok9SabS7ljG8dlZpWsUh7c28CUdNf7dWQj4auA59oyKDOrfBUx6l0vIk5Pq9dKehDonW7wNDPbcpWQKCXtnWtfRLzcNiGZmbUvuVqUv8yxr/6hmO2OunWjauiupQ7DWuCfh95Y6hCshapa4TMqousdEaOKGYiZdSBBxUxhNDNrO5XQojQza0sV0fU2M2tTZZQoC3nCuSR9RdIP0/ZOkjZ7xpuZWYu00hPOi6GQKYxXk71/4stpeyXwuzaLyMwqnqLwpT0opOu9X0TsLekVgIhYKqlrG8dlZpWuwka9N0iqIjWCJQ2g3UxVN7Ny1V5ai4UopOv9G7L35A6UdDnZI9Z+0qZRmVnlK6NrlIXM9b5F0ktkj1oT8LmImNnmkZlZ5WpH1x8LUciDe3cC1gD3NiyLiHfaMjAzq3CVlCiB+9n4krHuwM7ALOBjbRiXmVU4ldFIR95rlBHx8YjYM/0cRvaeXD+P0szaBUk7SnpC0gxJ0yWdncovkTRP0tS0HN3gmO9Jmi1plqQj852jxTNzIuJlSfu19Dgzs020Xte7Bvh2yk29gJckPZL2XRURv2hYWdII4EtkveIdgEclfSQiaps7QSHXKM9tsNkJ2BuY37LvYWbWQCsO5kTEu8C7aX2lpJnA4ByHjAVui4h1wJuSZpOnp1zI7UG9GizdyK5Zji3oG5iZNafw24P6S5rSYBnX3EdKGgrsBbyQis6UNE3SDZK2S2WDgTkNDptL7sSau0WZbjTvFRHn5apnZtZihbcoF0XEPvkqSeoJ/AU4JyJWSLoGuCyd6TKyh5GftCWh5noVROeIqJF00JZ8sJlZc0TrjnpL6kKWJG+JiLsBImJBg/3XAfelzXls+ibZIamsWblalJPJrkdOlTQRuBNYXb+zPhgzsxZrxWuUkgRMAGZGxJUNygel65cAxwCvp/WJwK2SriQbzBlGlu+aVciod3dgMdk7curvpwzAidLMtlzrjXofBPw38JqkqansQuDLkkamM70FnAIQEdMl3QHMIBsxPyPXiDfkTpQD04j362xMkPXK6J56M2uXWm/U+xk2zU/1JuU45nLg8kLPkStRVgE9mwnAidLMtkqlzPV+NyIuLVokZtaxVEiiLJ+nappZeYnymuudK1EeVrQozKzjqYQWZUQsKWYgZtaxVMo1SjOztuNEaWaWQzt6zUMhnCjNrOiEu95mZnk5UZqZ5eNEaWaWhxOlmVkOlfa6WjOzNuFEaWaWW6VMYTQzazPuepuZ5eIbzs3MCuBEaWbWPM/MMTMrgOrKJ1M6UZpZ8fkapZlZfu56m5nl40RpZpZbObUoO5U6ADProKLAJQ9JO0p6QtIMSdMlnZ3K+0p6RNIb6ed2qVySfiNptqRpkvbOdw4nSjMrvvQWxkKWAtQA346IEcD+wBmSRgAXAI9FxDDgsbQNcBQwLC3jgGvyncCJ0syKrv4+ykKWfCLi3Yh4Oa2vBGYCg4GxwE2p2k3A59L6WODmyDwPVEsalOscTpRmVhoRhS0tIGkosBfwArB9RLybdv0b2D6tDwbmNDhsbiprlgdzzKwkWjCY01/SlAbb4yNi/GafJ/UE/gKcExErJH2wLyJC2vLhI7co24lzvvMSt95zP1ff+Ohm+4459g0mPXk3vfus26R82O5LuPexezjo0/OKFaY1UlsLpx/+EX7w1Z03Kb/6osGM3e3jH2w/fHtfjt1jD04bvTunjd6dB27pW+xQ25dCB3Ky1LYoIvZpsDSVJLuQJclbIuLuVLygvkudfi5M5fOAHRscPiSVNasoiVJStaTTi3GucvXogx/mB985cLPy/gPWsPc+C1j47x6blHfqFJx0ynRefnFgsUK0Jvz1+gHsOGzTP2D/82oPVi2v2qzuwf+5lGsencU1j87iqBOWFCvEdqu1BnOUNR0nADMj4soGuyYCJ6b1E4G/NSj/ahr93h9Y3qCL3qRitSirgc0SpSR3/ZPXp/Vn5cqum5WPO3MaN/x+DwJtUv4f//VPnn1qB5Yt61asEK2R9+Z3YfJjvTnq+MUflNXWwnWX7cDJF80vYWTloRVHvQ8C/hs4VNLUtBwNXAEcLukNYHTaBpgE/AuYDVxHE7mpsWIlqiuAXSVNBTYA7wNLgeGSjgDui4g9ACSdB/SMiEsk7Qr8DhgArAH+T0T8o0gxl9z+B81n8Xs9ePOf1ZuU9+u/lgP/93wu+NanOGf4S6UJzrj24sF846L5rFm1sfU48cb+HHDECvptX7NZ/WcnVfP6Cz0ZvMs6TrlkHgMHbyhmuO1L0OKBmmY/KuIZaNSS2OiwJuoHcEZLzlGsFuUFwD8jYiRwPrA3cHZEfCTPceOBb0bE/wLOA65uqpKkcZKmSJqyvnZNK4ZdOt261XDcCbP4440jNts37sxp3DB+DyKa+3/D2trzj/Smun8Nw/Zc+0HZ4n935ul7qxl70nub1d//8OXc9MIMrn1sFnsfvJJfnLNTMcNtl1rr9qBiKFXXd3JEvJmrQhrBOhC4s8HoVZP9zHRxdzxAn+6D2sl/2q0zaIfVbD9oDb+b8BgA/Qes5TfjH+dbp41i2O5LueCHkwHo3Wcdn9xvAXW14rlndihlyB3KjBe35fmHe/PiYyNYv06sWVnFuFHD6dI1+PqB2R+3dWs78bUDP8of/j6T3n1rPzh2zPGLuf7H/l15rnd+qxus17Bpy7Z7+tkJWJZaoR3OW2/24fhjPvPB9o23PcjZp4xixfJunPTlMR+Uf+uCKUx+bpCTZJGddOG7nHRhdv3/1b/35K5rB3DZzZv+7R+728f5w99nArB4QecPuuPPP9yHnYa9X9yA2xk/uLdpK4FezexbAAyU1A9YBXwWeDDdB/WmpC9GxJ1pZGvPiHi1SDEX1Xd+MJk9R75H7z7rufnOSfzpxhE8PGloqcOyVvK3CQN47uHeVHWGXtU1fPuqd0odUmlFlNWDexWtdEE174mkW4E9gbXAgoj4bIN9ZwFnk93L9C/grTSYszPZPMxBQBfgtoi4NNd5+nQfFAcMPTFXFWtnJj35l1KHYC1UNWj2SxGxz5Ye36t6SOx18NkF1X363u9s1blaQ9G63hFxfI59vwF+00T5m8CYzY8ws3LnrreZWS4BlFHX24nSzEqjfPKkE6WZlYa73mZmeZTTqLcTpZkVn19Xa2aWW3bDeflkSidKMyuNwp4M1C44UZpZSbhFaWaWi69RmpnlU15zvZ0ozaw03PU2M8shCn7NQ7vgRGlmpeEWpZlZHuWTJ50ozaw0VFc+fW8nSjMrvsA3nJuZ5SKirG44L9bras3MNhVR2JKHpBskLZT0eoOySyTNkzQ1LUc32Pc9SbMlzZJ0ZCGhOlGaWWm0UqIE/kDTr4y5KiJGpmUSgKQRwJeAj6VjrpZUle8ETpRmVnz11ygLWfJ9VMRTwJICzzyW7CWF69I7uWYD++Y7yInSzEpCdXUFLVvhTEnTUtd8u1Q2GJjToM7cVJaTE6WZlUCB3e6s691f0pQGy7gCTnANsCswEngX+OXWROtRbzMrvqAlM3MWtfS93hGxoH5d0nXAfWlzHrBjg6pDUllOblGaWWm00jXKpkga1GDzGKB+RHwi8CVJ3STtDAwDJuf7PLcozawkWus+Skl/Bg4h66LPBS4GDpE0kqzt+hZwCkBETJd0BzADqAHOiIjafOdwojSz0milRBkRX26ieEKO+pcDl7fkHE6UZlZ8EVBbPnMYnSjNrDTKaAqjE6WZlYYTpZlZDgH4nTlmZrkEhK9Rmpk1L/BgjplZXr5GaWaWhxOlmVkuBT9rsl1wojSz4gvALxczM8vDLUozs1w8hdHMLLeA8H2UZmZ5eGaOmVkevkZpZpZDhEe9zczycovSzCyXIGrzvoGh3XCiNLPi82PWzMwK4NuDzMyaF0C4RWlmlkP4wb1mZnmV02COooyG6Ash6T3g7VLH0Ub6A4tKHYQVrJJ/Xx+OiAFberCkB8n++xRiUUSM2dJztYaKS5SVTNKUiNin1HFYYfz7qhydSh2AmVl750RpZpaHE2V5GV/qAKxF/PuqEL5GaWaWh1uUZmZ5OFGameXhRFliks6SNFPSLc3sP0TSfcWOy5onqVrS6aWOw4rHibL0TgcOj4gTSh2IFaya7Pe2CUme6VahnChLSNK1wC7AA5K+K+k5Sa9I+ruk3Zuo/2lJU9PyiqReqfx8SS9KmibpR8X+Hh3QFcCu6ffwoqSnJU0EZkgaKun1+oqSzpN0SVrfVdKDkl5KxwwvUfzWQv4LWEIRcaqkMcAoYD3wy4iokTQa+Anw+UaHnAecERHPSuoJvC/pCGAYsC8gYKKkgyPiqeJ9kw7nAmCPiBgp6RDg/rT9pqShOY4bD5waEW9I2g+4Gji0rYO1redE2X70AW6SNIzsKVRdmqjzLHBlup55d0TMTYnyCOCVVKcnWeJ0oiyeyRHxZq4K6Q/bgcCdkuqLu7V1YNY6nCjbj8uAJyLimNQqebJxhYi4QtL9wNHAs5KOJGtF/jQifl/MYG0Tqxus17DpJa3u6WcnYFlEjCxWUNZ6fI2y/egDzEvrX2uqgqRdI+K1iPgZ8CIwHHgIOCm1WJA0WNLAIsTbka0EejWzbwEwUFI/Sd2AzwJExArgTUlfBFDmE0WJ1raaW5Ttx8/Jut4XkV3zaso5kkYBdcB04IGIWCfpo8BzqUu3CvgKsLAIMXdIEbFY0rNp0GYtWXKs37dB0qXAZLI/fP9ocOgJwDXpd9wFuA14tXiR25byFEYzszzc9TYzy8OJ0swsDydKM7M8nCjNzPJwojQzy8OJsgOSVJvmKb8u6U5J22zFZ/1B0hfS+vWSRuSoe4ikA7fgHG9J2uyNfc2VN6qzqoXnukTSeS2N0SqbE2XHtDYiRkbEHmRzzE9tuHNLn4ITEd+IiBk5qhxCNo3PrKw4UdrTwG6ptdfwKThVkv5vg6cSnQIfzCj5raRZkh4FPpgFJOlJSfuk9TGSXpb0qqTH0rTMU4FvpdbspyQNkPSXdI4XJR2Uju0n6WFJ0yVdTzZNMydJf01P5ZkuaVyjfVel8sckDUhlfpKPFcwzczqw1HI8CngwFe3NxqfgjAOWR8Qn01S8ZyU9DOwF7A6MALYHZgA3NPrcAcB1wMHps/pGxBJlj5VbFRG/SPVuBa6KiGck7UQ2HfOjwMXAMxFxqaTPACcX8HVOSufoAbwo6S8RsRjYFpgSEd+S9MP02WfiJ/lYCzhRdkw9JE1N608DE8i6xA2fgnMEsGf99UeyuejDgIOBP0dELTBf0uNNfP7+wFP1nxURS5qJYzQwosHTdHqnOesHA/+Vjr1f0tICvtNZko5J6zumWBeTTfe8PZX/CbjbT/KxlnKi7JjWNn6KTUoYDZ+CI+CbEfFQo3pHt2IcnYD9I+L9JmIpWHom5GjggIhYI+lJNj61p7HAT/KxFvI1SmvOQ8BpkroASPqIpG3JnnN5XLqGOYjsocONPQ8cLGnndGzfVN74qTsPA9+s35A0Mq0+BRyfyo4CtssTax9gaUqSw8latPU6AfWt4uPJuvR+ko+1iBOlNed6suuPL6en5PyerAdyD/BG2ncz8FzjAyPiPWAcWTf3VTZ2fe8FjqkfzAHOAvZJg0Uz2Dj6/iOyRDudrAv+Tp5YHwQ6S5pJ9pqG5xvsWw3sm77DocClqfwE4OQU33RgbAH/TayD8tODzMzycIvSzCwPJ0ozszycKM3M8nCiNDPLw4nSzCwPJ0ozszycKM3M8vj/1Vhk6QSaJX8AAAAASUVORK5CYII=\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "disp = ConfusionMatrixDisplay(confusion_matrix=cm, display_labels=['false', 'true']) #usikker på rekkefølgen her\n", + "disp.plot()" + ] + }, + { + "cell_type": "code", + "execution_count": 255, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Triple validity prediction vs its documents label: 0.5933333333333334\n" + ] + } + ], + "source": [ + "triple_classification_acc = precision_score(labels, pred_classes)\n", + "\n", + "print('Triple validity prediction vs its documents label:', triple_classification_acc)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Document classification\n", + "The fist strategy will be to assign the lable 'False' to a document as long as one false triple was observed in it.." + ] + }, + { + "cell_type": "code", + "execution_count": 256, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "(280,)" + ] + }, + "execution_count": 256, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "text_pred_dict = {}\n", + "for i in range(edge_index.shape[1]):\n", + " text_id = FNN.index[i]\n", + " pred_class = pred_classes[i]\n", + " if text_id in text_pred_dict:\n", + " if pred_class == 0:\n", + " text_pred_dict[text_id] = pred_class\n", + " else:\n", + " text_pred_dict[text_id] = pred_class\n", + " \n", + "pred_text_labels = np.array(list(text_pred_dict.values())).flatten()\n", + "pred_text_labels.shape" + ] + }, + { + "cell_type": "code", + "execution_count": 257, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Percentage of true documents: 0.5321428571428571\n" + ] + } + ], + "source": [ + "text_labels = FNN.groupby('text_id').mean().values.flatten()\n", + "\n", + "print('Percentage of true documents:', text_labels.mean())" + ] + }, + { + "cell_type": "code", + "execution_count": 258, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "0.5228758169934641" + ] + }, + "execution_count": 258, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "precision_score(text_labels, pred_text_labels)" + ] + }, + { + "cell_type": "code", + "execution_count": 251, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 251, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAUoAAAEGCAYAAAADs9wSAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAbUklEQVR4nO3de7xVdZ3/8df7gILc5SISaJiSlyE1h5TyMQ5eUktHs7E0qTGzn5mmZkOJM82o9StvmelvRotRJyzHa5aU9+tPJVPxkiJakqRCIgGKoiCccz7zx1oHNodz9l5ns/dee+/zfj4e68Ha37X2Wp8j+Dnfy/p+lyICMzPrXkveAZiZ1TsnSjOzEpwozcxKcKI0MyvBidLMrIS+eQdQaZurX/RnYN5hWA+Mnrgq7xCsh+bPXb00IkaV+/2D9h0Yy5a3ZTr3iWfeuzMiDi73XpXQdImyPwPZS/vnHYb1wKm3vJB3CNZDh20/9+VN+f6y5W08due2mc7tM+bFkZtyr0poukRpZvUvgHba8w4jMydKM6u5IFgb2Zre9cCJ0sxy4RqlmVkRQdDWQNOnnSjNLBftOFGamXUrgDYnSjOz4lyjNDMrIoC17qM0M+teEG56m5kVFdDWOHnSidLMai+ZmdM4nCjNLAeiDeUdRGZOlGZWc8lgjhOlmVm3kuconSjNzIpqd43SzKx7rlGamZUQiLYGehONE6WZ5cJNbzOzIgKxJvrkHUZmjVP3NbOmkTxw3pJpK0XSVZKWSJpbUHahpBckPSPpl5KGFRw7U9J8SX+QdFCWeJ0ozSwXbelD56W2DH4KdH5L493AxIjYFfgjcCaApF2Ao4G/Sb9zmaSSVVsnSjOruQjRFi2ZttLXigeB5Z3K7oqI1vTj74Bx6f7hwHUR8V5ELADmA3uWuocTpZnloh1l2oCRkuYUbCf08FZfAm5P98cCrxYcW5iWFeXBHDOruWQwJ3P6WRoRk8q5j6R/BVqBa8r5fgcnSjOruY7BnGqS9EXgUGD/iHWrBC8Ctik4bVxaVpSb3maWi7ZQpq0ckg4GvgUcFhHvFhyaBRwtqZ+k7YAJwGOlrucapZnVXCVn5ki6FphC0pe5EDiLZJS7H3C3JIDfRcSJEfGcpBuAeSRN8pMjoq3UPZwozSwX7RlGtLOIiM91UXxlkfO/B3yvJ/dwojSzmksWxWicnj8nSjOruUCsbaApjE6UZlZzEWR6mLxeOFGaWQ7WPUzeEJwozazmAtcozcxK8mCOmVkRgbxwr5lZMcnrahsn/TROpGbWRDKvNVkXnCjNrOaCys3MqQUnSjPLhWuUZmZFRMg1SjOzYpLBHE9hNDMrQn7g3MysmGQwx32UZmZFeWaOmVkRnpljZpZBtV8uVklOlGZWcxGwtt2J0sysW0nT24nSzKwoz8yxTfKNH77CXge8zZtL+/KV/XYE4Mv/9hcmf/wt1q4Rr728ORedvi3vvNU4D+w2o3umb82C+waxxYg2Pn/7AgBevG0wj146kuV/2pyjbn6Z0R9aDUDbGrjv37ZmybP9UQvs8+0ljJv8brHLN7VGezyoanVfSadKel7SNd0cnyLpN9W6fyO76/rh/OvU7TYoe/LBwZyw74589YAdWfRSP44+5fWcorMOO396BYdf9eoGZSM++B6HXLaIsR9ZtUH53OuHATD1tj/zqZmv8tC5WxHttYq0HiVN7yxbPahmFCcBH4+IqVW8R1Oa++gg3n5jw8r+k/9/MO1tyW/g558YyMgxa/MIzQqM3XMV/YdtmO2G77CGLT+wZqNzl8/vt64GOWBEG/2GtPH6s/1rEme9ak/fm1NqqwdVSZSSfgx8ALhd0hmSHpH0lKTfStqxi/P/XtLT6faUpMFp+TclPS7pGUnnVCPWRnTQ55bz+H1D8g7DemDUzqtZcO8g2lthxaubsWRuf1a+tlneYeUmGfXuk2mrB1Xpo4yIEyUdDOwLrAEuiohWSQcA3wf+sdNXpgEnR8RsSYOA1ZIOBCYAewICZknaJyIe7Hw/SScAJwD0Z0A1fqS68blTX6etFe67eVjeoVgP7HLkCpbP78d1R4xn8PvWMmaPVahP5B1WbvzA+caGAjMlTSDpw+3q1+hs4Idpf+bNEbEwTZQHAk+l5wwiSZwbJcqImAHMABii4U37r+/jn13Onge8xfSjtoc6aZJYNi19kwGcDjd8ZluGjd+4id6b1EuzOota9JR+F7g/IiYC/wBs1DETEecBXwa2AGZL2okkE5wbEbun2w4RcWUN4q1Lk6a8xWdOWsLZX9yO91bVRwe3Zbd2lVj7bpIYXnl4AC19YcSE3psoO0a9s2ylSLpK0hJJcwvKhku6W9KL6Z9bpuWSdKmk+WmX3h5Z4q1VjXJRuv/Frk6QtH1EPAs8K+kjwE7AncB3JV0TESsljQXWRsSSrq7RTKZf9jK7fnQlQ4e38vM58/jZRaM5+mtL2KxfcO71fwLghScGcun0cTlH2rvd8fX3sfDRAax+ow9X7r09k09bSv9hbTxwzmhWLe/DrC+PY9TOq/nUTxeyallffnXcONQCg0a3cuAP/pJ3+Lmr4Ij2T4H/AK4uKJsO3BsR50mann4+A/gESct0ArAXcHn6Z1G1SJQXkDS9vw3c2s05X5e0L9AOPAfcHhHvSdoZeEQSwErg80DTJ8rzTnr/RmV3Xjsih0ismIN/1HWy2/7AlRuVDRm3ln+6e0G1Q2oYEaK1QokyIh6UNL5T8eHAlHR/JvAASaI8HLg6IgL4naRhksZExGvF7lG1RBkR49PdpcAHCw59Oz3+AEnwRMQp3VzjEuCSasVoZvnpwWDOSElzCj7PSMclihldkPwWA6PT/bFA4cOvC9OyfBKlmVl3ejgzZ2lETCr7XhEhaZMGeZ0ozSwXVX486PWOJrWkMazvslsEbFNw3jjWj6F0y8OnZlZzHc9RVmLUuxuzgGPT/WOBWwrK/ykd/Z4MrCjVPwmuUZpZTir1HKWka0kGbkZKWgicBZwH3CDpeOBl4LPp6bcBnwTmA+8Cx2W5hxOlmdVcBLRWaOHeiPhcN4f27+LcAE7u6T2cKM0sF57CaGZWhOd6m5llEE6UZmbFNdKiGE6UZlZzEe6jNDMrQbT5dbVmZsW5j9LMrIhGewujE6WZ1V4k/ZSNwonSzHLhUW8zsyLCgzlmZqW56W1mVoJHvc3MiohwojQzK8mPB5mZleA+SjOzIgLR7lFvM7PiGqhC6URpZjnwYI6ZWQYNVKV0ojSzXDRFjVLS/6NIzo+IU6sSkZk1vQDa25sgUQJzahaFmfUuATRDjTIiZhZ+ljQgIt6tfkhm1hs00nOUJR9kkvRRSfOAF9LPu0m6rOqRmVlzi4xbHcjyxOePgIOAZQAR8XtgnyrGZGZNT0Rk2+pBpkfjI+LVTkVtVYjFzHqTCtYoJZ0u6TlJcyVdK6m/pO0kPSppvqTrJW1ebqhZEuWrkj4GhKTNJE0Dni/3hmZmBES7Mm2lSBoLnApMioiJQB/gaOB84OKI2AF4Azi+3HCzJMoTgZOBscBfgN3Tz2Zmm0AZt0z6AltI6gsMAF4D9gNuSo/PBD5VbqQlHziPiKXA1HJvYGbWpQoN1ETEIkk/AF4BVgF3AU8Ab0ZEa3raQpLKXlmyjHp/QNKvJf1V0hJJt0j6QLk3NDMDetJHOVLSnILthMLLSNoSOBzYDngfMBA4uJKhZpnC+D/AfwJHpJ+PBq4F9qpkIGbWi/TsgfOlETGpyPEDgAUR8VcASTcDewPDJPVNa5XjgEXlhpulj3JARPwsIlrT7edA/3JvaGYGHa+DKL1l8AowWdIASQL2B+YB9wNHpuccC9xSbqzF5noPT3dvlzQduI7k98BRwG3l3tDMDIAKzfWOiEcl3QQ8CbQCTwEzgFuB6yT937TsynLvUazp/QRJYuz4ab5SGBtwZrk3NTNTBWfdRMRZwFmdil8C9qzE9YvN9d6uEjcwM9tIHU1PzCLTepSSJgK7UNA3GRFXVysoM2t2ao7VgzpIOguYQpIobwM+ATwMOFGaWfkaqEaZZdT7SJJRpMURcRywGzC0qlGZWfNrz7jVgSxN71UR0S6pVdIQYAmwTZXjMrNm1iwL9xaYI2kY8F8kI+ErgUeqGZSZNb9KjnpXW5a53ieluz+WdAcwJCKeqW5YZtb0miFRStqj2LGIeLI6IZmZ1ZdiNcqLihwLkiWM6o5aWmgZMDDvMKwHDhmwOu8QLAdN0fSOiH1rGYiZ9SJBxaYw1kKmB87NzCquGWqUZmbV1BRNbzOzqmqgRJllhXNJ+rykf08/byupIitymFkv1mTv9b4M+CjwufTz2yQrnpuZlUWRfasHWZree0XEHpKeAoiINzbl/bhmZkDTjXqvldSHtBIsaRR1M1XdzBpVvdQWs8jS9L4U+CWwlaTvkSyx9v2qRmVmza+B+iizzPW+RtITJEutCfhURDxf9cjMrHnVUf9jFlkW7t0WeBf4dWFZRLxSzcDMrMk1U6IkeZNZx0vG+pO8ZPwPwN9UMS4za3JqoJGOLE3vDxV+TlcVOqmb083Mmk6PZ+ZExJOS9qpGMGbWizRT01vSNwo+tgB7AH+pWkRm1vyabTAHGFyw30rSZ/mL6oRjZr1GsyTK9EHzwRExrUbxmFlv0QyJUlLfiGiVtHctAzKz5ieaZ9T7MZL+yKclzQJuBN7pOBgRN1c5NjNrVhXuo0zfFHsFMDG5Ol8ieYzxemA88GfgsxHxRjnXzzKFsT+wjOQdOYcC/5D+aWZWvspOYbwEuCMidgJ2A54HpgP3RsQE4N70c1mK1Si3Ske857L+gfMODdS7YGZ1qUJZRNJQYB/giwARsQZYI+lwYEp62kzgAeCMcu5RLFH2AQaxYYLs4ERpZpukgk3v7YC/Av8taTfgCeA0YHREvJaesxgYXe4NiiXK1yLiO+Ve2MysqOyJcqSkOQWfZ0TEjILPfUnGU06JiEclXUKnZnZEhFR+ai6WKBtnVU0zayzRo1HvpRExqcjxhcDCiHg0/XwTSaJ8XdKYiHhN0hhgSbnhFhvM2b/ci5qZlVShwZyIWAy8KmnHtGh/YB4wCzg2LTsWuKXcULutUUbE8nIvamZWSoWnMJ4CXJO+puYl4DiSiuANko4HXgY+W+7F/bpaM8tHBRNlRDwNdNU8r0jL2InSzGqvjl7zkIUTpZnVnGi+1YPMzCrOidLMrBQnSjOzEpwozcyKaMIVzs3MKs+J0sysuGZZuNfMrGrc9DYzK8YPnJuZZeBEaWbWPc/MMTPLQO2NkymdKM2s9txHaWZWmpveZmalOFGamRXnGqWZWSlOlGZmRfTsLYy5c6I0s5rzc5RmZllE42RKJ0ozy4VrlLZJTj93Pnvuu5w3l23GVw/58Lryw77wGodOXUx7Ozz2wJZcdcH4/II0Ljp9Gx69ZwjDRrYy4/4/ADDzgq155M6hSDBs5Fqm/egVRmzdym/vGMLVF45Bgj59gxPPWcTEvd7J+SfIkR8435ikYcAxEXFZLe7X6O6+eRSzfrY10y58cV3ZrnutYPL+yzn5sN1Yu6aFocPX5BihARx41HIOO24pF5627bqyI7+6hGO/tRiAX10xkp9fvDWnnb+QD//dSj560B+Q4KV5/fneV8Zz5UMv5BV6XWikwZyWGt1nGHBS50JJrtF2Ye7jQ3l7xYb/aQ45ZjE3zBjL2jXJX9mK5ZvnEZoV+NDkdxi8ZdsGZQMHr/+/f/WqFqRkf4uB7ev2V7+7vrw3U3u2rR7UKlGdB2wv6WlgLbAaeAPYSdKBwG8iYiKApGnAoIg4W9L2wH8Co4B3gf8TEb3y1/DY7VYxcdJbHPuNV1j7XgtXnPd+/vjs4LzDsi7893lbc8+Nwxk4pI0Lbpq/rnz27UO56vtjeHNZX7579Us5RlgHgoYazKlVjXI68KeI2B34JrAHcFpEfLDE92YAp0TE3wLTgC6b7pJOkDRH0pw1sbqCYdePPn2CwUNbOf3ID3HF+e/nzEv+SEN18vQix01fzDVPzGO/T7/BrKtGrSvf+xMruPKhFzj7qgXMvGBMjhHWB0W2rR7UKlF29lhELCh2gqRBwMeAG9Oa6E+ALv91RcSMiJgUEZM2V/+KB1sPli7ux+y7RgDij88MJgKGDm/NOywrYr8j3uDh24ZuVP6hye+w+JXNWbGsTw5R1ZHIuGUkqY+kpyT9Jv28naRHJc2XdL2ksvur8kqUhcN9rZ3i6Mh0LcCbEbF7wbZzzSKsM4/cM5zdJq8AYOz4VfTdLFix3F289WbRS+v/X3zkzqFss8N7SfmCzde1NF98ZgvWrhFDhrd1dYleoeOB8wrXKE8Dni/4fD5wcUTsQNLVd3y58dbq/7S3ge461F4HtpI0AlgJHArcERFvSVog6TMRcaMkAbtGxO9rFHNuzrj4j+y65wqGbNnKzx6aw88u2Ya7btqK08+dz+W3PkXr2hYu+tYEkn9ulpdzv/p+nnlkECuW92Xq3+7CF/55MY/dN4SFf+pHSwtsNXYNp56/EICHbx3GPTdtSd++0G+Ldv7l8pd794BOREUX7pU0DjgE+B7wjTRf7Acck54yEzgbuLyc69ckUUbEMkmzJc0FVpEkx45jayV9B3gMWAQUDtZMBS6X9G1gM+A6oOkT5fmnd911e+G0Ul26VktnXv7yRmUHH7O8y3OP+toSjvrakmqH1Fiy58mRkuYUfJ4RETM6nfMj4Fusr5CNIGmRdvRPLQTGlhdoDR84j4hjihy7FLi0i/IFwMHVjMvM8tGDZvXSiJjU7XWkQ4ElEfGEpCmbHtnG3MllZrUXQOWa3nsDh0n6JMkYxxDgEmCYpL5prXIcSYu1LHkN5phZb1ehUe+IODMixkXEeOBo4L6ImArcDxyZnnYscEu5oTpRmlkuavAc5RkkAzvzSfosryz3Qm56m1kuqvG62oh4AHgg3X8J2LMS13WiNLPa8+pBZmbFJQ+cN06mdKI0s3zUycpAWThRmlkuXKM0MyvGfZRmZqVUdq53tTlRmlk+3PQ2Mysi6uc1D1k4UZpZPlyjNDMroXHypBOlmeVD7Y3T9naiNLPaC/zAuZlZMSL8wLmZWUlOlGZmJThRmpkV4T5KM7PSPOptZlZUuOltZlZU4ERpZlZS47S8nSjNLB9+jtLMrBQnSjOzIiKgrXHa3k6UZpYP1yjNzEpwojQzKyIAvzPHzKyYgGicPsqWvAMws14oSAZzsmwlSNpG0v2S5kl6TtJpaflwSXdLejH9c8tyw3WiNLN8RGTbSmsF/jkidgEmAydL2gWYDtwbEROAe9PPZXGiNLN8VChRRsRrEfFkuv828DwwFjgcmJmeNhP4VLmhuo/SzHLQo0UxRkqaU/B5RkTM6OpESeOBDwOPAqMj4rX00GJgdJnBOlGaWQ4CyL7M2tKImFTqJEmDgF8AX4+ItyStv11ESCp7mN1NbzPLR+X6KJG0GUmSvCYibk6LX5c0Jj0+BlhSbqhOlGaWg6jkqLeAK4HnI+KHBYdmAcem+8cCt5QbrZveZlZ7AVG55yj3Br4APCvp6bTsX4DzgBskHQ+8DHy23Bs4UZpZPio0MyciHgbUzeH9K3EPJ0ozy4fnepuZFRHRk1Hv3DlRmlk+XKM0MysmiLa2vIPIzInSzGrPy6yZmWXQQMusOVGaWc0FEK5RmpkVEY21cK8TpZnlopEGcxQNNESfhaS/kkxXakYjgaV5B2GZNfPf1/sjYlS5X5Z0B8l/nyyWRsTB5d6rEpouUTYzSXOyLDdl9cF/X83DqweZmZXgRGlmVoITZWPpcvl7q1v++2oS7qM0MyvBNUozsxKcKM3MSnCizJmkUyU9L+mabo5PkfSbWsdl3ZM0TNJJecdhteNEmb+TgI9HxNS8A7HMhpH8vW1Akme6NSknyhxJ+jHwAeB2SWdIekTSU5J+K2nHLs7/e0lPp9tTkgan5d+U9LikZySdU+ufoxc6D9g+/Xt4XNJDkmYB8ySNlzS340RJ0ySdne5vL+kOSU+k39kpp/ith/wbMEcRcaKkg4F9gTXARRHRKukA4PvAP3b6yjTg5IiYnb7sfbWkA4EJwJ4kL1iaJWmfiHiwdj9JrzMdmBgRu0uaAtyafl4gaXyR780AToyIFyXtBVwG7FftYG3TOVHWj6HATEkTSFah2qyLc2YDP0z7M2+OiIVpojwQeCo9ZxBJ4nSirJ3HImJBsRPSX2wfA25MXkMNQL9qB2aV4URZP74L3B8RR6S1kgc6nxAR50m6FfgkMFvSQSS1yHMj4ie1DNY28E7Bfisbdmn1T/9sAd6MiN1rFZRVjvso68dQYFG6/8WuTpC0fUQ8GxHnA48DOwF3Al9KayxIGitpqxrE25u9DQzu5tjrwFaSRkjqBxwKEBFvAQskfQZAid1qEq1tMtco68cFJE3vb5P0eXXl65L2BdqB54DbI+I9STsDj6RNupXA54ElNYi5V4qIZZJmp4M2q0iSY8extZK+AzxG8ovvhYKvTgUuT/+ONwOuA35fu8itXJ7CaGZWgpveZmYlOFGamZXgRGlmVoITpZlZCU6UZmYlOFH2QpLa0nnKcyXdKGnAJlzrp5KOTPevkLRLkXOnSPpYGff4s6SN3tjXXXmnc1b28F5nS5rW0xituTlR9k6rImL3iJhIMsf8xMKD5a6CExFfjoh5RU6ZQjKNz6yhOFHaQ8AOaW2vcBWcPpIuLFiV6CuwbkbJf0j6g6R7gHWzgCQ9IGlSun+wpCcl/V7Svem0zBOB09Pa7N9JGiXpF+k9Hpe0d/rdEZLukvScpCtIpmkWJelX6ao8z0k6odOxi9PyeyWNSsu8ko9l5pk5vVhac/wEcEdatAfrV8E5AVgRER9Jp+LNlnQX8GFgR2AXYDQwD7iq03VHAf8F7JNea3hELFeyrNzKiPhBet7/ABdHxMOStiWZjrkzcBbwcER8R9IhwPEZfpwvpffYAnhc0i8iYhkwEJgTEadL+vf02l/DK/lYDzhR9k5bSHo63X8IuJKkSVy4Cs6BwK4d/Y8kc9EnAPsA10ZEG/AXSfd1cf3JwIMd14qI5d3EcQCwS8FqOkPSOev7AJ9Ov3urpDcy/EynSjoi3d8mjXUZyXTP69PynwM3eyUf6yknyt5pVedVbNKEUbgKjoBTIuLOTud9soJxtACTI2J1F7Fklq4JeQDw0Yh4V9IDrF+1p7PAK/lYD7mP0rpzJ/BVSZsBSPqgpIEk61welfZhjiFZdLiz3wH7SNou/e7wtLzzqjt3Aad0fJC0e7r7IHBMWvYJYMsSsQ4F3kiT5E4kNdoOLUBHrfgYkia9V/KxHnGitO5cQdL/+GS6Ss5PSFogvwReTI9dDTzS+YsR8VfgBJJm7u9Z3/T9NXBEx2AOcCowKR0smsf60fdzSBLtcyRN8FdKxHoH0FfS8ySvafhdwbF3gD3Tn2E/4Dtp+VTg+DS+54DDM/w3sV7KqweZmZXgGqWZWQlOlGZmJThRmpmV4ERpZlaCE6WZWQlOlGZmJThRmpmV8L9dUTC6QL7UowAAAABJRU5ErkJggg==\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "text_cm = confusion_matrix(text_labels, pred_text_labels)\n", + "text_disp = ConfusionMatrixDisplay(confusion_matrix=text_cm, display_labels=['false', 'true']) #usikker på rekkefølgen her\n", + "text_disp.plot()" + ] + }, + { + "cell_type": "code", + "execution_count": 236, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "env37test", + "language": "python", + "name": "env37test" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.5" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/run_fn_detection.py b/run_fn_detection.py new file mode 100644 index 0000000..779c8fd --- /dev/null +++ b/run_fn_detection.py @@ -0,0 +1,194 @@ +import sys +import ssl +from os import path, mkdir +import numpy as np +import pandas as pd +import torch +import trainer +import utils.gsn_argparse as gap +import utils.label_encode_dataset as led +import utils.create_node_embedding as cne +import utils.create_relation_embedding as cre +from gensim.models import KeyedVectors +from sklearn.preprocessing import LabelEncoder +from torch_geometric.utils import structured_negative_sampling + +ssl._create_default_https_context = ssl._create_unverified_context + + +def train_val_test_split(dataset, train, val, test): + # edge_indexes + dataset.train_pos_edge_index = train.edge_index + dataset.val_pos_edge_index = val.edge_index + dataset.test_pos_edge_index = test.edge_index + + # relations + dataset.train_edge_type = train.edge_type + dataset.val_edge_type = val.edge_type + dataset.test_edge_type = test.edge_type + + # negatives + neg_train = structured_negative_sampling(dataset.train_pos_edge_index) + dataset.train_neg_edge_index = torch.tensor( + [list(neg_train[0]), list(neg_train[2])], dtype=torch.long + ) + + neg_val = structured_negative_sampling(dataset.val_pos_edge_index) + dataset.val_neg_edge_index = torch.tensor( + [list(neg_val[0]), list(neg_val[2])], dtype=torch.long + ) + + neg_test = structured_negative_sampling(dataset.test_pos_edge_index) + dataset.test_neg_edge_index = torch.tensor( + [list(neg_test[0]), list(neg_test[2])], dtype=torch.long + ) + + return dataset + + +def load_data(news_dataset, kg_dataset, dataset_name, hidden=64, node_embedding_size=16, embedding_path="fn_embeddings"): + print("Loading open KG: " + kg_dataset + "...") + columns = { + "FB15k": ["head", "tail", "relation"], + "FB15k_237": ["head", "relation", "tail"], + } + + train = pd.read_csv( + "./data/" + kg_dataset + "/train.txt", + sep="\t", + header=None, + names=columns[kg_dataset], + engine="python", + ) + valid = pd.read_csv( + "./data/" + kg_dataset + "/valid.txt", + sep="\t", + header=None, + names=columns[kg_dataset], + engine="python", + ) + test = pd.read_csv( + "./data/" + kg_dataset + "/test.txt", + sep="\t", + header=None, + names=columns[kg_dataset], + engine="python", + ) + + print("Loading fake news dataset: " + news_dataset + "...") + fn_test = pd.read_csv( + "./data/" + news_dataset + "/train.csv", + sep=",", + header=0, + engine="python", + index_col=0, + ) + + # needed for embedding all nodes across datasets (will use edge_index to split datasets) + all_data = pd.concat([train, valid]) + all_data = pd.concat([all_data, test]) + all_data.drop_duplicates(inplace=True) + + entity_id = pd.read_csv('data/FB15k/entities.txt', sep='\t', header=None, names=['entity', 'id'], engine='python') + entities = entity_id['entity'].values + + news_dataset_entities = np.concatenate([fn_test["fb_head"], fn_test["fb_tail"]]) + news_dataset_entities = np.unique(news_dataset_entities) + new_entities = np.array([ent for ent in news_dataset_entities if ent not in entities]) + entities = np.concatenate([entities, new_entities]) + entity_ids = torch.arange(0, len(entities), 1, dtype=torch.float) + + relation_id = pd.read_csv('data/FB15k/relations.txt', sep='\t', header=None, names=['relation', 'id'], engine='python') + relations = relation_id['relation'].values + + + # fit entity and relation encoder + le_entity = LabelEncoder() + le_entity.fit(entities) + le_relation = LabelEncoder() + le_relation.fit(relations) + + if not path.exists(embedding_path): + mkdir(embedding_path) + + np.save(path.join(embedding_path, "le_relation_classes_" + dataset_name + ".npy"), le_relation.classes_) + np.save(path.join(embedding_path, "le_entity_classes_" + dataset_name + ".npy"), le_entity.classes_) + + train = led.label_encode_dataset(le_entity, le_relation, train, entity_ids) + valid = led.label_encode_dataset(le_entity, le_relation, valid, entity_ids) + test = led.label_encode_dataset(le_entity, le_relation, test, entity_ids) + + all_data = led.label_encode_dataset(le_entity, le_relation, all_data, entity_ids) + news_dataset_x = torch.tensor(le_entity.transform(new_entities), dtype=torch.float) + all_data.x = torch.cat([all_data.x, news_dataset_x]) + print('all_data.x.size: ', all_data.x.size()) + # create node embeddings if none exists + cne.create_node_embedding( + all_data, dataset_name, dimensions=node_embedding_size, workers=4, path=embedding_path + ) + cre.create_relation_embedding(relations, le_relation, dataset_name, dimensions=hidden, path=embedding_path) + embedded_nodes = KeyedVectors.load_word2vec_format( + "{}/node_embedding_{}_{}.kv".format(embedding_path, dataset_name, str(node_embedding_size)) + ) + + embedded_relations = KeyedVectors.load_word2vec_format( + "{}/relation_embedding_le_{}_{}.bin".format(embedding_path, dataset_name, str(hidden)), + binary=True, + ) + # need to sort to get correct indexing + sorted_embedding = [] + for i in range(0, len(embedded_nodes.vectors)): + sorted_embedding.append(embedded_nodes.get_vector(str(i))) # error her + all_data.x = torch.tensor(sorted_embedding, dtype=torch.float) + + sorted_embedding = [] + for i in range(0, len(embedded_relations.vectors)): + sorted_embedding.append(embedded_relations.get_vector(str(i))) + embedded_relations = torch.tensor(sorted_embedding, dtype=torch.float) + + all_data.batch = torch.zeros((1, all_data.num_nodes), dtype=torch.int64).view(-1) + + num_features = all_data.x.shape[-1] + num_relations = len(np.unique(relations)) + + data = train_val_test_split(all_data, train, valid, test) + + data.edge_index = torch.cat( + [data.train_pos_edge_index, data.val_pos_edge_index, data.test_pos_edge_index], + dim=1, + ) + data.edge_type = torch.cat( + [train.edge_type, valid.edge_type, test.edge_type], dim=0 + ) + + return data, num_features, num_relations, embedded_relations + + +def main(_args): + print( + "\033[1;32m" + + "@@@@@@@@@@@@@@@@ Fake News Detection through Multi-Relational Graph Star @@@@@@@@@@@@@@@@" + + "\033[0m" + ) + args = gap.parser.parse_args(_args) + dataset_name = args.dataset + '_' + args.news_dataset + data, num_features, num_relations, embedded_relations = load_data( + hidden=args.hidden, kg_dataset=args.dataset, news_dataset=args.news_dataset, dataset_name=dataset_name + ) + + embedded_relations.to(args.device) + # python run_mr_lp.py --dropout=0 --hidden=128 --l2=5e-4 --num_layers=3 --cross_layer=False --patience=200 --residual=True --residual_star=True --dataset=FB15k_237 --device=cpu --epochs=2 + + trainer.trainer( + args, + dataset_name, + data, + num_features=num_features, + num_relations=num_relations, + relation_embeddings=embedded_relations, + num_epoch=args.epochs, + ) + + +if __name__ == "__main__": + main(sys.argv[1:]) diff --git a/trainer.py b/trainer.py index a1b6814..18d9e14 100644 --- a/trainer.py +++ b/trainer.py @@ -219,7 +219,7 @@ def trainer( scheduler.step(val_loss) scheduler.step(test_loss) - if epoch % save_per_epoch == 0: - torch.save(model, path.join(out_path, DATASET_NAME + ".pkl")) + if epoch == num_epoch or epoch % save_per_epoch == 0: + torch.save(model, path.join(out_path, DATASET_NAME + "_" + str(num_epoch) + 'epochs' + ".pkl")) tw.writer.close() diff --git a/utils/gsn_argparse.py b/utils/gsn_argparse.py index 42e9c72..f0e13f9 100644 --- a/utils/gsn_argparse.py +++ b/utils/gsn_argparse.py @@ -97,4 +97,5 @@ def tempDevice(x): "--relation_score_function", type=str, default="DistMult", help="DistMult" ) parser.add_argument("--dataset", type=str, default="FB15k_237") +parser.add_argument("--news_dataset", type=str, default="LIAR") parser.add_argument("--epochs", type=int, default=2000) diff --git a/utils/label_encode_dataset.py b/utils/label_encode_dataset.py index e9b9bbd..971a482 100644 --- a/utils/label_encode_dataset.py +++ b/utils/label_encode_dataset.py @@ -1,9 +1,8 @@ from torch_geometric.data import Data -import numpy as np import torch -def label_encode_dataset(le_entity, le_relation, data): +def label_encode_dataset(le_entity, le_relation, data, entity_ids): head = data["head"].values tail = data["tail"].values relations = data["relation"].values @@ -15,8 +14,5 @@ def label_encode_dataset(le_entity, le_relation, data): edge_attributes = torch.tensor(relations, dtype=torch.long) edge_index = torch.tensor([heads, tails], dtype=torch.long) - unique_entities = torch.tensor( - np.unique(edge_index.reshape(edge_index.shape[-1] * 2, 1)), dtype=torch.float - ) - return Data(x=unique_entities, edge_type=edge_attributes, edge_index=edge_index) + return Data(x=entity_ids, edge_type=edge_attributes, edge_index=edge_index) From 28310932d74feb3cd8e72e0cddf049ea1ec971a4 Mon Sep 17 00:00:00 2001 From: oscarvik Date: Thu, 22 Apr 2021 15:34:15 +0200 Subject: [PATCH 02/12] added dataset --- .gitignore | 1 + data/FakeNewsNet/train.csv | 1001 ++++++++++++++++++++++++++++++++++++ 2 files changed, 1002 insertions(+) create mode 100644 data/FakeNewsNet/train.csv diff --git a/.gitignore b/.gitignore index 3cce0b1..7519f64 100644 --- a/.gitignore +++ b/.gitignore @@ -138,6 +138,7 @@ output #Block data data/* !data/FB15k/ +!data/FakeNewsNet #Block embeddings embeddings/* diff --git a/data/FakeNewsNet/train.csv b/data/FakeNewsNet/train.csv new file mode 100644 index 0000000..aa2b1f6 --- /dev/null +++ b/data/FakeNewsNet/train.csv @@ -0,0 +1,1001 @@ +,text_id,head,relation,tail,fb_head,fb_tail,label +0,0,Bill Clinton,organization/role/leaders./organization/leadership/person,NPR,/m/0157m,/m/0c0sl,False +1,1,Bill Clinton,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0157m,/m/02mjmr,False +2,1,Marsha Blackburn,people/person/employment_history./business/employment_tenure/title,Rep,/m/01fnkt,/m/02qkv0r,False +3,1,Roger Bate,organization/role/leaders./organization/leadership/person,American Enterprise Institute,/m/0b23zg,/m/0p8q4,False +4,1,Hillary Clinton,people/person/employment_history./business/employment_tenure/title,Candidate,/m/0d06m5,/m/07r9r0,False +5,1,Rod Rosenstein,organization/role/leaders./organization/leadership/person,Rod Rosenstein,/m/04y7p0_,/m/04y7p0_,False +6,1,Rod Rosenstein,people/person/employment_history./business/employment_tenure/title,Attorney,/m/04y7p0_,/m/04gc2,False +7,1,Barack Obama,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0h7q0rq,/m/02mjmr,False +8,3,Barack Obama,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0h7q0rq,/m/02mjmr,False +9,3,HES,organization/role/leaders./organization/leadership/person,Getty+ImagesJoe+Raedle,HES,Getty+ImagesJoe+Raedle,False +10,5,Barack Obama,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0h7q0rq,/m/02mjmr,False +11,5,Barack Obama,organization/role/leaders./organization/leadership/person,United Nations,/m/0h7q0rq,/m/07t65,False +12,5,Barack Obama,people/person/employment_history./business/employment_tenure/title,Alex+Pfeiffer,/m/0h7q0rq,Alex+Pfeiffer,False +13,5,Barack Obama,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0h7q0rq,/m/02mjmr,False +14,8,Barack Obama,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0h7q0rq,/m/02mjmr,False +15,8,Bill Clinton,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0157m,/m/02mjmr,False +16,10,Barack Obama,organization/role/leaders./organization/leadership/person,Islamic State of Iraq and the Levant,/m/0h7q0rq,/m/0h7q0rq,False +17,11,Bashar al-Assad,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/01_vwx,/m/02mjmr,False +18,11,Nafeez Ahmed,people/person/employment_history./business/employment_tenure/title,Journalist,/m/0cb1tz,/m/0d8qb,False +19,11,Islamic State of Iraq and the Levant,organization/organization/headquarters./location/mailing_address/country,Iraq,/m/0h7q0rq,/m/0v74,False +20,13,United States Secret Service,organization/role/leaders./organization/leadership/person,Hillary Clinton,/m/0fynw,/m/0d06m5,False +21,13,Hillary Clinton,organization/role/leaders./organization/leadership/person,United States Secret Service,/m/0d06m5,/m/0fynw,False +22,14,Barack Obama,people/person/places_lived./people/place_lived/location,The West Wing,/m/0h7q0rq,/m/0g60z,False +23,14,Barack Obama,people/person/places_lived./people/place_lived/location,The West Wing,/m/0h7q0rq,/m/0g60z,False +24,14,United States Secret Service,organization/role/leaders./organization/leadership/person,Barack Obama,/m/0fynw,/m/0h7q0rq,False +25,14,Islamic State of Iraq and the Levant,organization/role/leaders./organization/leadership/person,Barack Obama,/m/0h7q0rq,/m/0h7q0rq,False +26,14,Jodie Foster,people/person/places_lived./people/place_lived/location,Nell,/m/0chw_,/m/03m7gdr,False +27,15,Barack Obama,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0h7q0rq,/m/02mjmr,False +28,15,Bill Clinton,people/person/employment_history./business/employment_tenure/title,Secretary,/m/0157m,/m/06slz5,False +29,16,Raymond T. Odierno,people/person/employment_history./business/employment_tenure/title,Chief of Staff of the United States Army,/m/0268g3w,/m/02g5lp,False +30,17,Ted Cruz,people/person/employment_history./business/employment_tenure/title,Senator,/m/07j6ty,/m/04c_3b,False +31,20,Michelle,people/person/employment_history./business/employment_tenure/title,First Lady of the United States,Michelle,/m/02xy5,False +32,23,Peter Allen,people/person/employment_history./business/employment_tenure/title,Sergeant,/m/01ndfjw,/m/01g2j2,False +33,24,Minnesota,organization/role/leaders./organization/leadership/person,Takbir,/m/04ykg,/m/017bdl,False +34,24,Dave Kleis,people/person/employment_history./business/employment_tenure/title,Mayor,/m/05f9m8h,/m/03x_sks,False +35,24,Antonio Adán,organization/role/leaders./organization/leadership/person,Minnesota,/m/0gsksn,/m/04ykg,False +36,24,Antonio Adán,organization/role/leaders./organization/leadership/person,Minnesota,/m/0gsksn,/m/04ykg,False +37,25,J. Christian Bollwage,people/person/religion,Christianity,/m/012bzwng,/m/01lp8,False +38,26,Andrew Puzder,people/person/employment_history./business/employment_tenure/title,Chief executive officer,/m/02wx77q,/m/02mwvv,False +39,26,Andrew Puzder,organization/role/leaders./organization/leadership/person,Business Insider,/m/02wx77q,/m/09gd5v1,False +40,26,Andrew Puzder,people/person/employment_history./business/employment_tenure/title,Chief executive officer,/m/02wx77q,/m/02mwvv,False +41,27,Huma Abedin,organization/role/leaders./organization/leadership/person,Federal Bureau of Investigation,/m/02x2tr8,/m/02_1m,False +42,27,James Comey,people/person/employment_history./business/employment_tenure/title,Director,/m/06r04p,/m/02mjmr,False +43,27,James Comey,organization/role/leaders./organization/leadership/person,Federal Bureau of Investigation,/m/06r04p,/m/02_1m,False +44,27,James Comey,people/person/employment_history./business/employment_tenure/title,Director,/m/06r04p,/m/02mjmr,False +45,27,Hillary Clinton,people/person/employment_history./business/employment_tenure/title,Chairperson,/m/0d06m5,/m/05_xks,False +46,27,United States Department of Justice,organization/role/leaders./organization/leadership/person,John Podesta,/m/0dtj5,/m/01zty6,False +47,27,Hillary Clinton,organization/role/leaders./organization/leadership/person,United States Department of Justice,/m/0d06m5,/m/0dtj5,False +48,27,Barack Obama,organization/role/leaders./organization/leadership/person,Federal Bureau of Investigation,/m/0h7q0rq,/m/02_1m,False +49,27,Barack Obama,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0h7q0rq,/m/02mjmr,False +50,27,Barack Obama,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0h7q0rq,/m/02mjmr,False +51,27,Obama.This,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/02mjmr,/m/02mjmr,False +52,27,White House,people/person/places_lived./people/place_lived/location,Hillary Clinton,/m/03mdbqx,/m/0d06m5,False +53,27,White House,people/person/places_lived./people/place_lived/location,Hillary Clinton,/m/03mdbqx,/m/0d06m5,False +54,27,Bill Clinton,organization/role/leaders./organization/leadership/person,Federal Bureau of Investigation,/m/0157m,/m/02_1m,False +55,28,Angela Merkel,people/person/employment_history./business/employment_tenure/title,Chancellor,/m/0jl0g,/m/01fch0,False +56,30,Jason+Falconer,people/person/employment_history./business/employment_tenure/title,Instructor,Jason+Falconer,Instructor,False +57,30,Falconer,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/01lk__r,/m/02mjmr,False +58,30,Falconer,organization/role/leaders./organization/leadership/person,Tactical+Advantage,/m/01lk__r,/m/0138n5_r,False +59,30,Bill Clinton,people/person/places_lived./people/place_lived/location,Protection of Lawful Commerce in Arms Act,/m/0157m,/m/08gb40,False +60,30,Bill Clinton,people/person/places_lived./people/place_lived/location,Protection of Lawful Commerce in Arms Act,/m/0157m,/m/08gb40,False +61,30,Hillary Clinton,people/person/employment_history./business/employment_tenure/title,Jason+Falconer,/m/0d06m5,Jason+Falconer,False +62,30,Hillary Clinton,people/person/employment_history./business/employment_tenure/title,Jason+Falconer,/m/0d06m5,Jason+Falconer,False +63,31,Huma Abedin,people/person/employment_history./business/employment_tenure/title,Google Assistant,/m/02x2tr8,Google Assistant,False +64,33,Bill,organization/role/leaders./organization/leadership/person,White House,/m/021583,/m/03mdbqx,False +65,35,TwitterBernard+Sansaricq,organization/role/leaders./organization/leadership/person,SHARES+Facebook,TwitterBernard+Sansaricq,SHARES+Facebook,False +66,35,TwitterBernard+Sansaricq,organization/role/leaders./organization/leadership/person,Senate,TwitterBernard+Sansaricq,/m/07t58,False +67,35,Bill Clinton,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0157m,/m/02mjmr,False +68,36,Barack Obama,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0h7q0rq,/m/02mjmr,False +69,38,Adam+Gitlin,organization/role/leaders./organization/leadership/person,Brennan Center for Justice,Adam+Gitlin,/m/02pjfvw,False +70,38,Brennan Center for Justice,organization/role/leaders./organization/leadership/person,Adam+Gitlin,/m/02pjfvw,Adam+Gitlin,False +71,41,Barack Obama,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0h7q0rq,/m/02mjmr,False +72,41,Barack Obama,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0h7q0rq,/m/02mjmr,False +73,42,Journalist,people/person/employment_history./business/employment_tenure/title,Tim Tebow,/m/0d8qb,/m/059yj,False +74,43,Islamic State of Iraq and the Levant,organization/organization/headquarters./location/mailing_address/country,Americas,/m/0h7q0rq,/m/07c5l,False +75,43,Héctor Garza,organization/role/leaders./organization/leadership/person,National+Border+Patrol+Council+Local+2455,/m/0685rw,National+Border+Patrol+Council+Local+2455,False +76,45,Pauline Hanson,people/person/employment_history./business/employment_tenure/title,Senator,/m/0j5pc,/m/04c_3b,False +77,47,County,location/hud_county_place/county,Luna,/m/094z8,/m/0czf4m8,False +78,49,Wilson Sporting Goods,people/person/employment_history./business/employment_tenure/title,Quarterback,/m/05sj2z,/m/06b1q,False +79,49,Wilson Sporting Goods,people/person/employment_history./business/employment_tenure/title,Quarterback,/m/05sj2z,/m/06b1q,False +80,49,Wilson Sporting Goods,people/person/employment_history./business/employment_tenure/title,Quarterback,/m/05sj2z,/m/06b1q,False +81,49,Wilson Sporting Goods,people/person/employment_history./business/employment_tenure/title,Quarterback,/m/05sj2z,/m/06b1q,False +82,49,Wilson Sporting Goods,people/person/employment_history./business/employment_tenure/title,Quarterback,/m/05sj2z,/m/06b1q,False +83,49,Wilson Sporting Goods,people/person/employment_history./business/employment_tenure/title,Quarterback,/m/05sj2z,/m/06b1q,False +84,49,Wilson Sporting Goods,people/person/employment_history./business/employment_tenure/title,Quarterback,/m/05sj2z,/m/06b1q,False +85,49,Wilson Sporting Goods,people/person/employment_history./business/employment_tenure/title,Quarterback,/m/05sj2z,/m/06b1q,False +86,50,Trump,organization/role/leaders./organization/leadership/person,Fox News,/m/0cqt90,/m/02z_b,False +87,50,Trump,organization/role/leaders./organization/leadership/person,Fox News,/m/0cqt90,/m/02z_b,False +88,51,Time,organization/role/leaders./organization/leadership/person,Donald Trump,/m/07c_l,/m/0cqt90,False +89,52,Frank Luntz,people/person/employment_history./business/employment_tenure/title,pollster,/m/0cxslf,pollster,False +90,52,Greg+Pollowitz,organization/role/leaders./organization/leadership/person,Drudge Report,Greg+Pollowitz,/m/01dgg6,False +91,53,Steven Knight,organization/role/leaders./organization/leadership/person,Charlotte-Mecklenburg Police Department,/m/04glwxr,/m/03d5lmj,False +92,53,Steven Knight,organization/role/leaders./organization/leadership/person,Charlotte-Mecklenburg Police Department,/m/04glwxr,/m/03d5lmj,False +93,53,Steven Knight,people/person/employment_history./business/employment_tenure/title,Shooting,/m/04glwxr,/m/071rm,False +94,53,Steven Knight,people/person/employment_history./business/employment_tenure/title,Shooting,/m/04glwxr,/m/071rm,False +95,53,Steven Knight,people/person/employment_history./business/employment_tenure/title,Minister,/m/04glwxr,/m/0377k9,False +96,53,Steven Knight,organization/role/leaders./organization/leadership/person,Missiongathering+Christian+Church,/m/04glwxr,Missiongathering+Christian+Church,False +97,54,Vladimir Putin,people/person/employment_history./business/employment_tenure/title,Russia,/m/08193,/m/06bnz,False +98,54,Alexander Lebedev,organization/role/leaders./organization/leadership/person,KGB,/m/027nvsp,/m/04b6p,False +99,54,Donald Trump,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0cqt90,/m/02mjmr,False +100,54,Donald Trump,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0cqt90,/m/02mjmr,False +101,54,Donald Trump,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0cqt90,/m/02mjmr,False +102,54,Donald Trump,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0cqt90,/m/02mjmr,False +103,56,Chris Christie,people/person/employment_history./business/employment_tenure/title,governor,/m/0f8t6k,/m/0btx2g,False +104,56,Vincent Prieto,people/person/employment_history./business/employment_tenure/title,Loudspeaker,/m/08cnqc,/m/0cfpc,False +105,56,Donald Trump,organization/role/leaders./organization/leadership/person,2020 Republican Party presidential primaries,/m/0cqt90,2020 Republican Party presidential primaries,False +106,56,Christie's,people/person/employment_history./business/employment_tenure/title,governor,/m/014df6,/m/0btx2g,False +107,57,Jeb Bush,people/person/employment_history./business/employment_tenure/title,governor,/m/019x9z,/m/0btx2g,False +108,57,Bill Clinton,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0157m,/m/02mjmr,False +109,57,Bill Clinton,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0157m,/m/02mjmr,False +110,57,George W. Bush,people/person/employment_history./business/employment_tenure/title,governor,/m/0d05fv,/m/0btx2g,False +111,60,Peter Thiel,organization/role/leaders./organization/leadership/person,Republican Party,/m/02w8m6,/m/085srz,False +112,60,Peter Thiel,people/person/employment_history./business/employment_tenure/title,Businessman,/m/02w8m6,/m/012t_z,False +113,60,Peter Thiel,organization/role/leaders./organization/leadership/person,Stanford University,/m/02w8m6,/m/06pwq,False +114,60,Patricia A. Shiu,organization/role/leaders./organization/leadership/person,Office of Federal Contract Compliance Programs,/m/09g6z41,/m/07gmmy,False +115,60,Lisa Gordon-Hagerty,people/person/employment_history./business/employment_tenure/title,Palantir Technologies,Lisa Gordon-Hagerty,/m/0bwkm08,False +116,68,Hillary Clinton,people/person/employment_history./business/employment_tenure/title,State.Well,/m/0d06m5,State.Well,False +117,68,Hillary Clinton,people/person/employment_history./business/employment_tenure/title,Secretary,/m/0d06m5,/m/06slz5,False +118,68,Bill Clinton,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0157m,/m/02mjmr,False +119,68,Hillary Clinton,people/person/employment_history./business/employment_tenure/title,Candidate,/m/0d06m5,/m/07r9r0,False +120,69,John Pfaff,organization/role/leaders./organization/leadership/person,Fordham University,John Pfaff,/m/027kp3,False +121,71,Rod Rosenstein,people/person/employment_history./business/employment_tenure/title,Attorney,/m/04y7p0_,/m/04gc2,False +122,71,Bill Clinton,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0157m,/m/02mjmr,False +123,71,Rod Rosenstein,organization/role/leaders./organization/leadership/person,Rod Rosenstein,/m/04y7p0_,/m/04y7p0_,False +124,71,Marsha Blackburn,people/person/employment_history./business/employment_tenure/title,Rep,/m/01fnkt,/m/02qkv0r,False +125,71,Roger Bate,organization/role/leaders./organization/leadership/person,American Enterprise Institute,/m/0b23zg,/m/0p8q4,False +126,72,Hillary Clinton,people/person/employment_history./business/employment_tenure/title,United States Secretary of State,/m/0d06m5,/m/0bwl7t0,False +127,72,Richard J. Leon,people/person/employment_history./business/employment_tenure/title,Judge,/m/0bydx6,/m/06lvrr,False +128,72,Cheryl Mills,people/person/employment_history./business/employment_tenure/title,Chief of staff,/m/026swz9,/m/025whr1,False +129,72,Huma Abedin,people/person/employment_history./business/employment_tenure/title,White House Deputy Chief of Staff,/m/02x2tr8,/m/0820qp,False +130,73,Jennifer Palmieri,people/person/employment_history./business/employment_tenure/title,Bill Clinton,/m/0120w9bh,/m/0157m,False +131,73,Jennifer Palmieri,people/person/employment_history./business/employment_tenure/title,Director of communications,/m/0120w9bh,/m/07g7bl,False +132,74,Bill Clinton,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0157m,/m/02mjmr,False +133,74,Kyle,people/person/employment_history./business/employment_tenure/title,Dolley Madison,Kyle,/m/0ptt0,False +134,77,George Lucas,people/person/employment_history./business/employment_tenure/title,Filmmaking,/m/0343h,/m/03r8gp,False +135,77,Juanita Irizarry,people/person/employment_history./business/employment_tenure/title,Friends of the Parks,Juanita Irizarry,/m/0cqf2_,False +136,78,Stanley A. McChrystal,people/person/employment_history./business/employment_tenure/title,General officer,/m/05557v4,/m/0130xz,False +137,78,United States Armed Forces,organization/organization/headquarters./location/mailing_address/country,Afghanistan,/m/07xg1,/m/0jdd,False +138,78,Stanley A. McChrystal,people/person/employment_history./business/employment_tenure/title,Commander,/m/05557v4,/m/01gzh0,False +139,78,Stanley A. McChrystal,people/person/employment_history./business/employment_tenure/title,General officer,/m/05557v4,/m/0130xz,False +140,81,Mark Ruffalo,people/person/employment_history./business/employment_tenure/title,Actor,/m/035rnz,/m/014g_s,False +141,81,MoveOn,organization/role/leaders./organization/leadership/person,John McCain,/m/0257nv,/m/0bymv,False +142,81,MoveOn,organization/role/leaders./organization/leadership/person,Donald Trump,/m/0257nv,/m/0cqt90,False +143,81,Parry O'Brien,people/person/employment_history./business/employment_tenure/title,peaceout.com,/m/07rlz7,peaceout.com,False +144,81,Parry O'Brien,people/person/employment_history./business/employment_tenure/title,Organizer,/m/07rlz7,Organizer,False +145,81,O'Brien,organization/role/leaders./organization/leadership/person,Veteran,O'Brien,/m/01wcxw,False +146,84,Pat McCrory,people/person/employment_history./business/employment_tenure/title,governor,/m/05tjbz,/m/0btx2g,False +147,85,James Comey,organization/role/leaders./organization/leadership/person,Federal Bureau of Investigation,/m/06r04p,/m/02_1m,False +148,85,James Comey,people/person/employment_history./business/employment_tenure/title,Film director,/m/06r04p,/m/02jknp,False +149,85,James Comey,people/person/employment_history./business/employment_tenure/title,Film director,/m/06r04p,/m/02jknp,False +150,85,James Comey,organization/role/leaders./organization/leadership/person,Federal Bureau of Investigation,/m/06r04p,/m/02_1m,False +151,85,Cheryl Mills,people/person/employment_history./business/employment_tenure/title,Chief of staff,/m/026swz9,/m/025whr1,False +152,85,Mills,organization/role/leaders./organization/leadership/person,Federal Bureau of Investigation,/m/045m16,/m/02_1m,False +153,85,Joseph diGenova,people/person/employment_history./business/employment_tenure/title,Attorney,Joseph diGenova,/m/04gc2,False +154,86,Islamic State of Iraq and the Levant,organization/organization/headquarters./location/mailing_address/country,Saudi Arabia,/m/0h7q0rq,/m/01z215,False +155,86,Islamic State of Iraq and the Levant,organization/organization/headquarters./location/mailing_address/country,Saudi Arabia,/m/0h7q0rq,/m/01z215,False +156,87,USA+Politics+Today,organization/role/leaders./organization/leadership/person,Bill Clinton,USA+Politics+Today,/m/0157m,False +157,87,USA+Politics+Today,organization/role/leaders./organization/leadership/person,Donald Trump,USA+Politics+Today,/m/0cqt90,False +158,90,Nicholas Merrill,people/person/employment_history./business/employment_tenure/title,Bill Clinton,/m/0dsdp74,/m/0157m,False +159,90,Nicholas Merrill,people/person/employment_history./business/employment_tenure/title,Spokesperson,/m/0dsdp74,/m/03mdbqx,False +160,91,Andrew Cuomo,people/person/employment_history./business/employment_tenure/title,governor,/m/02pjpd,/m/0btx2g,True +161,93,Barack Obama,organization/role/leaders./organization/leadership/person,United Nations,/m/0h7q0rq,/m/07t65,True +162,93,Barack Obama,organization/role/leaders./organization/leadership/person,United Nations General Assembly,/m/0h7q0rq,/m/07vp7,True +163,94,Barack Obama,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0h7q0rq,/m/02mjmr,True +164,95,Barack Obama,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0h7q0rq,/m/02mjmr,True +165,97,Hofstra University,organization/organization/headquarters./location/mailing_address/country,Island,/m/02607j,/m/03s0c,True +166,97,Bill Clinton,people/person/employment_history./business/employment_tenure/title,Secretary,/m/0157m,/m/06slz5,True +167,97,Bill Clinton,people/person/employment_history./business/employment_tenure/title,Secretary,/m/0157m,/m/06slz5,True +168,97,Barack Obama,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0h7q0rq,/m/02mjmr,True +169,97,Bill Clinton,people/person/employment_history./business/employment_tenure/title,Secretary,/m/0157m,/m/06slz5,True +170,97,Barack Obama,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0h7q0rq,/m/02mjmr,True +171,97,Donald,people/person/employment_history./business/employment_tenure/title,Barack Obama,Donald,/m/0h7q0rq,True +172,97,Donald,people/person/employment_history./business/employment_tenure/title,Barack Obama,Donald,/m/0h7q0rq,True +173,97,Barack Obama,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0h7q0rq,/m/02mjmr,True +174,97,Barack Obama,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0h7q0rq,/m/02mjmr,True +175,97,Stern,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/03jmnt,/m/02mjmr,True +176,98,Jennifer Roberts,people/person/employment_history./business/employment_tenure/title,Mayor,Jennifer Roberts,/m/03x_sks,True +177,98,Putney,organization/organization/headquarters./location/mailing_address/country,ABC,/m/0nb_v,/m/0gsg7,True +178,98,Putney,organization/organization/headquarters./location/mailing_address/country,ABC,/m/0nb_v,/m/0gsg7,True +179,100,Robby Mook,people/person/employment_history./business/employment_tenure/title,Bill Clinton,/m/0y4zr4f,/m/0157m,True +180,100,Robby Mook,people/person/employment_history./business/employment_tenure/title,Campaign manager,/m/0y4zr4f,/m/03xk7t,True +181,100,Robby Mook,organization/role/leaders./organization/leadership/person,CBS,/m/0y4zr4f,/m/09d5h,True +182,100,Kellyanne Conway,people/person/employment_history./business/employment_tenure/title,Campaign manager,/m/0dq819,/m/03xk7t,True +183,100,Kellyanne Conway,people/person/employment_history./business/employment_tenure/title,Donald Trump,/m/0dq819,/m/0cqt90,True +184,100,Bill Clinton,organization/role/leaders./organization/leadership/person,Republican Party,/m/0157m,/m/085srz,True +185,100,Barack Obama,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0h7q0rq,/m/02mjmr,True +186,100,Matt Lauer,organization/role/leaders./organization/leadership/person,Lester Holt,/m/049_zz,/m/06jvj7,True +187,100,Brian Fallon,people/person/employment_history./business/employment_tenure/title,White House Press Secretary,/m/0fqmphx,/m/01pt0z,True +188,100,Corey Lewandowski,people/person/employment_history./business/employment_tenure/title,Donald Trump,Corey Lewandowski,/m/0cqt90,True +189,100,Corey Lewandowski,people/person/employment_history./business/employment_tenure/title,Campaign manager,Corey Lewandowski,/m/03xk7t,True +190,100,Mook,organization/role/leaders./organization/leadership/person,NBCs+Today.Asked,Mook,NBCs+Today.Asked,True +191,101,John Anderson,people/person/employment_history./business/employment_tenure/title,Candidate,/m/0b9ndp,/m/07r9r0,True +192,101,Ross Perot,people/person/religion,The Independent,/m/0bxhx,/m/0q4tw,True +193,104,Andrew Cuomo,people/person/employment_history./business/employment_tenure/title,governor,/m/02pjpd,/m/0btx2g,True +194,105,Hillary Clinton,people/person/employment_history./business/employment_tenure/title,Candidate,/m/0d06m5,/m/07r9r0,True +195,105,Harry Reid,organization/role/leaders./organization/leadership/person,List of current United States senators,/m/0204x1,List of current United States senators,True +196,105,Mitt Romney,organization/role/leaders./organization/leadership/person,Republican Party,/m/05k7sb,/m/085srz,True +197,105,Harry Reid,people/person/employment_history./business/employment_tenure/title,Leadership,/m/0204x1,/m/0h7q0rq,True +198,105,Gerald Ford,people/person/places_lived./people/place_lived/location,Oval Office,/m/0c_md_,/m/06c0j,True +199,105,Donald Trump,organization/role/leaders./organization/leadership/person,Republican Party,/m/0cqt90,/m/085srz,True +200,105,George W. Romney,organization/role/leaders./organization/leadership/person,Mitt Romney,/m/01v6nf,/m/05k7sb,True +201,106,Sander Levin,organization/role/leaders./organization/leadership/person,CNN,/m/02n8p2,/m/0gsgr,True +202,106,The Washington Post,organization/role/leaders./organization/leadership/person,John Koskinen,/m/0px38,/m/06v2j5,True +203,107,Dean+Washington,people/person/employment_history./business/employment_tenure/title,CPL,/m/0wbh29p,CPL,True +204,107,Dean+Washington,people/person/places_lived./people/place_lived/location,"Gwinnett County, Georgia",/m/0wbh29p,/m/0nzlp,True +205,107,Dean+Washington,people/person/employment_history./business/employment_tenure/title,"Gwinnett County, Georgia",/m/0wbh29p,/m/0nzlp,True +206,108,Donald Trump,organization/role/leaders./organization/leadership/person,Fox News,/m/0cqt90,/m/02z_b,True +207,108,Ainsley Earhardt,organization/role/leaders./organization/leadership/person,Fox News,/m/02qlgwq,/m/02z_b,True +208,109,Jennifer Roberts,people/person/employment_history./business/employment_tenure/title,Mayor,Jennifer Roberts,/m/03x_sks,True +209,112,Ted Cruz,people/person/employment_history./business/employment_tenure/title,John+F.+Kennedy+Cruz,/m/07j6ty,John+F.+Kennedy+Cruz,True +210,112,John F. Kennedy,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0d3k14,/m/02mjmr,True +211,113,John King,people/person/employment_history./business/employment_tenure/title,Correspondent,/m/0b1qpc,/m/02k13d,True +212,113,"Wake County, North Carolina",people/person/places_lived./people/place_lived/location,Bill Clinton,/m/0n3ll,/m/0157m,True +213,115,Bill Clinton,organization/role/leaders./organization/leadership/person,Associated Press,/m/0157m,/m/0cv_2,True +214,115,Bill Clinton,organization/role/leaders./organization/leadership/person,Associated Press,/m/0157m,/m/0cv_2,True +215,115,Barack Obama,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0h7q0rq,/m/02mjmr,True +216,115,Hillary Clinton,organization/role/leaders./organization/leadership/person,Twitter,/m/0d06m5,/m/0289n8t,True +217,116,John F. Kennedy,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0d3k14,/m/02mjmr,True +218,116,Washington,people/person/employment_history./business/employment_tenure/title,John F. Kennedy,/m/081yw,/m/0d3k14,True +219,116,John F. Kennedy,people/person/employment_history./business/employment_tenure/title,Robert F. Kennedy,/m/0d3k14,/m/06hx2,True +220,116,Robert F. Kennedy,people/person/employment_history./business/employment_tenure/title,Attorney general,/m/06hx2,/m/0465sdr,True +221,116,Vladimir Putin,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/08193,/m/02mjmr,True +222,116,Vladimir Putin,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/08193,/m/02mjmr,True +223,116,Nikita Khrushchev,people/person/employment_history./business/employment_tenure/title,Leadership,/m/0bphp,/m/0h7q0rq,True +224,116,John F. Kennedy,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0d3k14,/m/02mjmr,True +225,116,Lyndon B. Johnson,people/person/employment_history./business/employment_tenure/title,War,/m/0f7fy,/m/02mjmr,True +226,116,Lyndon B. Johnson,people/person/employment_history./business/employment_tenure/title,Vietnam,/m/0f7fy,/m/01crd5,True +227,116,Lyndon B. Johnson,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0f7fy,/m/02mjmr,True +228,116,Tulsi Gabbard,organization/role/leaders./organization/leadership/person,United States Congress,/m/0cnyrfq,/m/0d06m5,True +229,116,Sea,location/country/languages_spoken,South,/m/06npx,/m/0k7ny,True +230,116,Sea,location/country/languages_spoken,China,/m/06npx,/m/0d05w3,True +231,116,Vladimir Putin,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/08193,/m/02mjmr,True +232,116,William Perry,people/person/employment_history./business/employment_tenure/title,Europe,/m/013y1y,/m/02j9z,True +233,116,William Perry,people/person/employment_history./business/employment_tenure/title,United States Secretary of Defense,/m/013y1y,/m/0c01v,True +234,118,Bryce+Heinlein,people/person/employment_history./business/employment_tenure/title,Sergeant,Bryce+Heinlein,/m/01g2j2,True +235,119,Colin Kaepernick,people/person/employment_history./business/employment_tenure/title,Quarterback,/m/03cm6b3,/m/06b1q,True +236,119,Keith Urban,organization/role/leaders./organization/leadership/person,ESPN,/m/05cljf,/m/0kc6x,True +237,119,Keith Urban,organization/role/leaders./organization/leadership/person,ESPN,/m/05cljf,/m/0kc6x,True +238,120,Linda Thompson,people/person/employment_history./business/employment_tenure/title,Actor,/m/05yz6j,/m/014g_s,True +239,121,Donna Brazile,organization/role/leaders./organization/leadership/person,Associated Press,/m/03n1q0,/m/0cv_2,True +240,121,Donna Brazile,people/person/employment_history./business/employment_tenure/title,Chairperson,/m/03n1q0,/m/05_xks,True +241,121,Donna Brazile,organization/role/leaders./organization/leadership/person,Associated Press,/m/03n1q0,/m/0cv_2,True +242,122,Ministry of Defence,organization/organization/headquarters./location/mailing_address/country,United Kingdom,/m/01cy4y,/m/07ssc,True +243,126,Joe Biden,people/person/employment_history./business/employment_tenure/title,Vice President of the United States,/m/012gx2,/m/0d05fv,True +244,126,Ronald Reagan,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/06c0j,/m/02mjmr,True +245,127,Jill Stein,people/person/employment_history./business/employment_tenure/title,Candidate,/m/05st_r,/m/07r9r0,True +246,127,Jill Stein,organization/role/leaders./organization/leadership/person,Green Party of the United States,/m/05st_r,/m/07k5l,True +247,128,Evelyn+Arnold,people/person/employment_history./business/employment_tenure/title,Principal,Evelyn+Arnold,/m/07s82n4,True +248,128,W. Shannon Morris,people/person/employment_history./business/employment_tenure/title,Principal,W. Shannon Morris,/m/07s82n4,True +249,132,Ahmad+Khan+RahamiHall,organization/role/leaders./organization/leadership/person,New York New Jersey Rail,Ahmad+Khan+RahamiHall,/m/06v0py,True +250,133,Hillary Clinton,people/person/employment_history./business/employment_tenure/title,Campaign manager,/m/0d06m5,/m/03xk7t,True +251,133,Hillary Clinton,organization/role/leaders./organization/leadership/person,CNN,/m/0d06m5,/m/0gsgr,True +252,133,Steve Harvey,people/person/employment_history./business/employment_tenure/title,Host,/m/02lhtf,/m/04wk4z,True +253,134,Kellyanne Conway,people/person/employment_history./business/employment_tenure/title,Donald Trump,/m/0dq819,/m/0cqt90,True +254,134,Kellyanne Conway,organization/role/leaders./organization/leadership/person,Islamic State of Iraq and the Levant,/m/0dq819,/m/0h7q0rq,True +255,134,Kellyanne Conway,people/person/employment_history./business/employment_tenure/title,Kellyanne Conway,/m/0dq819,/m/0dq819,True +256,134,Kellyanne Conway,people/person/employment_history./business/employment_tenure/title,Campaign manager,/m/0dq819,/m/03xk7t,True +257,135,Barack Obama,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0h7q0rq,/m/02mjmr,True +258,135,CNN,organization/organization/headquarters./location/mailing_address/country,New Jersey,/m/0gsgr,/m/0f8t6k,True +259,136,Barack Obama,organization/role/leaders./organization/leadership/person,United+Nations+CNN,/m/0h7q0rq,United+Nations+CNN,True +260,136,Barack Obama,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0h7q0rq,/m/02mjmr,True +261,136,Barack Obama,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0h7q0rq,/m/02mjmr,True +262,136,Barack Obama,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0h7q0rq,/m/02mjmr,True +263,136,Donald Tusk,organization/role/leaders./organization/leadership/person,European Council,/m/02_fg6,/m/02m1b,True +264,136,John Kerry,people/person/employment_history./business/employment_tenure/title,United States Secretary of State,/m/0d3qd0,/m/0bwl7t0,True +265,136,Margaret+Huang,organization/role/leaders./organization/leadership/person,Amnesty+Internationals+Interim+Executive,Margaret+Huang,Amnesty+Internationals+Interim+Executive,True +266,136,Donald Tusk,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/02_fg6,/m/02mjmr,True +267,137,Hillary Clinton,people/person/religion,Islam,/m/0d06m5,/m/0flw86,True +268,138,Jay Rosen,organization/role/leaders./organization/leadership/person,New York University,/m/0b2xfq,/m/0bwfn,True +269,138,Jay Rosen,people/person/employment_history./business/employment_tenure/title,Professor,/m/0b2xfq,/m/016fly,True +270,138,Hayreddin Barbarossa,organization/role/leaders./organization/leadership/person,Peter Beinart,/m/0dkr2r,/m/04tpcx,True +271,138,Barry Marshall,people/person/employment_history./business/employment_tenure/title,Doctor,/m/023fbk,/m/02g9n,True +272,138,Alan Greenspan,organization/role/leaders./organization/leadership/person,Federal Reserve Board of Governors,/m/015h4y,/m/0hhqj3p,True +273,138,Michael Oreskes,organization/role/leaders./organization/leadership/person,NPR+News,Michael Oreskes,NPR+News,True +274,138,Michael Oreskes,people/person/employment_history./business/employment_tenure/title,Film director,Michael Oreskes,/m/02jknp,True +275,139,David Clarke,people/person/employment_history./business/employment_tenure/title,Sheriff,/m/0_yglzq,/m/0_yglzq,True +276,139,County,location/hud_county_place/county,Milwaukee,/m/094z8,/m/094z8,True +277,139,Erik+J.+Heipt,people/person/employment_history./business/employment_tenure/title,Attorney,Erik+J.+Heipt,/m/04gc2,True +278,140,Donald Trump,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0cqt90,/m/02mjmr,True +279,141,Donald Trump,organization/role/leaders./organization/leadership/person,AP+Photo+Trump,/m/0cqt90,AP+Photo+Trump,True +280,141,Kellyanne Conway,people/person/employment_history./business/employment_tenure/title,Campaign manager,/m/0dq819,/m/03xk7t,True +281,141,Donald Trump,organization/role/leaders./organization/leadership/person,CNN,/m/0cqt90,/m/0gsgr,True +282,142,François Hollande,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/02qg4z,/m/02mjmr,True +283,142,Bates,people/person/employment_history./business/employment_tenure/title,Pack2Go+Europe,/m/05y6yms,Pack2Go+Europe,True +284,144,Kerr+Putney,people/person/employment_history./business/employment_tenure/title,Chief of police,Kerr+Putney,/m/039h8_,True +285,145,Mark Kelly,people/person/employment_history./business/employment_tenure/title,Astronaut,/m/02wsxy,/m/0htp,True +286,146,President of the United States,people/person/employment_history./business/employment_tenure/title,Donald Trump,/m/02mjmr,/m/0cqt90,True +287,147,Donald Trump,organization/role/leaders./organization/leadership/person,Fox News,/m/0cqt90,/m/02z_b,True +288,147,Donald Trump,organization/role/leaders./organization/leadership/person,Fox News,/m/0cqt90,/m/02z_b,True +289,147,Barack Obama,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0h7q0rq,/m/02mjmr,True +290,152,Gabriel Gorenstein,people/person/employment_history./business/employment_tenure/title,United States magistrate judge,Gabriel Gorenstein,/m/05b105r,True +291,153,Joseph Dunford,people/person/employment_history./business/employment_tenure/title,Gene,/m/05b2r22,/m/0bscct,True +292,153,Joint Chiefs of Staff,organization/role/leaders./organization/leadership/person,Joseph Dunford,/m/01brfl,/m/05b2r22,True +293,155,Barack Obama,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0h7q0rq,/m/02mjmr,True +294,155,"Charles, Prince of Wales",people/person/places_lived./people/place_lived/location,Wales,/m/0xnc3,/m/0j5g9,True +295,156,Gabriel Gorenstein,people/person/employment_history./business/employment_tenure/title,United States magistrate judge,Gabriel Gorenstein,/m/05b105r,True +296,157,Trent Franks,organization/role/leaders./organization/leadership/person,RepublicanWashington+CNN,/m/024p5b,RepublicanWashington+CNN,True +297,158,AP+Photo+Trump,organization/role/leaders./organization/leadership/person,Bill Clinton,AP+Photo+Trump,/m/0157m,True +298,158,Barack Obama,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0h7q0rq,/m/02mjmr,True +299,159,Monday.David+Wildstein,organization/role/leaders./organization/leadership/person,Interstate+Capital+Projects,Monday.David+Wildstein,Interstate+Capital+Projects,True +300,159,Bridget Kelly,people/person/employment_history./business/employment_tenure/title,White House Deputy Chief of Staff,/m/0803htk,/m/0820qp,True +301,159,Chris Christie,people/person/employment_history./business/employment_tenure/title,governor,/m/0f8t6k,/m/0btx2g,True +302,159,Bob+Durando,people/person/employment_history./business/employment_tenure/title,Contract bridge,Bob+Durando,/m/019hy,True +303,160,Bono,organization/role/leaders./organization/leadership/person,CBS,/m/01vswwx,/m/09d5h,True +304,161,Susan+MacManus,organization/role/leaders./organization/leadership/person,University of South Florida,Susan+MacManus,/m/09s5q8,True +305,161,Susan+MacManus,people/person/employment_history./business/employment_tenure/title,Political science,Susan+MacManus,/m/062z7,True +306,163,Tim Kaine,people/person/employment_history./business/employment_tenure/title,Candidate,/m/053f8h,/m/07r9r0,True +307,164,Donald Trump,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0cqt90,/m/02mjmr,True +308,165,Neema+Hakim,people/person/employment_history./business/employment_tenure/title,Spokesperson,Neema+Hakim,/m/03mdbqx,True +309,165,John P. Roth,people/person/employment_history./business/employment_tenure/title,Inspector general,John P. Roth,/m/0kvg94,True +310,166,J. Paul Austin,organization/role/leaders./organization/leadership/person,Tim Baker,/m/0j42f1z,Tim Baker,True +311,166,B.J. Nikkel,organization/role/leaders./organization/leadership/person,Josh Penry,/m/05c3rfn,/m/02q03dy,True +312,168,Donald Trump,people/person/employment_history./business/employment_tenure/title,Night,/m/0cqt90,/m/01d74z,True +313,168,Donald Trump,people/person/employment_history./business/employment_tenure/title,Comedian,/m/0cqt90,/m/018gz8,True +314,168,Barack Obama,people/person/employment_history./business/employment_tenure/title,United States,/m/0h7q0rq,/m/07t58,True +315,168,Barack Obama,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0h7q0rq,/m/02mjmr,True +316,169,David Martosko,organization/role/leaders./organization/leadership/person,Daily Mail,David Martosko,/m/0180v2,True +317,169,Jeremy Diamond,people/person/employment_history./business/employment_tenure/title,Journalist,Jeremy Diamond,/m/0d8qb,True +318,169,Jeremy Diamond,organization/role/leaders./organization/leadership/person,CNN,Jeremy Diamond,/m/0gsgr,True +319,169,Ron+Bonjean,people/person/employment_history./business/employment_tenure/title,Strategist,Ron+Bonjean,/m/03gq32j,True +320,171,ABC+News+Rahami,organization/organization/headquarters./location/mailing_address/country,Us,ABC+News+Rahami,/m/0b3wk,True +321,171,Cindy+Galli,organization/role/leaders./organization/leadership/person,2016 New York and New Jersey bombings,Cindy+Galli,2016 New York and New Jersey bombings,True +322,172,Donald Trump,people/person/employment_history./business/employment_tenure/title,Candidate,/m/0cqt90,/m/07r9r0,True +323,172,Donald Trump,people/person/employment_history./business/employment_tenure/title,Gladiator,/m/0cqt90,/m/037l9,True +324,172,King,people/person/employment_history./business/employment_tenure/title,Donald Trump,/m/03w9bnr,/m/0cqt90,True +325,172,King,people/person/employment_history./business/employment_tenure/title,Candidate,/m/03w9bnr,/m/07r9r0,True +326,172,John Lewis,people/person/employment_history./business/employment_tenure/title,Rep,/m/01pjq21,/m/02qkv0r,True +327,172,Michael Cohen,people/person/employment_history./business/employment_tenure/title,Attorney,Michael Cohen,/m/04gc2,True +328,172,Michael Flynn,people/person/employment_history./business/employment_tenure/title,Lieutenant general,/m/09v12g0,/m/01cpjg,True +329,173,Donald Trump,organization/role/leaders./organization/leadership/person,Barack Obama,/m/0cqt90,/m/0h7q0rq,True +330,173,Barack Obama,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0h7q0rq,/m/02mjmr,True +331,173,Getty+Joe+Arpaio+Trump,people/person/places_lived./people/place_lived/location,USI,Getty+Joe+Arpaio+Trump,USI,True +332,174,Barack Obama,organization/role/leaders./organization/leadership/person,CNN,/m/0h7q0rq,/m/0gsgr,True +333,174,Benjamin Netanyahu,people/person/employment_history./business/employment_tenure/title,Prime Minister of the United Kingdom,/m/0fm2h,/m/060s9,True +334,174,Barack Obama,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0h7q0rq,/m/02mjmr,True +335,175,Terry McAuliffe,people/person/employment_history./business/employment_tenure/title,governor,/m/02dgtb,/m/0btx2g,True +336,175,Terry McAuliffe,people/person/employment_history./business/employment_tenure/title,governor,/m/02dgtb,/m/0btx2g,True +337,176,Vladimir Putin,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/08193,/m/02mjmr,True +338,176,Donald Trump,people/person/nationality,Quds Force,/m/0cqt90,/m/0bd931,True +339,176,Hugh Hewitt,people/person/employment_history./business/employment_tenure/title,Host,/m/042f3_,/m/04wk4z,True +340,176,Donald Trump,organization/role/leaders./organization/leadership/person,Quds Force,/m/0cqt90,/m/0bd931,True +341,176,Mikhail Leontyev,organization/role/leaders./organization/leadership/person,Rosneft,/m/0b7rm2,/m/04s4_v,True +342,176,Igor+Sechinduring,organization/role/leaders./organization/leadership/person,Rosneft,Igor+Sechinduring,/m/04s4_v,True +343,176,Donald Trump,organization/role/leaders./organization/leadership/person,Donald Trump,/m/0cqt90,/m/0cqt90,True +344,176,Barack Obama,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0h7q0rq,/m/02mjmr,True +345,176,Ian Craig,organization/role/leaders./organization/leadership/person,Sakhalin Energy,/m/061xw_,/m/03cnqkd,True +346,176,Craig,organization/role/leaders./organization/leadership/person,Merrill,Craig,/m/01mlrk,True +347,176,Sergei+Aleksashenko,people/person/employment_history./business/employment_tenure/title,Chairperson,Sergei+Aleksashenko,/m/05_xks,True +348,176,Sergei+Aleksashenko,organization/role/leaders./organization/leadership/person,Central Bank of Russia,Sergei+Aleksashenko,/m/0382np,True +349,176,Shooting of Michael Brown,organization/role/leaders./organization/leadership/person,Global Policy,/m/011l2bwd,/m/0bmgs01,True +350,176,Ivanov,organization/role/leaders./organization/leadership/person,Federal Bureau of Investigation,/m/05h3wxl,/m/02_1m,True +351,176,Ivanov,organization/role/leaders./organization/leadership/person,Federal Bureau of Investigation,/m/05h3wxl,/m/02_1m,True +352,176,Petr Aven,organization/role/leaders./organization/leadership/person,Alfa-Bank,/m/02849v5,/m/037ppp,True +353,176,Ivanov,organization/role/leaders./organization/leadership/person,New Economic School,/m/05h3wxl,/m/0bhp2c,True +354,176,Aleksei N. Leontiev,organization/role/leaders./organization/leadership/person,Rosneft,/m/02ybc5,/m/04s4_v,True +355,176,James Comey,people/person/employment_history./business/employment_tenure/title,Film director,/m/06r04p,/m/02jknp,True +356,176,James Comey,organization/role/leaders./organization/leadership/person,Federal Bureau of Investigation,/m/06r04p,/m/02_1m,True +357,176,Richard Burt,people/person/employment_history./business/employment_tenure/title,Germany,/m/09gd0s0,/m/0345h,True +358,176,Richard Burt,organization/role/leaders./organization/leadership/person,Alfa-Bank,/m/09gd0s0,/m/037ppp,True +359,176,Richard Burt,organization/role/leaders./organization/leadership/person,Alfa-Bank,/m/09gd0s0,/m/037ppp,True +360,176,Jeff Sessions,people/person/employment_history./business/employment_tenure/title,Senator,/m/020yjg,/m/04c_3b,True +361,176,Rick Dearborn,people/person/employment_history./business/employment_tenure/title,Chief of staff,Rick Dearborn,/m/025whr1,True +362,176,HES,organization/role/leaders./organization/leadership/person,Donald Trump,HES,/m/0cqt90,True +363,176,Narendra Modi,people/person/employment_history./business/employment_tenure/title,Prime Minister of the United Kingdom,/m/0296q2,/m/060s9,True +364,176,Michael McFaul,people/person/employment_history./business/employment_tenure/title,Russia,/m/02qwnj2,/m/06bnz,True +365,176,Carter Page,people/person/employment_history./business/employment_tenure/title,Candidate,Carter Page,/m/07r9r0,True +366,176,Carter Page,people/person/employment_history./business/employment_tenure/title,Candidate,Carter Page,/m/07r9r0,True +367,177,The Washington Post,organization/role/leaders./organization/leadership/person,Donald Trump,/m/0px38,/m/0cqt90,True +368,177,Shalom Auslander,people/person/employment_history./business/employment_tenure/title,Columnist,/m/02z2l74,/m/028fjr,True +369,178,Barack Obama,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0h7q0rq,/m/02mjmr,True +370,178,Daniel Pfeiffer,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/05c2_fd,/m/02mjmr,True +371,178,Barack Obama,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0h7q0rq,/m/02mjmr,True +372,178,Josh Earnest,people/person/employment_history./business/employment_tenure/title,White House,/m/0l8ps43,/m/03mdbqx,True +373,178,Josh Earnest,people/person/employment_history./business/employment_tenure/title,White House Press Secretary,/m/0l8ps43,/m/01pt0z,True +374,179,Senator,people/person/employment_history./business/employment_tenure/title,Donald Trump,/m/04c_3b,/m/0cqt90,True +375,179,Senator,people/person/employment_history./business/employment_tenure/title,Donald Trump,/m/04c_3b,/m/0cqt90,True +376,181,Donald Trump,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0cqt90,/m/02mjmr,True +377,181,Donald Trump,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0cqt90,/m/02mjmr,True +378,181,Candidate,people/person/employment_history./business/employment_tenure/title,Hillary Clinton,/m/07r9r0,/m/0d06m5,True +379,181,Donald Trump,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0cqt90,/m/02mjmr,True +380,181,Donald Trump,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0cqt90,/m/02mjmr,True +381,181,Donald Trump,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0cqt90,/m/02mjmr,True +382,181,Donald Trump,people/person/employment_history./business/employment_tenure/title,United States,/m/0cqt90,/m/07t58,True +383,181,Bill Clinton,people/person/employment_history./business/employment_tenure/title,Candidate,/m/0157m,/m/07r9r0,True +384,181,Peter Baker,people/person/employment_history./business/employment_tenure/title,List of biographers,Peter Baker,List of biographers,True +385,181,Barack Obama,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0h7q0rq,/m/02mjmr,True +386,181,Blake Farenthold,people/person/employment_history./business/employment_tenure/title,Member of Congress,/m/0ds2ztj,/m/04l8cr,True +387,181,Paul Ryan,people/person/employment_history./business/employment_tenure/title,United States Speaker of the House,/m/024v2j,United States Speaker of the House,True +388,181,David Fahrenthold,organization/role/leaders./organization/leadership/person,The Washington Post,David Fahrenthold,/m/0px38,True +389,181,David Fahrenthold,people/person/employment_history./business/employment_tenure/title,Journalist,David Fahrenthold,/m/0d8qb,True +390,183,Hillary Clinton,people/person/employment_history./business/employment_tenure/title,Candidate,/m/0d06m5,/m/07r9r0,False +391,183,Barack Obama,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0h7q0rq,/m/02mjmr,False +392,183,United States Electoral College,organization/role/leaders./organization/leadership/person,Hillary Clinton,/m/0lq3t,/m/0d06m5,False +393,185,Saul+Florez,people/person/employment_history./business/employment_tenure/title,safety+manager,Saul+Florez,/m/0j28ksx,False +394,186,Donald Trump,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0cqt90,/m/02mjmr,False +395,186,Andrew McCabe,people/person/employment_history./business/employment_tenure/title,Deputy Director,Andrew McCabe,Deputy Director,False +396,186,Trey Gowdy,people/person/employment_history./business/employment_tenure/title,Rep,/m/0c3x3rz,/m/02qkv0r,False +397,186,Andrew McCabe,organization/role/leaders./organization/leadership/person,Federal Bureau of Investigation,Andrew McCabe,/m/02_1m,False +398,187,Donald Trump,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0cqt90,/m/02mjmr,False +399,187,Donald Trump,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0cqt90,/m/02mjmr,False +400,188,Minneapolis+Department+of+Civil+Rights,organization/role/leaders./organization/leadership/person,Velma+Korbel,Minneapolis+Department+of+Civil+Rights,Velma+Korbel,False +401,188,Velma+Korbel,organization/role/leaders./organization/leadership/person,Minneapolis+Department+of+Civil+Rights,Velma+Korbel,Minneapolis+Department+of+Civil+Rights,False +402,188,Velma+Korbel,organization/role/leaders./organization/leadership/person,Film director,Velma+Korbel,/m/02jknp,False +403,188,Minneapolis+Department+of+Civil+Rights,organization/role/leaders./organization/leadership/person,Velma+Korbel,Minneapolis+Department+of+Civil+Rights,Velma+Korbel,False +404,188,Donald Trump,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0cqt90,/m/02mjmr,False +405,188,Velma+Korbel,organization/role/leaders./organization/leadership/person,Minneapolis+Department+of+Civil+Rights,Velma+Korbel,Minneapolis+Department+of+Civil+Rights,False +406,188,Velma+Korbel,organization/role/leaders./organization/leadership/person,Film director,Velma+Korbel,/m/02jknp,False +407,190,Paul Ryan,people/person/employment_history./business/employment_tenure/title,United States Speaker of the House,/m/024v2j,United States Speaker of the House,False +408,190,Vladimir Putin,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/08193,/m/02mjmr,False +409,190,Thomas Downey,people/person/employment_history./business/employment_tenure/title,Analyst,/m/0cn4c3,/m/0h1vgc,False +410,190,Mike Pence,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/022r9r,/m/02mjmr,False +411,190,Mike Pence,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/022r9r,/m/02mjmr,False +412,191,Australia,organization/organization/headquarters./location/mailing_address/country,NBC+news,/m/0chghy,/m/016bvz,False +413,191,Australia,organization/organization/headquarters./location/mailing_address/country,NBC+news,/m/0chghy,/m/016bvz,False +414,191,Amal,people/person/employment_history./business/employment_tenure/title,The Terminator,/m/02p3gzs,/m/07ghq,False +415,191,Amal,people/person/employment_history./business/employment_tenure/title,The Terminator,/m/02p3gzs,/m/07ghq,False +416,192,Sean Spicer,organization/role/leaders./organization/leadership/person,White+House+Press,Sean Spicer,/m/02r4jz,False +417,192,Donald Trump,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0cqt90,/m/02mjmr,False +418,192,Donald Trump,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0cqt90,/m/02mjmr,False +419,192,Supreme Court of the United States,organization/organization/headquarters./location/mailing_address/country,United States,/m/0hx5v,/m/07t58,False +420,193,James Comey,people/person/employment_history./business/employment_tenure/title,Film director,/m/06r04p,/m/02jknp,False +421,195,Michele Bachmann,organization/role/leaders./organization/leadership/person,Breitbart News,/m/06jm4y,/m/0k0symb,False +422,195,Barack Obama,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0h7q0rq,/m/02mjmr,False +423,195,Barack Obama,people/person/employment_history./business/employment_tenure/title,Violence,/m/0h7q0rq,/m/0chbx,False +424,196,Paul+Ryan+RWI,people/person/employment_history./business/employment_tenure/title,United States Speaker of the House,Paul+Ryan+RWI,United States Speaker of the House,False +425,196,Brian Kilmeade,people/person/employment_history./business/employment_tenure/title,Host,/m/05ghhl,/m/04wk4z,False +426,196,Brian Kilmeade,organization/role/leaders./organization/leadership/person,Fox News,/m/05ghhl,/m/02z_b,False +427,196,Brian Kilmeade,organization/role/leaders./organization/leadership/person,Ryan's World,/m/05ghhl,Ryan's World,False +428,197,Donald Trump,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0cqt90,/m/02mjmr,False +429,197,Barack Obama,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0h7q0rq,/m/02mjmr,False +430,198,Muhammad,people/person/employment_history./business/employment_tenure/title,Prophet,/m/04s9n,/m/066dv,False +431,199,Walter Shaub,organization/role/leaders./organization/leadership/person,Office of Government Commerce,Walter Shaub,/m/05qwyl,False +432,199,Donald Trump,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0cqt90,/m/02mjmr,False +433,199,Office of Government Commerce,organization/role/leaders./organization/leadership/person,Walter Shaub,/m/05qwyl,Walter Shaub,False +434,199,Walter Shaub,organization/role/leaders./organization/leadership/person,Office of Government Commerce,Walter Shaub,/m/05qwyl,False +435,199,Lisa+Mead.Mead,people/person/employment_history./business/employment_tenure/title,Chef,Lisa+Mead.Mead,/m/01pn0r,False +436,199,Lisa+Mead.Mead,organization/role/leaders./organization/leadership/person,White House,Lisa+Mead.Mead,/m/03mdbqx,False +437,199,Lisa+Mead.Mead,organization/role/leaders./organization/leadership/person,White House,Lisa+Mead.Mead,/m/03mdbqx,False +438,199,Shinzo Abe,people/person/employment_history./business/employment_tenure/title,Prime Minister of the United Kingdom,/m/07t7hy,/m/060s9,False +439,199,Lisa+Mead,people/person/employment_history./business/employment_tenure/title,Chef,Lisa+Mead,/m/01pn0r,False +440,201,Carlson,organization/role/leaders./organization/leadership/person,Breitbart News,Carlson,/m/0k0symb,False +441,202,Cher,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0kw03,/m/02mjmr,False +442,202,Cher,people/person/employment_history./business/employment_tenure/title,Canada,/m/0kw03,/m/02rnkmh,False +443,202,Cher,people/person/employment_history./business/employment_tenure/title,Singer Corporation,/m/0kw03,/m/09l7n,False +444,202,Cher,organization/role/leaders./organization/leadership/person,Twitter,/m/0kw03,/m/0289n8t,False +445,202,Donald Trump,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0cqt90,/m/02mjmr,False +446,204,Donald Trump,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0cqt90,/m/02mjmr,False +447,204,John McCain,people/person/employment_history./business/employment_tenure/title,Senator,/m/0bymv,/m/04c_3b,False +448,206,Imran Khan,people/person/employment_history./business/employment_tenure/title,Imran Awan,/m/025cj4,Imran Awan,False +449,206,Debbie Wasserman Schultz,people/person/employment_history./business/employment_tenure/title,Rep,/m/04cmyw,/m/02qkv0r,False +450,206,House,organization/role/leaders./organization/leadership/person,Debbie Wasserman Schultz,/m/081sq,/m/04cmyw,False +451,207,House,organization/organization/headquarters./location/mailing_address/country,President of the United States,/m/081sq,/m/02mjmr,False +452,207,United States House of Representatives,organization/organization/headquarters./location/mailing_address/country,Us,/m/0b3wk,/m/0b3wk,False +453,211,Bill Murray,people/person/employment_history./business/employment_tenure/title,Actor,/m/0p_pd,/m/014g_s,False +454,213,Ghana,people/person/employment_history./business/employment_tenure/title,Donald Trump,/m/035dk,/m/0cqt90,False +455,214,Barack Obama,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0h7q0rq,/m/02mjmr,False +456,217,Mike Pence,people/person/employment_history./business/employment_tenure/title,governor,/m/022r9r,/m/0btx2g,False +457,218,Barack Obama,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0h7q0rq,/m/02mjmr,False +458,218,Barack Obama,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0h7q0rq,/m/02mjmr,False +459,218,Richard Smith,people/person/employment_history./business/employment_tenure/title,Consultant,/m/0n8_swr,/m/02n9jv,False +460,218,Paul Ryan,people/person/employment_history./business/employment_tenure/title,Member of Congress,/m/024v2j,/m/04l8cr,False +461,218,Barack Obama,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0h7q0rq,/m/02mjmr,False +462,218,John Boehner,people/person/employment_history./business/employment_tenure/title,Minority leader,/m/039rwf,/m/0499g1,False +463,218,Federal Reserve,organization/role/leaders./organization/leadership/person,John Boehner,/m/05_xks,/m/039rwf,False +464,218,Nancy Pelosi,organization/role/leaders./organization/leadership/person,House,/m/012v1t,/m/081sq,False +465,219,Donald Trump,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0cqt90,/m/02mjmr,False +466,219,Donald Trump,organization/role/leaders./organization/leadership/person,Los Angeles Times,/m/0cqt90,/m/01p06z,False +467,219,Donald Trump,organization/role/leaders./organization/leadership/person,Los Angeles Times,/m/0cqt90,/m/01p06z,False +468,219,Sean Spicer,people/person/employment_history./business/employment_tenure/title,Spokesperson,Sean Spicer,/m/03mdbqx,False +469,219,Sean Spicer,people/person/nationality,Spanish language,Sean Spicer,/m/06nm1,False +470,219,Jeb Bush,people/person/employment_history./business/employment_tenure/title,governor,/m/019x9z,/m/0btx2g,False +471,219,Donald Trump,people/person/nationality,Spanish language,/m/0cqt90,/m/06nm1,False +472,220,County,location/hud_county_place/county,Baker,/m/094z8,/m/01lbst,False +473,220,County,location/hud_county_place/county,Baker,/m/094z8,/m/01lbst,False +474,220,Watsons,organization/role/leaders./organization/leadership/person,Heather,/m/065nfw,Heather,False +475,221,Doug Ericksen,people/person/employment_history./business/employment_tenure/title,State senator,/m/0k0q7zz,/m/04zz1zx,False +476,224,Donald Trump,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0cqt90,/m/02mjmr,False +477,225,County,location/hud_county_place/county,"Westchester County, New York",/m/094z8,/m/0cymp,False +478,225,Bill Clinton,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0157m,/m/02mjmr,False +479,226,John McCain,organization/role/leaders./organization/leadership/person,United States Senate Committee on Armed Services,/m/0bymv,/m/02kfv7,False +480,226,Chuck Todd,organization/role/leaders./organization/leadership/person,Meet the Press,/m/0g12y5,/m/0167_c,False +481,226,John McCain,organization/role/leaders./organization/leadership/person,Meet,/m/0bymv,Meet,False +482,226,John McCain,people/person/employment_history./business/employment_tenure/title,Chairperson,/m/0bymv,/m/05_xks,False +483,226,Malcolm Rifkind,people/person/employment_history./business/employment_tenure/title,Foreign Secretary,/m/01zq6b,Foreign Secretary,False +484,226,Kim Jong-un,people/person/employment_history./business/employment_tenure/title,Leadership,/m/0fgw19,/m/0h7q0rq,False +485,228,John Hagee,organization/role/leaders./organization/leadership/person,Cornerstone Church,/m/047qlq,Cornerstone Church,False +486,229,Southeastern United States,location/country/languages_spoken,South Africa,/m/03dwt_,/m/0hzlz,False +487,229,Greenpeace,organization/role/leaders./organization/leadership/person,James+Ben+Shahali,/m/036qv,James+Ben+Shahali,False +488,230,Barack Obama,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0h7q0rq,/m/02mjmr,False +489,230,Barack Obama,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0h7q0rq,/m/02mjmr,False +490,230,Richard Smith,people/person/employment_history./business/employment_tenure/title,Consultant,/m/0n8_swr,/m/02n9jv,False +491,230,Paul Ryan,people/person/employment_history./business/employment_tenure/title,Member of Congress,/m/024v2j,/m/04l8cr,False +492,230,Barack Obama,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0h7q0rq,/m/02mjmr,False +493,230,John Boehner,people/person/employment_history./business/employment_tenure/title,Minority leader,/m/039rwf,/m/0499g1,False +494,230,Federal Reserve,organization/role/leaders./organization/leadership/person,John Boehner,/m/05_xks,/m/039rwf,False +495,230,Nancy Pelosi,organization/role/leaders./organization/leadership/person,House,/m/012v1t,/m/081sq,False +496,232,Donald Trump,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0cqt90,/m/02mjmr,False +497,233,Jim Porter,people/person/employment_history./business/employment_tenure/title,President of the United States,Jim Porter,/m/02mjmr,False +498,233,Jim Porter,organization/role/leaders./organization/leadership/person,List of presidents of the National Rifle Association,Jim Porter,List of presidents of the National Rifle Association,False +499,233,Barack Obama,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0h7q0rq,/m/02mjmr,False +500,233,Eric Holder,people/person/employment_history./business/employment_tenure/title,Attorney general,/m/02mjmr,/m/0465sdr,False +501,233,Wayne LaPierre,people/person/employment_history./business/employment_tenure/title,Vice President of the United States,/m/04k0vk,/m/0d05fv,False +502,235,Steve Bannon,organization/role/leaders./organization/leadership/person,Breitbart News,/m/0bqscsg,/m/0k0symb,False +503,235,Steve Bannon,organization/role/leaders./organization/leadership/person,Electric battery,/m/0bqscsg,/m/01c0z,False +504,235,Steve Bannon,organization/role/leaders./organization/leadership/person,Politico,/m/0bqscsg,/m/027_2w9,False +505,235,Steve Bannon,organization/role/leaders./organization/leadership/person,Politico,/m/0bqscsg,/m/027_2w9,False +506,235,Steve Bannon,people/person/employment_history./business/employment_tenure/title,Chairperson,/m/0bqscsg,/m/05_xks,False +507,236,Donald Trump,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0cqt90,/m/02mjmr,False +508,237,Jimmy Carter,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/042kg,/m/02mjmr,False +509,238,Barack+Obama.Yesterday+Winters,people/person/employment_history./business/employment_tenure/title,President of the United States,Barack+Obama.Yesterday+Winters,/m/02mjmr,False +510,239,Donald Trump,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0cqt90,/m/02mjmr,False +511,240,Barack Obama,people/person/places_lived./people/place_lived/location,Donald Trump,/m/0h7q0rq,/m/0cqt90,False +512,240,Barack Obama,people/person/places_lived./people/place_lived/location,Donald Trump,/m/0h7q0rq,/m/0cqt90,False +513,241,Donald Trump,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0cqt90,/m/02mjmr,False +514,241,Todd+McMartin,organization/role/leaders./organization/leadership/person,Federal Bureau of Investigation,Todd+McMartin,/m/02_1m,False +515,241,Donald Trump,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0cqt90,/m/02mjmr,False +516,242,Donald Trump,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0cqt90,/m/02mjmr,False +517,244,Donald Trump,organization/role/leaders./organization/leadership/person,United States Congress,/m/0cqt90,/m/0d06m5,False +518,244,Donald Trump,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0cqt90,/m/02mjmr,False +519,244,Donald Trump,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0cqt90,/m/02mjmr,False +520,245,Kelly Services,people/person/employment_history./business/employment_tenure/title,Journalist,/m/03p2hqb,/m/0d8qb,False +521,245,Kelly Services,people/person/employment_history./business/employment_tenure/title,Journalist,/m/03p2hqb,/m/0d8qb,False +522,245,Donald Trump,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0cqt90,/m/02mjmr,False +523,245,Joseph Gordon-Levitt,organization/role/leaders./organization/leadership/person,NBC,/m/05zbm4,/m/05gnf,False +524,245,Joseph Gordon-Levitt,people/person/employment_history./business/employment_tenure/title,Spokesperson,/m/05zbm4,/m/03mdbqx,False +525,245,Joseph Gordon-Levitt,organization/role/leaders./organization/leadership/person,Megyn Kelly,/m/05zbm4,/m/07wpbx,False +526,246,Joshua Johnson,organization/role/leaders./organization/leadership/person,Clark Atlanta University,/m/052g5x,/m/03818y,False +527,246,Stephawn,people/person/employment_history./business/employment_tenure/title,Stephawn,Stephawn,Stephawn,False +528,251,Vince Foster,people/person/employment_history./business/employment_tenure/title,Attorney,/m/080kq,/m/04gc2,False +529,251,Vince Foster,people/person/employment_history./business/employment_tenure/title,Bill Clinton,/m/080kq,/m/0157m,False +530,251,Trey Gowdy,people/person/employment_history./business/employment_tenure/title,Judge,/m/0c3x3rz,/m/06lvrr,False +531,251,Trey Gowdy,people/person/employment_history./business/employment_tenure/title,Rep,/m/0c3x3rz,/m/02qkv0r,False +532,251,Trey Gowdy,people/person/employment_history./business/employment_tenure/title,Judge,/m/0c3x3rz,/m/06lvrr,False +533,251,James+McFitting,people/person/employment_history./business/employment_tenure/title,Col,James+McFitting,Col,False +534,251,James Comey,people/person/employment_history./business/employment_tenure/title,Film director,/m/06r04p,/m/02jknp,False +535,253,Road,transportation/road/end1./transportation/road_starting_point/location,South,/m/06gfj,/m/0k7ny,False +536,253,Road,transportation/road/end1./transportation/road_starting_point/location,Casimir Pulaski,/m/06gfj,/m/0lrr8,False +537,253,Road,transportation/road/end1./transportation/road_starting_point/location,South,/m/06gfj,/m/0k7ny,False +538,253,Kyle+Jennings,people/person/employment_history./business/employment_tenure/title,LT,Kyle+Jennings,/m/015czt,False +539,253,Road,transportation/road/end1./transportation/road_starting_point/location,South,/m/06gfj,/m/0k7ny,False +540,253,Road,transportation/road/end1./transportation/road_starting_point/location,Casimir Pulaski,/m/06gfj,/m/0lrr8,False +541,253,Road,transportation/road/end1./transportation/road_starting_point/location,South,/m/06gfj,/m/0k7ny,False +542,253,Kyle+Jennings,people/person/employment_history./business/employment_tenure/title,LT,Kyle+Jennings,/m/015czt,False +543,254,"Leon County Court, Florida",organization/role/leaders./organization/leadership/person,Hillary Clinton,"Leon County Court, Florida",/m/0d06m5,False +544,255,Donald Trump,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0cqt90,/m/02mjmr,False +545,255,Johnny Appleseed,people/person/employment_history./business/employment_tenure/title,Rep,/m/0159qc,/m/02qkv0r,False +546,256,Donald Trump,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0cqt90,/m/02mjmr,False +547,257,John Kasich,people/person/employment_history./business/employment_tenure/title,governor,/m/02zzm_,/m/0btx2g,False +548,257,John Kasich,organization/role/leaders./organization/leadership/person,Daily Star,/m/02zzm_,/m/01sv_b,False +549,258,Vin Diesel,organization/role/leaders./organization/leadership/person,Facebook+FacebookShare,/m/025n3p,/m/0glpjll,False +550,258,Vin Diesel,people/person/employment_history./business/employment_tenure/title,Actor,/m/025n3p,/m/014g_s,False +551,259,Clinton+Greenwood,people/person/employment_history./business/employment_tenure/title,Deputy,Clinton+Greenwood,/m/03yc4j2,False +552,262,Little,people/person/employment_history./business/employment_tenure/title,Chancellor,/m/0ftvg,/m/01fch0,False +553,263,Donald Trump,people/person/employment_history./business/employment_tenure/title,Entrepreneurship,/m/0cqt90,/m/02nwq,False +554,263,County,location/hud_county_place/county,Orange,/m/094z8,/m/094z8,False +555,264,Donald Trump,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0cqt90,/m/02mjmr,False +556,264,Donald Trump,people/person/employment_history./business/employment_tenure/title,North Korea,/m/0cqt90,/m/05b7q,False +557,264,Donald Trump,people/person/employment_history./business/employment_tenure/title,China,/m/0cqt90,/m/0d05w3,False +558,264,Donald Trump,people/person/employment_history./business/employment_tenure/title,North Korea,/m/0cqt90,/m/05b7q,False +559,264,Donald Trump,people/person/employment_history./business/employment_tenure/title,Florida,/m/0cqt90,/m/02xry,False +560,264,Donald Trump,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0cqt90,/m/02mjmr,False +561,264,Donald Trump,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0cqt90,/m/02mjmr,False +562,267,Federation,location/country/languages_spoken,Russia,/m/01zf6x,/m/06bnz,False +563,267,Federation,location/country/languages_spoken,Russia,/m/014j8_,/m/06bnz,False +564,267,Donald Trump,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0cqt90,/m/02mjmr,False +565,267,Donald Trump,location/country/languages_spoken,Federation,/m/0cqt90,/m/01zf6x,False +566,267,Yuri+Ivanenko,people/person/employment_history./business/employment_tenure/title,Judge,Yuri+Ivanenko,/m/06lvrr,False +567,267,Yuri+Ivanenko,people/person/employment_history./business/employment_tenure/title,Judge,Yuri+Ivanenko,/m/06lvrr,False +568,268,Donald Trump,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0cqt90,/m/02mjmr,False +569,268,Donald Trump,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0cqt90,/m/02mjmr,False +570,268,Donald Trump,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0cqt90,/m/02mjmr,False +571,268,Mike Pence,people/person/employment_history./business/employment_tenure/title,Vice President of the United States,/m/022r9r,/m/0d05fv,False +572,271,Gonzalo P. Curiel,people/person/employment_history./business/employment_tenure/title,Judge,/m/0hht8px,/m/06lvrr,False +573,271,Mitch McConnell,organization/role/leaders./organization/leadership/person,United States Senate,/m/01z6ls,/m/07r9r0,False +574,271,Gonzalo P. Curiel,people/person/employment_history./business/employment_tenure/title,Judge,/m/0hht8px,/m/06lvrr,False +575,271,Susan Collins,organization/role/leaders./organization/leadership/person,WGAN,/m/020y8m,/m/02pxjcm,False +576,271,Susan Collins,people/person/employment_history./business/employment_tenure/title,Rep,/m/020y8m,/m/02qkv0r,False +577,271,Paul Ryan,people/person/employment_history./business/employment_tenure/title,United States Speaker of the House,/m/024v2j,United States Speaker of the House,False +578,273,Bill Clinton,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0157m,/m/02mjmr,False +579,275,Michael John O'Leary,organization/role/leaders./organization/leadership/person,New York City Police Department,/m/04jprn,/m/01lvn4,False +580,275,Tucker Carlson,organization/role/leaders./organization/leadership/person,Fox News,/m/03cqm2,/m/02z_b,False +581,275,Michael John O'Leary,organization/role/leaders./organization/leadership/person,Saint Mary,/m/04jprn,/m/03mcdjf,False +582,277,Mizu+Tomazaki,people/person/employment_history./business/employment_tenure/title,Judge,Mizu+Tomazaki,/m/06lvrr,False +583,277,Dmitry Peskov,people/person/employment_history./business/employment_tenure/title,Spokesperson,/m/0280k2k,/m/03mdbqx,False +584,277,Dmitry Peskov,organization/role/leaders./organization/leadership/person,Moscow Kremlin,/m/0280k2k,/m/021lry,False +585,277,John McCain,people/person/employment_history./business/employment_tenure/title,Senator,/m/0bymv,/m/04c_3b,False +586,278,Mitch Landrieu,people/person/employment_history./business/employment_tenure/title,Mayor,/m/05xdp5,/m/03x_sks,False +587,279,Seth+RichWikiLeaks,organization/role/leaders./organization/leadership/person,Democratic National Committee,Seth+RichWikiLeaks,/m/018y36,False +588,279,Seth+RichWikiLeaks,organization/role/leaders./organization/leadership/person,Democratic National Committee,Seth+RichWikiLeaks,/m/018y36,False +589,279,Murder of Seth Rich,organization/role/leaders./organization/leadership/person,Democratic National Committee,Murder of Seth Rich,/m/018y36,False +590,279,Murder of Seth Rich,organization/role/leaders./organization/leadership/person,Democratic National Committee,Murder of Seth Rich,/m/018y36,False +591,279,Murder of Seth Rich,organization/role/leaders./organization/leadership/person,Democratic National Committee,Murder of Seth Rich,/m/018y36,False +592,279,Murder of Seth Rich,organization/role/leaders./organization/leadership/person,Democratic National Committee,Murder of Seth Rich,/m/018y36,False +593,279,Murder of Seth Rich,organization/role/leaders./organization/leadership/person,Democratic National Committee,Murder of Seth Rich,/m/018y36,False +594,280,Andrew Jackson,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0rlz,/m/02mjmr,False +595,280,Colonel Sanders,people/person/employment_history./business/employment_tenure/title,Col,/m/09b78,Col,False +596,280,Donald,organization/role/leaders./organization/leadership/person,Republican Party,Donald,/m/085srz,False +597,280,Bernie Sanders,people/person/employment_history./business/employment_tenure/title,Col,/m/01_gbv,Col,False +598,280,Donald Trump,organization/role/leaders./organization/leadership/person,KFC,/m/0cqt90,/m/09b6t,False +599,280,Bernie Sanders,people/person/employment_history./business/employment_tenure/title,Col,/m/01_gbv,Col,False +600,280,Bernie Sanders,people/person/employment_history./business/employment_tenure/title,Col,/m/01_gbv,Col,False +601,282,Donald Trump,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0cqt90,/m/02mjmr,False +602,282,Donald Trump,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0cqt90,/m/02mjmr,False +603,282,James Comey,people/person/employment_history./business/employment_tenure/title,Director,/m/06r04p,/m/02mjmr,False +604,282,James Comey,organization/role/leaders./organization/leadership/person,Federal Bureau of Investigation,/m/06r04p,/m/02_1m,False +605,282,Justin Amash,people/person/employment_history./business/employment_tenure/title,Rep,/m/0c00p_n,/m/02qkv0r,False +606,282,Donald Trump,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0cqt90,/m/02mjmr,False +607,282,Jason Chaffetz,organization/role/leaders./organization/leadership/person,House+Oversign+Committee+Chair,/m/047blr4,House+Oversign+Committee+Chair,False +608,282,Jason Chaffetz,people/person/employment_history./business/employment_tenure/title,Rep,/m/047blr4,/m/02qkv0r,False +609,282,Jason Chaffetz,organization/role/leaders./organization/leadership/person,House+Oversign+Committee+Chair,/m/047blr4,House+Oversign+Committee+Chair,False +610,282,Chris Hayes,organization/role/leaders./organization/leadership/person,MSNBC,/m/0h3wc7,/m/0152x_,False +611,282,Chris Hayes,people/person/employment_history./business/employment_tenure/title,Host,/m/0h3wc7,/m/04wk4z,False +612,282,Jonathan Turley,people/person/employment_history./business/employment_tenure/title,Google Scholar,/m/05kldj,/m/057mg4,False +613,282,Barack Obama,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0h7q0rq,/m/02mjmr,False +614,282,Donald Trump,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0cqt90,/m/02mjmr,False +615,284,Donald Trump,organization/role/leaders./organization/leadership/person,United States Congress,/m/0cqt90,/m/0d06m5,False +616,284,Donald Trump,organization/role/leaders./organization/leadership/person,United States Congress,/m/0cqt90,/m/0d06m5,False +617,284,Donald Trump,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0cqt90,/m/02mjmr,False +618,284,Donald Trump,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0cqt90,/m/02mjmr,False +619,284,Hillary+Clinton.Jason+Chaffetz,organization/role/leaders./organization/leadership/person,United+States+House+Committee+on+Oversight+and+Government+Reform,Hillary+Clinton.Jason+Chaffetz,United+States+House+Committee+on+Oversight+and+Government+Reform,False +620,284,Donald Trump,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0cqt90,/m/02mjmr,False +621,284,Donald Trump,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0cqt90,/m/02mjmr,False +622,285,Marco Rubio,people/person/employment_history./business/employment_tenure/title,Candidate,/m/0dpr5f,/m/07r9r0,False +623,286,Louis+Leweigh,people/person/employment_history./business/employment_tenure/title,Correspondent,Louis+Leweigh,/m/02k13d,False +624,287,Shopping mall,location/country/languages_spoken,Mall+of+Claus,/m/025s8bs,Mall+of+Claus,False +625,287,Mall of America,location/country/languages_spoken,Americas,/m/0pv1y,/m/07c5l,False +626,287,Shopping mall,location/country/languages_spoken,Mall+of+Claus,/m/025s8bs,Mall+of+Claus,False +627,287,Mall+of+Claus,location/country/languages_spoken,Americas,Mall+of+Claus,/m/07c5l,False +628,287,Mall+of+Claus,location/country/languages_spoken,Americas,Mall+of+Claus,/m/07c5l,False +629,287,Mall of America,location/country/languages_spoken,Americas,/m/0pv1y,/m/07c5l,False +630,291,Hal+Lindsay+DNJ,people/person/employment_history./business/employment_tenure/title,Senator,Hal+Lindsay+DNJ,/m/04c_3b,False +631,291,Lindsay,people/person/employment_history./business/employment_tenure/title,Senator,Lindsay,/m/04c_3b,False +632,292,Charles Bolden,organization/role/leaders./organization/leadership/person,NASA,/m/02j2mf,/m/05f4p,False +633,293,Donald Trump,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0cqt90,/m/02mjmr,False +634,294,Donald Trump,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0cqt90,/m/02mjmr,False +635,294,Kellyanne Conway,people/person/employment_history./business/employment_tenure/title,Donald Trump,/m/0dq819,/m/0cqt90,False +636,294,James Collins,organization/role/leaders./organization/leadership/person,Twitter,/m/06sjd0,/m/0289n8t,False +637,296,Republican Party,organization/role/leaders./organization/leadership/person,Chuck Schumer,/m/085srz,/m/01w74d,False +638,298,Donald Trump,organization/role/leaders./organization/leadership/person,Globe,/m/0cqt90,/m/01q_cy,False +639,299,Road,transportation/road/end1./transportation/road_starting_point/location,Bangkok,/m/06gfj,/m/0fn2g,False +640,299,Loung Ung,people/person/employment_history./business/employment_tenure/title,Loung Ung,/m/074fdq,/m/074fdq,False +641,300,Bill Clinton,people/person/employment_history./business/employment_tenure/title,Secretary,/m/0157m,/m/06slz5,True +642,302,Linda+Stanley,organization/role/leaders./organization/leadership/person,Franklin+County+Historical+Society,Linda+Stanley,Franklin+County+Historical+Society,True +643,302,County,location/hud_county_place/county,Franklin,/m/094z8,/m/019fz,True +644,302,County,location/hud_county_place/county,Franklin,/m/094z8,/m/019fz,True +645,302,Terry McAuliffe,people/person/employment_history./business/employment_tenure/title,governor,/m/02dgtb,/m/0btx2g,True +646,302,Edward L. Ayers,organization/role/leaders./organization/leadership/person,University of Richmond,/m/03cztnx,/m/03hpkp,True +647,302,Fred Taylor,people/person/employment_history./business/employment_tenure/title,Attorney,/m/01tz2p,/m/04gc2,True +648,302,Taylor Swift,organization/role/leaders./organization/leadership/person,Supreme Court of Virginia,/m/0dl567,/m/05ngqz,True +649,302,Taylor Swift,organization/role/leaders./organization/leadership/person,Supreme Court of Virginia,/m/0dl567,/m/05ngqz,True +650,302,Michael Signer,people/person/employment_history./business/employment_tenure/title,Mayor,/m/05h59wt,/m/03x_sks,True +651,302,Jefferson Davis,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/043q0,/m/02mjmr,True +652,302,Jefferson Davis,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/043q0,/m/02mjmr,True +653,302,County,location/hud_county_place/county,Franklin,/m/094z8,/m/019fz,True +654,302,County,location/hud_county_place/county,Franklin,/m/094z8,/m/019fz,True +655,302,County,location/hud_county_place/county,Franklin,/m/094z8,/m/019fz,True +656,302,County,location/hud_county_place/county,Franklin,/m/094z8,/m/019fz,True +657,302,Landon+Howard,organization/role/leaders./organization/leadership/person,Visit+Virginias+Blue+Ridge,/m/01wl8_,Visit+Virginias+Blue+Ridge,True +658,302,County,location/hud_county_place/county,Franklin,/m/094z8,/m/019fz,True +659,302,County,location/hud_county_place/county,Franklin,/m/094z8,/m/019fz,True +660,304,Hillary Clinton,people/person/employment_history./business/employment_tenure/title,Lawyer,/m/0d06m5,/m/02vy3_j,True +661,306,Josh+Hendler,people/person/employment_history./business/employment_tenure/title,Campaign manager,Josh+Hendler,/m/03xk7t,True +662,306,Donald Trump,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0cqt90,/m/02mjmr,True +663,307,Marco Rubio,organization/role/leaders./organization/leadership/person,Gang of Eight,/m/0dpr5f,/m/09z___,True +664,307,Beck,people/person/employment_history./business/employment_tenure/title,My First First Love,/m/0137g1,My First First Love,True +665,307,Beck,people/person/employment_history./business/employment_tenure/title,Politician,/m/0137g1,Jimmy Dimora,True +666,308,Donald Trump,people/person/employment_history./business/employment_tenure/title,Candidate,/m/0cqt90,/m/07r9r0,True +667,308,Andrew Cuomo,people/person/employment_history./business/employment_tenure/title,governor,/m/02pjpd,/m/0btx2g,True +668,308,Andrew Cuomo,organization/role/leaders./organization/leadership/person,CNN,/m/02pjpd,/m/0gsgr,True +669,308,Alisyn Camerota,organization/role/leaders./organization/leadership/person,CNN,/m/0c3ntp,/m/0gsgr,True +670,309,2016 United States presidential election,organization/role/leaders./organization/leadership/person,Republican Party,2016 United States presidential election,/m/085srz,True +671,309,United States Secretary of State,people/person/employment_history./business/employment_tenure/title,2016 United States presidential election,/m/0bwl7t0,2016 United States presidential election,True +672,311,Richard Nixon,organization/role/leaders./organization/leadership/person,Dwight D. Eisenhower,/m/06c97,/m/028rk,True +673,311,Richard Nixon,organization/role/leaders./organization/leadership/person,White House,/m/06c97,/m/03mdbqx,True +674,311,Gerald Ford,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0c_md_,/m/02mjmr,True +675,311,Walter Mondale,people/person/employment_history./business/employment_tenure/title,Vice President of the United States,/m/0bl83,/m/0d05fv,True +676,311,Henry Trewhitt,people/person/employment_history./business/employment_tenure/title,Journalist,Henry Trewhitt,/m/0d8qb,True +677,311,Henry Trewhitt,organization/role/leaders./organization/leadership/person,The Baltimore Sun,Henry Trewhitt,/m/01n0qv,True +678,311,George Bernard Shaw,organization/role/leaders./organization/leadership/person,CNN,/m/03cdg,/m/0gsgr,True +679,311,Michael Dukakis,people/person/employment_history./business/employment_tenure/title,governor,/m/0jw4v,/m/0btx2g,True +680,311,George Bernard Shaw,organization/role/leaders./organization/leadership/person,CNN,/m/03cdg,/m/0gsgr,True +681,311,Al Gore,people/person/employment_history./business/employment_tenure/title,Vice President of the United States,/m/0d05fv,/m/0d05fv,True +682,311,John Kerry,people/person/employment_history./business/employment_tenure/title,Senator,/m/0d3qd0,/m/04c_3b,True +683,311,George W. Bush,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0d05fv,/m/02mjmr,True +684,311,Al Gore,people/person/employment_history./business/employment_tenure/title,Vice President of the United States,/m/0d05fv,/m/0d05fv,True +685,311,Viviana Zelizer,organization/role/leaders./organization/leadership/person,University,/m/07t19q,/m/07tf8,True +686,311,Lyndon B. Johnson,organization/role/leaders./organization/leadership/person,United States Congress,/m/0f7fy,/m/0d06m5,True +687,311,Richard Nixon,people/person/employment_history./business/employment_tenure/title,Author,/m/06c97,/m/0kyk,True +688,312,Shira Scheindlin,organization/role/leaders./organization/leadership/person,United States district court,/m/064x_k,/m/0j75z,True +689,312,Shira Scheindlin,people/person/employment_history./business/employment_tenure/title,Judge,/m/064x_k,/m/06lvrr,True +690,314,Robert F. Kennedy,organization/role/leaders./organization/leadership/person,Associated Press,/m/06hx2,/m/0cv_2,True +691,314,Bill Clinton,people/person/employment_history./business/employment_tenure/title,Secretary,/m/0157m,/m/06slz5,True +692,314,Dwight D. Eisenhower,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/028rk,/m/02mjmr,True +693,314,Barack Obama,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0h7q0rq,/m/02mjmr,True +694,314,Donald Trump,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0cqt90,/m/02mjmr,True +695,315,Jim Justice,people/person/employment_history./business/employment_tenure/title,West Virginia,Jim Justice,/m/081mh,True +696,315,Jim Justice,people/person/employment_history./business/employment_tenure/title,governor,Jim Justice,/m/0btx2g,True +697,315,Justice,organization/role/leaders./organization/leadership/person,Hillary Clinton,/m/045k9,/m/0d06m5,True +698,315,Bill Cole,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/06mg8q,/m/02mjmr,True +699,315,Justice,organization/role/leaders./organization/leadership/person,Bill Clinton,/m/045k9,/m/0157m,True +700,315,Bill Cole,organization/role/leaders./organization/leadership/person,United States Senate,/m/06mg8q,/m/07r9r0,True +701,315,Justice,organization/role/leaders./organization/leadership/person,Hillary Clinton,/m/045k9,/m/0d06m5,True +702,315,West Virginia Republican Party,organization/role/leaders./organization/leadership/person,Donald Trump,/m/02r81cr,/m/0cqt90,True +703,315,governor,people/person/employment_history./business/employment_tenure/title,Bill Cole,/m/0btx2g,/m/06mg8q,True +704,315,Kyle+Kondik,organization/role/leaders./organization/leadership/person,University of Virginia Center for Politics,Kyle+Kondik,University of Virginia Center for Politics,True +705,315,Kyle+Kondik,people/person/employment_history./business/employment_tenure/title,Sabato's Crystal Ball,Kyle+Kondik,/m/076xl9x,True +706,315,Scott+Crichlow,organization/role/leaders./organization/leadership/person,West Virginia University,Scott+Crichlow,/m/019dwp,True +707,315,Donald Grant Herring Estate,people/person/employment_history./business/employment_tenure/title,Spokesperson,/m/01370l_y,/m/03mdbqx,True +708,315,Donald Grant Herring Estate,organization/role/leaders./organization/leadership/person,Justice,/m/01370l_y,/m/045k9,True +709,315,Consultant,people/person/employment_history./business/employment_tenure/title,David Saunders,/m/02n9jv,David Saunders,True +710,319,Kerr+Putney,organization/role/leaders./organization/leadership/person,Charlotte-Mecklenburg Police Department,Kerr+Putney,/m/03d5lmj,True +711,319,Rakeyia+Scott,people/person/places_lived./people/place_lived/location,Scotts,Rakeyia+Scott,Scotts,True +712,319,Kerr+Putney,organization/role/leaders./organization/leadership/person,Charlotte-Mecklenburg Police Department,Kerr+Putney,/m/03d5lmj,True +713,319,Pat McCrory,people/person/employment_history./business/employment_tenure/title,governor,/m/05tjbz,/m/0btx2g,True +714,320,Jama+B.,people/person/employment_history./business/employment_tenure/title,Afghanistan,/m/05p0j_b,/m/0jdd,True +715,320,Jama+B.,people/person/employment_history./business/employment_tenure/title,Superstar,/m/05p0j_b,Superstar,True +716,320,Boostedt,location/country/languages_spoken,Iraq,/m/0ctllc,/m/0v74,True +717,320,Charles+Martlandfor,people/person/employment_history./business/employment_tenure/title,Sergeant,Charles+Martlandfor,/m/01g2j2,True +718,321,Bill Clinton,organization/role/leaders./organization/leadership/person,Getty+Trump,/m/0157m,Getty+Trump,True +719,324,Maxine Waters,people/person/employment_history./business/employment_tenure/title,Rep,/m/024tyk,/m/02qkv0r,True +720,324,Lynch,organization/role/leaders./organization/leadership/person,Office,Lynch,/m/021sj1,True +721,324,Lynch,organization/role/leaders./organization/leadership/person,Office for Civil Rights,Lynch,/m/0b9v8_,True +722,324,Lynch,organization/role/leaders./organization/leadership/person,Office,Lynch,/m/021sj1,True +723,324,Lynch,organization/role/leaders./organization/leadership/person,Office for Civil Rights,Lynch,/m/0b9v8_,True +724,324,Cedric+Richmond+DLa.,people/person/employment_history./business/employment_tenure/title,Rep,Cedric+Richmond+DLa.,/m/02qkv0r,True +725,325,Bain & Company,people/person/nationality,United States,/m/04vrnl,/m/07t58,True +726,326,Barack Obama,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0h7q0rq,/m/02mjmr,True +727,326,Donald Trump,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0cqt90,/m/02mjmr,True +728,327,Michael McDonald,organization/role/leaders./organization/leadership/person,University of Florida,/m/06cc_1,/m/0j_sncb,True +729,328,Peter King,people/person/employment_history./business/employment_tenure/title,Rep,/m/03tk4w,/m/02qkv0r,True +730,328,Peter King,people/person/employment_history./business/employment_tenure/title,Rep,/m/03tk4w,/m/02qkv0r,True +731,328,Hillary Clinton,people/person/employment_history./business/employment_tenure/title,United States Secretary of State,/m/0d06m5,/m/0bwl7t0,True +732,328,King,organization/role/leaders./organization/leadership/person,MSNBC,/m/03w9bnr,/m/0152x_,True +733,329,Bill de Blasio,people/person/employment_history./business/employment_tenure/title,Mayor,/m/0gjsd3,/m/03x_sks,True +734,329,New Jersey State Police,organization/role/leaders./organization/leadership/person,Elizabeth I,/m/083rp7,/m/02rg_,True +735,329,Barack Obama,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0h7q0rq,/m/02mjmr,True +736,329,Josh Earnest,people/person/employment_history./business/employment_tenure/title,Spokesperson,/m/0l8ps43,/m/03mdbqx,True +737,329,Kelly Services,organization/role/leaders./organization/leadership/person,Federal Bureau of Investigation,/m/03p2hqb,/m/02_1m,True +738,330,Barack Obama,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0h7q0rq,/m/02mjmr,True +739,331,Timothy Evans,organization/role/leaders./organization/leadership/person,Associated Press,Timothy Evans,/m/0cv_2,True +740,331,Chris Christie,people/person/employment_history./business/employment_tenure/title,governor,/m/0f8t6k,/m/0btx2g,True +741,331,Timothy Evans,organization/role/leaders./organization/leadership/person,Associated Press,Timothy Evans,/m/0cv_2,True +742,331,Charlie Baker,people/person/employment_history./business/employment_tenure/title,governor,/m/027cmm9,/m/0btx2g,True +743,331,Larry Hogan,people/person/employment_history./business/employment_tenure/title,governor,/m/0_ylz5n,/m/0btx2g,True +744,331,Dennis Daugaard,people/person/employment_history./business/employment_tenure/title,governor,/m/0bb5tw,/m/0btx2g,True +745,332,Bill Clinton,people/person/employment_history./business/employment_tenure/title,Secretary,/m/0157m,/m/06slz5,True +746,334,Barack Obama,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0h7q0rq,/m/02mjmr,True +747,334,Barack Obama,people/person/employment_history./business/employment_tenure/title,Mitt Romney,/m/0h7q0rq,/m/05k7sb,True +748,334,Barack Obama,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0h7q0rq,/m/02mjmr,True +749,336,Jean-Marc Puissesseau,people/person/employment_history./business/employment_tenure/title,Chief executive officer,Jean-Marc Puissesseau,/m/02mwvv,True +750,337,Friedrich,people/person/places_lived./people/place_lived/location,Bavaria,Friedrich,/m/017v_,True +751,337,Paul Roland,organization/role/leaders./organization/leadership/person,Palatinate,/m/0406vgn,/m/09pkf,True +752,337,Friedrich,people/person/places_lived./people/place_lived/location,Kallstadt,Friedrich,/m/02rscps,True +753,337,Friedrich,people/person/places_lived./people/place_lived/location,Kallstadt,Friedrich,/m/02rscps,True +754,337,Friedrich,people/person/places_lived./people/place_lived/location,Kallstadt,Friedrich,/m/02rscps,True +755,337,Simone Wendel,people/person/employment_history./business/employment_tenure/title,Kallstadt,/m/011x14ft,/m/02rscps,True +756,339,Rex Elsass,organization/role/leaders./organization/leadership/person,Republican Party,/m/0cx2r,/m/085srz,True +757,339,Kellyanne Conway,people/person/employment_history./business/employment_tenure/title,Campaign manager,/m/0dq819,/m/03xk7t,True +758,339,Todd Akin,organization/role/leaders./organization/leadership/person,Missouri Senate,/m/025xzm,/m/0694qd,True +759,339,Kellyanne Conway,people/person/employment_history./business/employment_tenure/title,Donald Trump,/m/0dq819,/m/0cqt90,True +760,339,Todd Akin,people/person/employment_history./business/employment_tenure/title,Candidate,/m/025xzm,/m/07r9r0,True +761,339,Alsace,organization/role/leaders./organization/leadership/person,Global Strategy Group,/m/0cx2r,/m/04mzmmw,True +762,339,Alsace,organization/role/leaders./organization/leadership/person,Strategy+Group+for+Media,/m/0cx2r,Strategy+Group+for+Media,True +763,340,Pat McCrory,people/person/employment_history./business/employment_tenure/title,governor,/m/05tjbz,/m/0btx2g,True +764,340,Pat McCrory,people/person/employment_history./business/employment_tenure/title,unrestNorth,/m/05tjbz,/m/0dln6mf,True +765,341,Etonde+Maloke,people/person/employment_history./business/employment_tenure/title,Student,Etonde+Maloke,/m/014cnc,True +766,342,William Bratton,organization/role/leaders./organization/leadership/person,New York City Police Department,/m/06lwq6,/m/01lvn4,True +767,342,Rudy Giuliani,people/person/employment_history./business/employment_tenure/title,Mayor,/m/06gn7,/m/03x_sks,True +768,342,Yvette Clarke,people/person/employment_history./business/employment_tenure/title,Rep,/m/06h90t,/m/02qkv0r,True +769,342,Hakeem Jeffries,people/person/employment_history./business/employment_tenure/title,Rep,/m/025_74m,/m/02qkv0r,True +770,343,John Kerry,people/person/employment_history./business/employment_tenure/title,United States Secretary of State,/m/0d3qd0,/m/0bwl7t0,True +771,345,County Kerry,organization/role/leaders./organization/leadership/person,Associated Press,/m/0m_w6,/m/0cv_2,True +772,345,John Kerry,people/person/employment_history./business/employment_tenure/title,United States Secretary of State,/m/0d3qd0,/m/0bwl7t0,True +773,345,County Kerry,organization/role/leaders./organization/leadership/person,Associated Press,/m/0m_w6,/m/0cv_2,True +774,345,Sergey Lavrov,people/person/employment_history./business/employment_tenure/title,Foreign minister,/m/02n670,/m/01t_55,True +775,345,John Kerry,people/person/employment_history./business/employment_tenure/title,John Kerry,/m/0d3qd0,/m/0d3qd0,True +776,345,Russian involvement in the Syrian civil war,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0jl0yx0,/m/02mjmr,True +777,346,Ford Motor Company,organization/organization/headquarters./location/mailing_address/country,United States,/m/02zs4,/m/07t58,True +778,346,Ford Motor Company,organization/organization/headquarters./location/mailing_address/country,United States,/m/02zs4,/m/07t58,True +779,346,Mark Fields,organization/role/leaders./organization/leadership/person,Ford Motor Company,/m/09k8g7,/m/02zs4,True +780,346,Mark Fields,people/person/employment_history./business/employment_tenure/title,Chief executive officer,/m/09k8g7,/m/02mwvv,True +781,347,Florence Center,organization/role/leaders./organization/leadership/person,Kendall Wall Band,/m/063xbk,Kendall Wall Band,True +782,348,Michael Flynn,people/person/employment_history./business/employment_tenure/title,Lieutenant general,/m/09v12g0,/m/01cpjg,True +783,349,David Chalian,people/person/employment_history./business/employment_tenure/title,Director,/m/0zbssjv,/m/02mjmr,True +784,350,Joseph Dunford,people/person/employment_history./business/employment_tenure/title,Gene,/m/05b2r22,/m/0bscct,True +785,350,Ri Yong-ho,people/person/employment_history./business/employment_tenure/title,Foreign minister,/m/0nbhk23,/m/01t_55,True +786,350,Joseph Dunford,organization/role/leaders./organization/leadership/person,Joint Chiefs of Staff,/m/05b2r22,/m/01brfl,True +787,350,Ri Yong-ho,organization/role/leaders./organization/leadership/person,United Nations General Assembly,/m/0nbhk23,/m/07vp7,True +788,350,United States,organization/organization/headquarters./location/mailing_address/country,North Korea,/m/07t58,/m/05b7q,True +789,350,United States,organization/organization/headquarters./location/mailing_address/country,North Korea,/m/07t58,/m/05b7q,True +790,350,John Kerry,people/person/employment_history./business/employment_tenure/title,United States Secretary of State,/m/0d3qd0,/m/0bwl7t0,True +791,350,Sea,location/country/languages_spoken,China,/m/06npx,/m/0d05w3,True +792,350,Sea,location/country/languages_spoken,South,/m/06npx,/m/0k7ny,True +793,350,Workers' Party,organization/organization/headquarters./location/mailing_address/country,Korea,/m/01vlpc,/m/048fz,True +794,350,Kim Jong-un,people/person/employment_history./business/employment_tenure/title,Leadership,/m/0fgw19,/m/0h7q0rq,True +795,350,Barack Obama,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0h7q0rq,/m/02mjmr,True +796,350,Bruce+Klingner,organization/role/leaders./organization/leadership/person,The Heritage Foundation,Bruce+Klingner,/m/01tw7q,True +797,350,Sea,location/country/languages_spoken,China,/m/06npx,/m/0d05w3,True +798,350,Sea,location/country/languages_spoken,South,/m/06npx,/m/0k7ny,True +799,355,Bank,organization/organization/headquarters./location/mailing_address/country,United States,/m/017ql,/m/07t58,True +800,355,James K. Polk,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/042f1,/m/02mjmr,True +801,355,William H. Seward,people/person/employment_history./business/employment_tenure/title,Senator,/m/0k_2z,/m/04c_3b,True +802,355,William H. Seward,organization/role/leaders./organization/leadership/person,Republican Party,/m/0k_2z,/m/085srz,True +803,355,John Tyler,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/042dk,/m/02mjmr,True +804,355,Donald Trump,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0cqt90,/m/02mjmr,True +805,355,Hillary Clinton,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0d06m5,/m/02mjmr,True +806,356,Nate Smith,organization/role/leaders./organization/leadership/person,Texas secession movements,Nate Smith,Texas secession movements,True +807,356,Bashar al-Assad,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/01_vwx,/m/02mjmr,True +808,356,Peter+Kreko,organization/role/leaders./organization/leadership/person,Indiana University,Peter+Kreko,/m/08qnnv,True +809,357,Barack Obama,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0h7q0rq,/m/02mjmr,True +810,357,Barack Obama,people/person/employment_history./business/employment_tenure/title,Barack Obama,/m/0h7q0rq,/m/0h7q0rq,True +811,357,Jon+Alterman,organization/role/leaders./organization/leadership/person,Center for Strategic and International Studies,Jon+Alterman,/m/02n4mz,True +812,357,Josh Earnest,organization/role/leaders./organization/leadership/person,White House,/m/0l8ps43,/m/03mdbqx,True +813,357,Josh Earnest,people/person/employment_history./business/employment_tenure/title,Spokesperson,/m/0l8ps43,/m/03mdbqx,True +814,357,Barack Obama,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0h7q0rq,/m/02mjmr,True +815,357,Barack Obama,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0h7q0rq,/m/02mjmr,True +816,357,Hosni Mubarak,people/person/nationality,Egypt,/m/0dnps,/m/02k54,True +817,357,Barack Obama,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0h7q0rq,/m/02mjmr,True +818,357,Fahad+Nazer,organization/role/leaders./organization/leadership/person,Saudi+Embassy,Fahad+Nazer,Saudi+Embassy,True +819,358,Mitch McConnell,people/person/employment_history./business/employment_tenure/title,Leadership,/m/01z6ls,/m/0h7q0rq,True +820,358,Harry Reid,organization/role/leaders./organization/leadership/person,United States Senate,/m/0204x1,/m/07r9r0,True +821,358,John Cornyn,people/person/employment_history./business/employment_tenure/title,Whip,/m/01xcqs,/m/0250w0,True +822,358,Barack Obama,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0h7q0rq,/m/02mjmr,True +823,358,Barack Obama,people/person/nationality,Saudi Arabia,/m/0h7q0rq,/m/01z215,True +824,360,Cathy Rodgers,people/person/employment_history./business/employment_tenure/title,Rep,/m/03dlf3,/m/02qkv0r,True +825,360,Cathy McMorris Rodgers,people/person/employment_history./business/employment_tenure/title,Rep,/m/03dlf3,/m/02qkv0r,True +826,360,Journalist,people/person/employment_history./business/employment_tenure/title,Cathy McMorris Rodgers,/m/0d8qb,/m/03dlf3,True +827,361,Occupy Wall Street,organization/role/leaders./organization/leadership/person,Frank,/m/0h63jfq,Frank,True +828,362,Pat McCrory,people/person/employment_history./business/employment_tenure/title,governor,/m/05tjbz,/m/0btx2g,True +829,362,Pat McCrory,organization/role/leaders./organization/leadership/person,State,/m/05tjbz,/m/0d06m5,True +830,362,Pat McCrory,organization/role/leaders./organization/leadership/person,State,/m/05tjbz,/m/0d06m5,True +831,362,Loretta Lynch,people/person/employment_history./business/employment_tenure/title,Attorney general,/m/0gg570x,/m/0465sdr,True +832,364,White House,organization/role/leaders./organization/leadership/person,Hillary Clinton,/m/03mdbqx,/m/0d06m5,True +833,364,Barack Obama,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0h7q0rq,/m/02mjmr,True +834,364,Barack Obama,people/person/employment_history./business/employment_tenure/title,Businessperson,/m/0h7q0rq,/m/012t_z,True +835,367,Donald Trump,organization/role/leaders./organization/leadership/person,Fox,/m/0cqt90,/m/0cjdk,True +836,369,Mitch McConnell,organization/role/leaders./organization/leadership/person,United States Senate,/m/01z6ls,/m/07r9r0,True +837,369,Nancy Pelosi,people/person/employment_history./business/employment_tenure/title,Minority leader,/m/012v1t,/m/0499g1,True +838,369,Mitch McConnell,people/person/employment_history./business/employment_tenure/title,Leadership,/m/01z6ls,/m/0h7q0rq,True +839,369,Nancy Pelosi,organization/role/leaders./organization/leadership/person,House,/m/012v1t,/m/081sq,True +840,369,Ryan's World,people/person/employment_history./business/employment_tenure/title,Loudspeaker,Ryan's World,/m/0cfpc,True +841,369,Nancy Pelosi,people/person/employment_history./business/employment_tenure/title,Leadership,/m/012v1t,/m/0h7q0rq,True +842,369,Harry Reid,organization/role/leaders./organization/leadership/person,United States Senate,/m/0204x1,/m/07r9r0,True +843,369,Dick Durbin,people/person/employment_history./business/employment_tenure/title,Whip,/m/01xcd1,/m/0250w0,True +844,369,Dick Durbin,organization/role/leaders./organization/leadership/person,United States Senate,/m/01xcd1,/m/07r9r0,True +845,369,Steny Hoyer,people/person/employment_history./business/employment_tenure/title,Whip,/m/025k5p,/m/0250w0,True +846,369,Steny Hoyer,people/person/employment_history./business/employment_tenure/title,Dick Durbin,/m/025k5p,/m/01xcd1,True +847,369,Bill Cassidy,organization/role/leaders./organization/leadership/person,Republican Party,/m/0286t7r,/m/085srz,True +848,370,Jill Stein,people/person/employment_history./business/employment_tenure/title,Candidate,/m/05st_r,/m/07r9r0,True +849,370,Gary Johnson,people/person/employment_history./business/employment_tenure/title,Candidate,/m/01tqr5,/m/07r9r0,True +850,370,Jill Stein,organization/role/leaders./organization/leadership/person,Green Party of the United States,/m/05st_r,/m/07k5l,True +851,370,Donald Trump,organization/role/leaders./organization/leadership/person,Republican Party,/m/0cqt90,/m/085srz,True +852,372,Barack Obama,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0h7q0rq,/m/02mjmr,True +853,372,Donald Trump,organization/role/leaders./organization/leadership/person,Fox News,/m/0cqt90,/m/02z_b,True +854,372,Pat McCrory,people/person/employment_history./business/employment_tenure/title,governor,/m/05tjbz,/m/0btx2g,True +855,372,Kerr+Putney,people/person/employment_history./business/employment_tenure/title,Chief of police,Kerr+Putney,/m/039h8_,True +856,372,Josh Earnest,organization/role/leaders./organization/leadership/person,House,/m/0l8ps43,/m/081sq,True +857,372,Josh Earnest,people/person/employment_history./business/employment_tenure/title,White House Press Secretary,/m/0l8ps43,/m/01pt0z,True +858,372,Rudy Giuliani,people/person/employment_history./business/employment_tenure/title,Mayor,/m/06gn7,/m/03x_sks,True +859,372,Newt Gingrich,people/person/employment_history./business/employment_tenure/title,United States Speaker of the House,/m/018fzs,United States Speaker of the House,True +860,372,Shooting of Terence Crutcher,people/person/employment_history./business/employment_tenure/title,Police officer,Shooting of Terence Crutcher,/m/01t2ln,True +861,373,Barack Obama,people/person/employment_history./business/employment_tenure/title,Omran Daqneesh,/m/0h7q0rq,Omran Daqneesh,True +862,373,Barack Obama,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0h7q0rq,/m/02mjmr,True +863,374,Dean Baquet,people/person/employment_history./business/employment_tenure/title,editor,/m/02pjhgt,/m/02h6676,True +864,374,Hillary Clinton,organization/role/leaders./organization/leadership/person,Democracy,/m/0d06m5,/m/026wp,True +865,377,Barack Obama,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0h7q0rq,/m/02mjmr,True +866,378,Barack Obama,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0h7q0rq,/m/02mjmr,True +867,378,Joe Biden,people/person/employment_history./business/employment_tenure/title,Vice President of the United States,/m/012gx2,/m/0d05fv,True +868,378,Joe Biden,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/012gx2,/m/02mjmr,True +869,378,Arnold Palmer,people/person/employment_history./business/employment_tenure/title,Lists of golfers,/m/0l0cx,Lists of golfers,True +870,379,Barack Obama,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0h7q0rq,/m/02mjmr,True +871,379,Hillary Clinton,people/person/employment_history./business/employment_tenure/title,United States Secretary of State,/m/0d06m5,/m/0bwl7t0,True +872,380,Lester Holt,organization/role/leaders./organization/leadership/person,NBC Nightly News,/m/06jvj7,/m/01bncw,True +873,380,Lester Holt,organization/role/leaders./organization/leadership/person,NBC Nightly News,/m/06jvj7,/m/01bncw,True +874,381,Jens Stoltenberg,organization/role/leaders./organization/leadership/person,NATO,/m/01b8g6,/m/059dn,True +875,381,Jens Stoltenberg,people/person/employment_history./business/employment_tenure/title,Secretary-General of the United Nations,/m/01b8g6,Secretary-General of the United Nations,True +876,381,NATO,organization/role/leaders./organization/leadership/person,Jens,/m/059dn,Jens,True +877,381,Jens Stoltenberg,organization/role/leaders./organization/leadership/person,NATO,/m/01b8g6,/m/059dn,True +878,382,Donald Trump,people/person/places_lived./people/place_lived/location,City,/m/0cqt90,/m/01n32,True +879,382,Rudy Giuliani,people/person/employment_history./business/employment_tenure/title,Mayor,/m/06gn7,/m/03x_sks,True +880,384,Jeffrey DeLaurentis,people/person/employment_history./business/employment_tenure/title,Americas,Jeffrey DeLaurentis,/m/07c5l,True +881,384,Getty+Obama,people/person/employment_history./business/employment_tenure/title,Cuba,Getty+Obama,/m/0d04z6,True +882,384,Jeffrey DeLaurentis,people/person/employment_history./business/employment_tenure/title,Diplomat,Jeffrey DeLaurentis,/m/080ntlp,True +883,384,Getty+Obama,people/person/employment_history./business/employment_tenure/title,Ambassador,Getty+Obama,/m/0n46_gt,True +884,384,Jeffrey DeLaurentis,people/person/employment_history./business/employment_tenure/title,Americas,Jeffrey DeLaurentis,/m/07c5l,True +885,384,Barack Obama,people/person/employment_history./business/employment_tenure/title,Ambassador,/m/0h7q0rq,/m/0n46_gt,True +886,384,Barack Obama,people/person/employment_history./business/employment_tenure/title,Ambassador,/m/0h7q0rq,/m/0n46_gt,True +887,385,Michelle Obama,people/person/employment_history./business/employment_tenure/title,First Lady of the United States,/m/025s5v9,/m/02xy5,True +888,385,Robin Roberts,organization/role/leaders./organization/leadership/person,Good Morning America,/m/06k3_t,/m/0275kr,True +889,385,Robin Roberts,people/person/employment_history./business/employment_tenure/title,Host,/m/06k3_t,/m/04wk4z,True +890,385,Barack Obama,organization/role/leaders./organization/leadership/person,Robin Roberts,/m/0h7q0rq,/m/06k3_t,True +891,386,Gary Johnson,organization/role/leaders./organization/leadership/person,Libertarian Party,/m/01tqr5,/m/07w42,True +892,386,Jill Stein,organization/role/leaders./organization/leadership/person,Green party,/m/05st_r,/m/0kq5b,True +893,387,Stephen Anderson,people/person/employment_history./business/employment_tenure/title,Pastor,Stephen Anderson,/m/01fhsb,True +894,387,Malusi Gigaba,organization/role/leaders./organization/leadership/person,Home+Affairs,/m/0d0xkd,/m/09gbckv,True +895,387,Barack Obama,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0h7q0rq,/m/02mjmr,True +896,388,Donald Trump,people/person/places_lived./people/place_lived/location,Umpa-Lumpaer,/m/0cqt90,Umpa-Lumpaer,True +897,389,Matt Pierce,organization/role/leaders./organization/leadership/person,Los Angeles Times,/m/03yjnwr,/m/01p06z,True +898,389,Matt Pierce,organization/role/leaders./organization/leadership/person,Twitter,/m/03yjnwr,/m/0289n8t,True +899,389,Matt Pierce,people/person/employment_history./business/employment_tenure/title,Journalist,/m/03yjnwr,/m/0d8qb,True +900,390,Donald Trump,organization/role/leaders./organization/leadership/person,Republican Party,/m/0cqt90,/m/085srz,True +901,390,Josh Earnest,people/person/employment_history./business/employment_tenure/title,White House Press Secretary,/m/0l8ps43,/m/01pt0z,True +902,390,Josh Earnest,people/person/employment_history./business/employment_tenure/title,White House,/m/0l8ps43,/m/03mdbqx,True +903,390,Harry Reid,people/person/employment_history./business/employment_tenure/title,Nelson,/m/0204x1,/m/0gs0g,True +904,390,Nancy Pelosi,organization/role/leaders./organization/leadership/person,House,/m/012v1t,/m/081sq,True +905,390,Nancy Pelosi,people/person/employment_history./business/employment_tenure/title,Minority leader,/m/012v1t,/m/0499g1,True +906,392,Nicole+Mainor,people/person/employment_history./business/employment_tenure/title,Spokesperson,Nicole+Mainor,/m/03mdbqx,True +907,392,Nicole+Mainor,organization/role/leaders./organization/leadership/person,United States Secret Service,Nicole+Mainor,/m/0fynw,True +908,392,Craig+Engle,people/person/employment_history./business/employment_tenure/title,Arent Fox,Craig+Engle,/m/04cs6sm,True +909,394,Paul Ryan,people/person/employment_history./business/employment_tenure/title,United States Speaker of the House,/m/024v2j,United States Speaker of the House,True +910,394,Paul Ryan,people/person/employment_history./business/employment_tenure/title,United States Speaker of the House,/m/024v2j,United States Speaker of the House,True +911,395,Chuck Schumer,organization/role/leaders./organization/leadership/person,United States Senate,/m/01w74d,/m/07r9r0,True +912,395,Chuck Schumer,organization/role/leaders./organization/leadership/person,United States Senate,/m/01w74d,/m/07r9r0,True +913,395,Harry Reid,organization/role/leaders./organization/leadership/person,United States Senate,/m/0204x1,/m/07r9r0,True +914,395,Chuck Schumer,people/person/employment_history./business/employment_tenure/title,Senator,/m/01w74d,/m/04c_3b,True +915,395,Democratic Senatorial Campaign Committee,organization/role/leaders./organization/leadership/person,Chuck Schumer,/m/03j740,/m/01w74d,True +916,395,National Republican Senatorial Committee,organization/role/leaders./organization/leadership/person,Reid,/m/03j7vg,Reid,True +917,395,National Republican Senatorial Committee,organization/role/leaders./organization/leadership/person,Reid,/m/03j7vg,Reid,True +918,395,Dean Heller,organization/role/leaders./organization/leadership/person,National Remote Sensing Centre,/m/0g92xz,/m/04vy0g,True +919,395,John Cornyn,organization/role/leaders./organization/leadership/person,United States Senate,/m/01xcqs,/m/07r9r0,True +920,395,John Cornyn,people/person/employment_history./business/employment_tenure/title,Whip,/m/01xcqs,/m/0250w0,True +921,397,Joey Soloway,people/person/employment_history./business/employment_tenure/title,creator,/m/06qsh0,/m/0f5d2,True +922,399,Petro Poroshenko,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/08w60w,/m/02mjmr,True +923,399,Royal Dutch Shell,organization/role/leaders./organization/leadership/person,Shinzo Abe,/m/0g5vy,/m/07t7hy,True +924,399,Shinzo Abe,people/person/employment_history./business/employment_tenure/title,Prime Minister of the United Kingdom,/m/07t7hy,/m/060s9,True +925,399,Royal Dutch Shell,people/person/nationality,Shinzo Abe,/m/0g5vy,/m/07t7hy,True +926,399,Royal Dutch Shell,people/person/nationality,Shinzo Abe,/m/0g5vy,/m/07t7hy,True +927,399,Royal Dutch Shell,organization/role/leaders./organization/leadership/person,Shinzo Abe,/m/0g5vy,/m/07t7hy,True +928,399,Enrique Peña Nieto,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/07zcdm,/m/02mjmr,True +929,399,Joe Biden,people/person/employment_history./business/employment_tenure/title,Vice President of the United States,/m/012gx2,/m/0d05fv,True +930,399,Barack Obama,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0h7q0rq,/m/02mjmr,True +931,399,Sadiq Khan,people/person/employment_history./business/employment_tenure/title,Chicago,/m/060nzt,/m/01_d4,True +932,399,Sadiq+Khan+Trump,people/person/employment_history./business/employment_tenure/title,Chicago,Sadiq+Khan+Trump,/m/01_d4,True +933,399,Sadiq Khan,people/person/employment_history./business/employment_tenure/title,Donald Trump,/m/060nzt,/m/0cqt90,True +934,399,Joe Biden,people/person/employment_history./business/employment_tenure/title,Trump,/m/012gx2,/m/0cqt90,True +935,399,Sadiq Khan,people/person/employment_history./business/employment_tenure/title,Mayor,/m/060nzt,/m/03x_sks,True +936,399,Matteo Renzi,people/person/employment_history./business/employment_tenure/title,Prime Minister of the United Kingdom,/m/06w3g7j,/m/060s9,True +937,399,Bill Clinton,people/person/employment_history./business/employment_tenure/title,Secretary,/m/0157m,/m/06slz5,True +938,399,Barack Obama,people/person/employment_history./business/employment_tenure/title,United States Secretary of State,/m/0h7q0rq,/m/0bwl7t0,True +939,399,Barack Obama,people/person/employment_history./business/employment_tenure/title,Secretary,/m/0h7q0rq,/m/06slz5,True +940,399,Barack Obama,people/person/employment_history./business/employment_tenure/title,United States Secretary of State,/m/0h7q0rq,/m/0bwl7t0,True +941,399,Barack Obama,people/person/employment_history./business/employment_tenure/title,Secretary,/m/0h7q0rq,/m/06slz5,True +942,399,Robert Mugabe,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0bsfy,/m/02mjmr,True +943,399,Barack Obama,people/person/employment_history./business/employment_tenure/title,Barack Obama,/m/0h7q0rq,/m/0h7q0rq,True +944,399,Ivo Daalder,organization/role/leaders./organization/leadership/person,Chicago Council on Global Affairs,/m/02wzck2,/m/0d42gh,True +945,400,Bill de Blasio,people/person/employment_history./business/employment_tenure/title,Mayor,/m/0gjsd3,/m/03x_sks,True +946,401,Clint Eastwood,organization/role/leaders./organization/leadership/person,Facebook+TwitterHollywood,/m/0bwh6,Facebook+TwitterHollywood,True +947,402,Bill Clinton,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0157m,/m/02mjmr,True +948,402,Matteo Renzi,people/person/employment_history./business/employment_tenure/title,Prime Minister of the United Kingdom,/m/06w3g7j,/m/060s9,True +949,402,Sadiq Khan,people/person/employment_history./business/employment_tenure/title,Mayor,/m/060nzt,/m/03x_sks,True +950,402,Bill Clinton,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0157m,/m/02mjmr,True +951,402,Matteo Renzi,people/person/employment_history./business/employment_tenure/title,Prime Minister of the United Kingdom,/m/06w3g7j,/m/060s9,True +952,402,Mario+Macri,people/person/employment_history./business/employment_tenure/title,President of the United States,Mario+Macri,/m/02mjmr,True +953,403,Roger Angell,organization/role/leaders./organization/leadership/person,New+Yorker,/m/06l7l,/m/07q5n,True +954,403,Barack Obama,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0h7q0rq,/m/02mjmr,True +955,403,Donald Trump,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0cqt90,/m/02mjmr,True +956,403,West,location/country/languages_spoken,Yemen,/m/0gkh2,/m/01z88t,True +957,404,Shooting of Jeremy Mardis,people/person/employment_history./business/employment_tenure/title,Deputy,Shooting of Jeremy Mardis,/m/03yc4j2,True +958,404,William Bennett,people/person/employment_history./business/employment_tenure/title,Judge,/m/01fhz1,/m/06lvrr,True +959,405,Paul Ryan,organization/role/leaders./organization/leadership/person,CNN+House,/m/024v2j,CNN+House,True +960,408,Robert+Milichs,people/person/employment_history./business/employment_tenure/title,Judge,Robert+Milichs,/m/06lvrr,True +961,408,Burton Snowboards,organization/role/leaders./organization/leadership/person,New York Daily News,/m/0781kl,/m/02jf15,True +962,409,Bill Clinton,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0157m,/m/02mjmr,True +963,409,Marsha Blackburn,people/person/employment_history./business/employment_tenure/title,Rep,/m/01fnkt,/m/02qkv0r,True +964,409,Roger Bate,organization/role/leaders./organization/leadership/person,American Enterprise Institute,/m/0b23zg,/m/0p8q4,True +965,409,Hillary Clinton,people/person/employment_history./business/employment_tenure/title,Candidate,/m/0d06m5,/m/07r9r0,True +966,409,Rod Rosenstein,organization/role/leaders./organization/leadership/person,Rod Rosenstein,/m/04y7p0_,/m/04y7p0_,True +967,409,Rod Rosenstein,people/person/employment_history./business/employment_tenure/title,Attorney,/m/04y7p0_,/m/04gc2,True +968,409,Barack Obama,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0h7q0rq,/m/02mjmr,True +969,410,Donald Trump,people/person/employment_history./business/employment_tenure/title,Politician,/m/0cqt90,Jimmy Dimora,True +970,410,Barack Obama,people/person/employment_history./business/employment_tenure/title,Barack Obama,/m/0h7q0rq,/m/0h7q0rq,True +971,410,Barack Obama,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0h7q0rq,/m/02mjmr,True +972,410,Federal Bureau of Investigation,organization/role/leaders./organization/leadership/person,Bill Clinton,/m/02_1m,/m/0157m,True +973,410,Donald Trump,people/person/employment_history./business/employment_tenure/title,Soldier Soldier,/m/0cqt90,/m/0t7g5,True +974,410,Vladimir Putin,people/person/employment_history./business/employment_tenure/title,Dictator,/m/08193,/m/02bms,True +975,410,HES,organization/role/leaders./organization/leadership/person,Donald Trump,HES,/m/0cqt90,True +976,410,Donald Trump,people/person/employment_history./business/employment_tenure/title,Vladimir Putin,/m/0cqt90,/m/08193,True +977,410,Donald Trump,people/person/nationality,Vladimir Putin,/m/0cqt90,/m/08193,True +978,410,Bill+Turpin,people/person/places_lived./people/place_lived/location,Brayton cycle,Bill+Turpin,/m/02542p,True +979,410,Lewis Smith,people/person/employment_history./business/employment_tenure/title,"Orlando, Florida",/m/0crpt8,/m/0ply0,True +980,410,Thomas+Davidenko,organization/role/leaders./organization/leadership/person,Temple,Thomas+Davidenko,/m/08g_yr,True +981,410,Lou Smith,organization/role/leaders./organization/leadership/person,AT&T,/m/06_w579,/m/08z129,True +982,411,Bill Maher,organization/role/leaders./organization/leadership/person,HBOs+Real+Time,/m/01n4f8,/m/04sv_8,True +983,411,Bill Maher,people/person/employment_history./business/employment_tenure/title,Host,/m/01n4f8,/m/04wk4z,True +984,413,Friedrich,people/person/places_lived./people/place_lived/location,Bavaria,Friedrich,/m/017v_,True +985,413,Paul Roland,organization/role/leaders./organization/leadership/person,Palatinate,/m/0406vgn,/m/09pkf,True +986,413,Friedrich,people/person/places_lived./people/place_lived/location,Kallstadt,Friedrich,/m/02rscps,True +987,413,Friedrich,people/person/places_lived./people/place_lived/location,Kallstadt,Friedrich,/m/02rscps,True +988,413,Friedrich,people/person/places_lived./people/place_lived/location,Kallstadt,Friedrich,/m/02rscps,True +989,413,Simone Wendel,people/person/employment_history./business/employment_tenure/title,Kallstadt,/m/011x14ft,/m/02rscps,True +990,414,Charmian Carr,organization/role/leaders./organization/leadership/person,"California State University, Northridge",/m/08r7gq,/m/0k__z,True +991,414,Charmian Carr,people/person/employment_history./business/employment_tenure/title,student,/m/08r7gq,/m/014cnc,True +992,415,Bank,organization/organization/headquarters./location/mailing_address/country,United States,/m/017ql,/m/07t58,True +993,415,James K. Polk,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/042f1,/m/02mjmr,True +994,415,William H. Seward,people/person/employment_history./business/employment_tenure/title,Senator,/m/0k_2z,/m/04c_3b,True +995,415,William H. Seward,organization/role/leaders./organization/leadership/person,Republican Party,/m/0k_2z,/m/085srz,True +996,415,John Tyler,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/042dk,/m/02mjmr,True +997,415,Donald Trump,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0cqt90,/m/02mjmr,True +998,415,Hillary Clinton,people/person/employment_history./business/employment_tenure/title,President of the United States,/m/0d06m5,/m/02mjmr,True +999,416,Donald Trump,people/person/employment_history./business/employment_tenure/title,King,/m/0cqt90,/m/03w9bnr,True From 224ee6feb12edfb70215cdee495f486755ebbca1 Mon Sep 17 00:00:00 2001 From: Odd Gunnar Aspaas Date: Thu, 6 May 2021 17:13:27 +0200 Subject: [PATCH 03/12] gold standard --- data/FakeNewsNet/gs_train.csv | 133 ++++++++++++++++++++++++++++++++++ run_fn_detection.py | 2 +- 2 files changed, 134 insertions(+), 1 deletion(-) create mode 100644 data/FakeNewsNet/gs_train.csv diff --git a/data/FakeNewsNet/gs_train.csv b/data/FakeNewsNet/gs_train.csv new file mode 100644 index 0000000..06ddec7 --- /dev/null +++ b/data/FakeNewsNet/gs_train.csv @@ -0,0 +1,133 @@ +text_id,head,relation,tail,fb_head,fb_relation,fb_tail,label +280,Jefferson Davis monument,located in,New Orleans,Jefferson Davis monument,/location/location/containedby,/m/0f2tj,FALSE +280,Jefferson Davis,president of,Confederate States of America,/m/043q0,/government/government_office_category/officeholders./government/government_position_held/jurisdiction_of_office,/m/020d5,FALSE +280,Mitch Landrieu,officeholder in,New Orleans,/m/05xdp5,/government/government_office_category/officeholders./government/government_position_held/jurisdiction_of_office,/m/0f2tj,FALSE +280,Mitch Landrieu,title,Mayor,/m/05xdp5,/people/person/employment_history./business/employment_tenure/title,/m/0pqc5,FALSE +280,Barack Obama,role,US president,/m/02mjmr,/government/politician/government_positions_held./government/government_position_held/office_position_or_title,/m/060d2,FALSE +280,Barack Obama,ethnicity,African American,/m/02mjmr,/people/person/ethnicity,/m/0x67,FALSE +280,Beaureguard BillyBob Johnson,supports,Confederate States of America,Beaureguard BillyBob Johnson,/base/popstra/organization/supporter./base/popstra/support/supporter,/m/020d5,FALSE +280,Fox news,reports on ,Black Lives Matter,/m/02z_b,/base/newsevents/news_reporting_organisation/news_reports./base/newsevents/news_report/event,/m/012hdhkj,FALSE +282,Andrew Jackson,role,US president,/m/0rlz,/government/politician/government_positions_held./government/government_position_held/office_position_or_title,/m/060d2,FALSE +282,Donald Trump,reports on ?,American Civil War,/m/0cqt90,,/m/0kbq,FALSE +282,Donald Trump,interviewed in,The Red Shtick,/m/0cqt90,,The Red Shtick,FALSE +282,Donald Trump,member of,Party of Lincoln (republican party),/m/0cqt90,/government/politician/party./government/political_party_tenure/party,/m/07wbk,FALSE +282,Andrew Jackson,asociated with,Democratic Party,/m/0rlz,/government/politician/party./government/political_party_tenure/party,/m/0d075m,FALSE +282,Donald Trump,fan of,Kentucky Fried Chicken,/m/0cqt90,/food/diet_follower/follows_diet,/m/09b6t,FALSE +282,The Red Shtick,intern,Dave Robicheaux,The Red Shtick,/business/employer/employees./business/employment_tenure/person,/m/04lcmzy,FALSE +282,American Civil War,involved person,Colonel Sanders,/m/0kbq,/military/military_conflict/military_personnel_involved,/m/09b78,FALSE +282,Colonel Sanders,is,hero,/m/09b78,"/people/profession/specializations , /people/person/profession",/m/03kll,FALSE +286,Donald Trump,role,US president,/m/0cqt90,/government/politician/government_positions_held./government/government_position_held/office_position_or_title,/m/060d2,FALSE +286,United States Congress,legal charges against,Hillary Clinton,/m/07t31,,/m/0d06m5,FALSE +286,United States House Committee on Oversight and Government Reform,chairman ,Jason Chaffetz,United States House Committee on Oversight and Government Reform,organization/role/leaders./organization/leadership/person,/m/047blr4,FALSE +286,Jason Chaffetz,iterviewed,Donald Trump,/m/047blr4,,/m/0cqt90,FALSE +286,FBI,director (former),James Comey,/m/02_1m,/organization/role/leaders./organization/leadership/person,/m/06r04p,FALSE +286,Hillary Clinton,betray,United States of America,/m/0d06m5,,/m/09c7w0,FALSE +288,Freedom Crossroads,employee,Louis Leweigh,Freedom Crossroads,/business/employer/employees./business/employment_tenure/person,Louis Leweigh,FALSE +288,Louis Leweigh,occupation,Correspondent,Louis Leweigh,/people/person/profession,/m/02k13d,FALSE +288,Monica Lewinsky,affair,Bill Clinton,/m/0509p,/celebrities/celebrity/sexual_relationships./celebrities/romantic_relationship/celebrity,/m/0157m,FALSE +288,Bill Clinton,suspected murder involvement,Monica Lewinsky,/m/0509p,,/m/0509p,FALSE +295,public servant (government official),people with role,Maxine Waters,/m/035y33,/people/profession/people_with_this_profession,/m/024tyk,FALSE +295,Donald Trump,role,US president,/m/0cqt90,/government/politician/government_positions_held./government/government_position_held/office_position_or_title,/m/060d2,FALSE +295,Donald Trump,asociated with,Russia,/m/0cqt90,,/m/06bnz,FALSE +295,Maxine Waters,condemn,White House,/m/024tyk,,/m/081sq,FALSE +295,Maxine Waters,medical condition,Dementia,/m/024tyk,/medicine/notable_person_with_medical_condition/condition,/m/09klv,FALSE +295,Maxine Waters,asociated with,Democratic Party,/m/024tyk,/government/politician/party./government/political_party_tenure/party,/m/0d075m,FALSE +295,Maxine Waters,part of,United States Congress,/m/024tyk,/people/appointed_role/appointment./people/appointment/appointed_by,/m/07t31,FALSE +295,Maxine Waters,ethnicity,African American,/m/024tyk,/people/person/ethnicity,/m/0x67,FALSE +295,1tch,employee,Teddy Stick,1tch,/business/employer/employees./business/employment_tenure/person,Teddy Stick,FALSE +300,Donald Trump,cheered for,crisis (Housing Crisis),/m/0cqt90,,/m/02gyy_,TRUE +300,Hillary Clinton,critiqued,Donald Trump,/m/0d06m5,,/m/0cqt90,TRUE +300,Donald Trump,interviewed in,The Globe and Mail,/m/0cqt90,,/m/01j1gk,TRUE +300,crisis (housing),asociated with,Great Recession,/m/02gyy_,,/m/03qk63k,TRUE +300,Great Recession,affected negatively,the poor,/m/03qk63k,/influence/influence_node/influenced,the poor,TRUE +303,Jean Shepard,member of,Grand Ole Opry,/m/01npnk3,/organization/membership_organization/members./organization/organization_membership/member,/m/0gg86,TRUE +303,Jean Shepard,inducted,Country Music Hall of Fame,/m/01npnk3,/award/hall_of_fame_inductee/hall_of_fame_inductions./award/hall_of_fame_induction/hall_of_fame,/m/0ggkm,TRUE +303,Country Music Hall of Fame,hall_of_fame_dicipline,music,/m/0ggkm,/award/hall_of_fame/discipline,/m/04rlf,TRUE +303,Jean Shepard,interviewed in,Women Of Country,/m/01npnk3,,Women Of Country,TRUE +303,Jean Shepard,medical condition,Parkinsons,/m/01npnk3,/medicine/notable_person_with_medical_condition/condition,/m/0g02vk,TRUE +303,Jean Shepard,born in,Oklahoma,/m/01npnk3,/people/person/place_of_birth,/m/05mph,TRUE +303,Jean Shepard,lived in,"Visalia, California",/m/01npnk3,/people/person/places_lived./people/place_lived/location,/m/0r80l,TRUE +303,Jean Shepard,part of band,Melody Ranch Girls,/m/01npnk3,/music/musical_group/member./music/group_membership/member,Melody Ranch Girls,TRUE +303,Hank Thompson,music genre,Country Music,/m/01k3fvw,/music/artist/genre,/m/01lyv,TRUE +303,singer,people with profession,Jean Shepard,/m/09l65,/people/profession/people_with_this_profession,/m/01npnk3,TRUE +303,bassist,musical profession,Jean Shepard,/m/0gbbt,/people/profession/people_with_this_profession,/m/01npnk3,TRUE +307,Tim Kaine,associated with,Democratic Party,/m/053f8h,/government/politician/party./government/political_party_tenure/party,/m/0d075m,TRUE +307,Tim Kaine,"""running mate""",Hillary Clinton,/m/053f8h,,/m/0d06m5,TRUE +307,Tim Kaine,role,Senator,/m/053f8h,/people/person/employment_history./business/employment_tenure/title,/m/048zv9l,TRUE +307,Tim Kaine,lives in ,Virginia,/m/053f8h,/people/person/places_lived./people/place_lived/location,/m/07z1m,TRUE +307,Tim Kaine,works in,Virginia,/m/053f8h,,/m/07z1m,TRUE +307,Tim Kaine,interviewed in,The Ellen DeGeneres Show,/m/053f8h,,/m/0528kc,TRUE +307,Tim Kaine,nominee,Vice President of the U.S,/m/053f8h,,/m/080v2,TRUE +307,Mike Pence,asociated with ,Donald Trump,/m/022r9r,,/m/0cqt90,TRUE +307,Tim Kaine,grew up as,Republican Party,/m/053f8h,,/m/07wbk,TRUE +307,Hillary Clinton,grew up as,Republican Party,/m/0d06m5,,/m/07wbk,TRUE +308,Hofstra University,located in,New York,/m/02607j,/location/location/containedby,/m/02_286,TRUE +308,Not Who We Are campaign,is against,Donald Trump,Not Who We Are campaign,,/m/0cqt90,TRUE +308,Not Who We Are campaign,campain at location,Hofstra University,Not Who We Are campaign,/location/location/containedby,/m/02607j,TRUE +308,Donald Trump,is part of ,racism,/m/0cqt90,,/m/06d4h,TRUE +308,Not Who We Are campaign,manager of,Josh Hendler,Not Who We Are campaign,organization/role/leaders./organization/leadership/person,Josh Hendler,TRUE +308,Rob Bielunas,student at,Hofstra University,Rob Bielunas,/people/person/education./education/education/institution,/m/02607j,TRUE +308,Donald Trump,role,presidential candidate,/m/0cqt90,/government/politician/government_positions_held./government/government_position_held/office_position_or_title,/m/066q5c,TRUE +328,Donald Trump,is part of,racism,/m/0cqt90,,racist,TRUE +328,Barack Obama,role,US president,/m/02mjmr,/government/politician/government_positions_held./government/government_position_held/office_position_or_title,/m/060d2,TRUE +328,Barack Obama,born in,Hawaii,/m/02mjmr,/people/person/place_of_birth,/m/03gh4,TRUE +328,Lester Holt,profession,moderator,/m/06jvj7,,moderator,TRUE +328,Donald Trump,role,presidential candidate,/m/0cqt90,/government/politician/government_positions_held./government/government_position_held/office_position_or_title,/m/066q5c,TRUE +328,Donald Trump,associated with,Republican Party,/m/0cqt90,/government/politician/party./government/political_party_tenure/party,/m/07wbk,TRUE +328,Donald Trump,role,presidential candidate,/m/0cqt90,/government/politician/government_positions_held./government/government_position_held/office_position_or_title,/m/066q5c,TRUE +328,Hillary Clinton,role,presidential candidate,/m/0d06m5,/government/politician/government_positions_held./government/government_position_held/office_position_or_title,/m/066q5c,TRUE +2,HIVAIDs drug,cause,death,/m/01cw8w,/medicine/disease/causes,/m/0294j,FALSE +2,Bill Clinton,associates with,Ranbaxy,/m/0157m,/base/popstra/celebrity/endorsements./base/popstra/paid_support/company,/m/0243xw,FALSE +2,Ranbaxy,banned from,The united states of america,/m/0243xw,,/m/09c7w0,FALSE +2,Bill Clinton,Chairman,Clinton Health Access Initiative,/m/0157m,/organization/organization/board_members./organization/organization_board_membership/member,Clinton Health Access Initiative,FALSE +2,Bill Clinton,involved with,corruption,/m/0157m,,/m/09pngm,FALSE +3,Donald Trump,encourage ,manslaugther,/m/0cqt90,/influence/influence_node/influenced,/m/0x2gl,FALSE +3,Hillary Clinton,supports,open border,/m/0d06m5,,/m/043l_lj,FALSE +3,Syria,will destory,The united states of america,/m/06vbd,,/m/09c7w0,FALSE +3,Donald Trump,ethnicity,Oompa Loompa,/m/0cqt90,/fictional_universe/fictional_character/species,Oompa Loompa,FALSE +3,Donald Trump,supports,concentration camps,/m/0cqt90,,/m/01b5_d,FALSE +8,Hillary Clinton,collaboration,CNN,/m/0d06m5,/influence/influence_node/influenced_by,/m/0gsgr,FALSE +8,Hillary Clinton,medical condition,Parkinsons disease,/m/0d06m5,/medicine/notable_person_with_medical_condition/condition,/m/0g02vk,FALSE +8,Hillary Clinton,involved with,corruption,/m/0d06m5,,/m/09pngm,FALSE +8,Democratic Party,oppose,1964 Civil Rights Act,/m/0d075m,,/m/019twb,FALSE +8,Democratic Party,fought to maintain,Slavery,/m/0d075m,,/m/06z49,FALSE +8,Republican Party,oppose ,Slavery,/m/0d075m,,/m/06z49,FALSE +8,Bill Clinton,is,sexual predator,/m/0157m,/base/fight/crime_type/people_convicted_of_this_crime./base/crime/criminal_conviction/guilty_of,/m/01fdjk,FALSE +8,Hillary Clinton,is ,crime,/m/0d06m5,,/m/01rbb,FALSE +8,Hillary Clinton,is,racism,/m/0d06m5,/base/fight/crime_type/people_convicted_of_this_crime./base/crime/criminal_conviction/guilty_of,/m/06d4h,FALSE +11,Barrack Obama,created,ISIS,/m/02mjmr,,/m/027x630,FALSE +11,Europe,supports,ISIS,/m/02j9z,/base/popstra/celebrity/supporter./base/popstra/support/supported_organization,/m/027x630,FALSE +11,The United States,supports,ISIS,/m/09c7w0,/base/popstra/celebrity/supporter./base/popstra/support/supported_organization,/m/027x630,FALSE +14,ISIS,creator,Barrack Obama,/m/027x630,/organization/organization/founders,/m/02mjmr,FALSE +14,Barrack Obama,appear at,Coachella,/m/02mjmr,,/m/02h71z,FALSE +14,Michelle Obama,medical condition,sleep apnea,/m/025s5v9,/medicine/notable_person_with_medical_condition/condition,/m/071d3,FALSE +14,Barrack Obama,loves,electronic music,/m/02mjmr,,/m/02lkt,FALSE +14,Barrack Obama,uses,ecstasy,/m/02mjmr,,/m/02qpq,FALSE +14,Barrack Obama,profession,terrorism,/m/02mjmr,/people/person/profession,/m/07jq_,FALSE +14,Barrack Obama,place of birth,Kenya,/m/02mjmr,/people/person/place_of_birth,/m/019rg5,FALSE +14,Barrack Obama,religion,Islam,/m/02mjmr,/people/person/religion,/m/0flw86,FALSE +14,Barrack Obama,guilty of,crime of possessing illegal drugs,/m/02mjmr,/base/fight/crime_type/people_convicted_of_this_crime./base/crime/criminal_conviction/guilty_of,/m/05cxz3,FALSE +14,Hillary Clinton,guilty of,crime of possessing illegal drugs,/m/0d06m5,/base/fight/crime_type/people_convicted_of_this_crime./base/crime/criminal_conviction/guilty_of,/m/05cxz3,FALSE +119,Megan Rapinoe,profession,soccer player,/m/05mz656,/people/person/profession,/m/0pcq81q,TRUE +119,Megan Rapinoe,supports,BLM,/m/05mz656,/base/popstra/celebrity/supporter./base/popstra/support/supported_organization,/m/012hdhkj,TRUE +119,Colin Kaepernick,profession,american football player,/m/03cm6b3,/people/person/profession,/m/02h665k,TRUE +119,Colin Kaepernick,supports,BLM,/m/03cm6b3,/base/popstra/celebrity/supporter./base/popstra/support/supported_organization,/m/012hdhkj,TRUE +138,Barry Marshall,nobel prize in,medicine,/m/023fbk,/base/nobelprizes/nobel_subject_area/nobel_awards./base/nobelprizes/nobel_honor/nobel_prize_winner,/m/04sh3,TRUE +138,Jonas Salk,pioneer,vaccine,/m/0g60g,/law/inventor/inventions,/m/07__7,TRUE +138,Michael Oreskes,profession,News Director,Michael Oreskes,/people/person/profession,/m/014l7h,TRUE +173,Joa Arpaio,profession,Sheriff,/m/0fzdz7,/people/person/profession,m/0mb31,TRUE +173,Donald Trump,associated with,GOP (republican party),/m/0cqt90,/government/politician/party./government/political_party_tenure/party,/m/07wbk,TRUE +173,Donald Trump,role,presidential candidate,/m/0cqt90,/government/politician/government_positions_held./government/government_position_held/office_position_or_title,/m/066q5c,TRUE +173,Barack Obama,born in,Hawaii,/m/02mjmr,/people/person/place_of_birth,/m/03gh4,TRUE +174,Benjamin Netanyahu,profession,prime minister,/m/0fm2h,/government/politician/government_positions_held./government/government_position_held/office_position_or_title,/m/060bp,TRUE +174,Benjamin Netanyahu,place of birth,Israel,/m/0fm2h,/people/person/place_of_birth,/m/03spz,TRUE +174,The United States,alliance,Israel,/m/09c7w0,,/m/03spz,TRUE +174,Hillary Clinton,associated with,Democratic Party,/m/0d06m5,/government/politician/party./government/political_party_tenure/party,/m/0d075m,TRUE +174,Hillary Clinton,role,presidential candidate,/m/0d06m5,/government/politician/government_positions_held./government/government_position_held/office_position_or_title,/m/066q5c,TRUE +178,Ben LaBolt,press secretary,Obamas 2012 reelection campaign,Ben LaBolt,,/m/0gk_h41,TRUE +178,Josh Ernest,press secretary,The White House,/m/0l8ps43,,/m/081sq,TRUE +178,Hillary Clinton,associated with,Democratic Party,/m/0d06m5,/government/politician/party./government/political_party_tenure/party,/m/0d075m,TRUE +178,Hillary Clinton,role,presidential candidate,/m/0d06m5,/government/politician/government_positions_held./government/government_position_held/office_position_or_title,/m/066q5c,TRUE +178,Barrack Obama,profession,president,/m/02mjmr,/government/politician/government_positions_held./government/government_position_held/office_position_or_title,/m/060d2,TRUE +178,Donald Trump,associated with,Republican Party,/m/0cqt90,/government/politician/party./government/political_party_tenure/party,/m/07wbk,TRUE +178,Donald Trump,role,presidential candidate,/m/0cqt90,/government/politician/government_positions_held./government/government_position_held/office_position_or_title,/m/066q5c,TRUE \ No newline at end of file diff --git a/run_fn_detection.py b/run_fn_detection.py index 779c8fd..52ba816 100644 --- a/run_fn_detection.py +++ b/run_fn_detection.py @@ -77,7 +77,7 @@ def load_data(news_dataset, kg_dataset, dataset_name, hidden=64, node_embedding_ print("Loading fake news dataset: " + news_dataset + "...") fn_test = pd.read_csv( - "./data/" + news_dataset + "/train.csv", + "./data/" + news_dataset + "/gs_train.csv", sep=",", header=0, engine="python", From 2ecf38b64b9aaf7b60130fd2285d06ec70fc604c Mon Sep 17 00:00:00 2001 From: Odd Gunnar Aspaas Date: Tue, 25 May 2021 11:56:19 +0200 Subject: [PATCH 04/12] symmetry --- run_fn_detection.py | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/run_fn_detection.py b/run_fn_detection.py index 52ba816..0768ff3 100644 --- a/run_fn_detection.py +++ b/run_fn_detection.py @@ -28,20 +28,29 @@ def train_val_test_split(dataset, train, val, test): dataset.test_edge_type = test.edge_type # negatives - neg_train = structured_negative_sampling(dataset.train_pos_edge_index) + # nb! add symmetry before + anti_symmetric_train = torch.flip(dataset.train_pos_edge_index,[0]) + symmetric_train = torch.cat([dataset.train_pos_edge_index, anti_symmetric_train], dim=1) + + neg_train = structured_negative_sampling(symmetric_train) dataset.train_neg_edge_index = torch.tensor( [list(neg_train[0]), list(neg_train[2])], dtype=torch.long - ) + ).narrow(1,0,anti_symmetric_train.size(1)) + + anti_symmetric_val = torch.flip(dataset.val_pos_edge_index,[0]) + symmetric_val = torch.cat([dataset.val_pos_edge_index, anti_symmetric_val], dim=1) - neg_val = structured_negative_sampling(dataset.val_pos_edge_index) + neg_val = structured_negative_sampling(symmetric_val) dataset.val_neg_edge_index = torch.tensor( [list(neg_val[0]), list(neg_val[2])], dtype=torch.long - ) + ).narrow(1,0,anti_symmetric_val.size(1)) - neg_test = structured_negative_sampling(dataset.test_pos_edge_index) + anti_symmetric_test = torch.flip(dataset.test_pos_edge_index,[0]) + symmetric_test = torch.cat([dataset.test_pos_edge_index, anti_symmetric_test], dim=1) + neg_test = structured_negative_sampling(symmetric_test) dataset.test_neg_edge_index = torch.tensor( [list(neg_test[0]), list(neg_test[2])], dtype=torch.long - ) + ).narrow(1,0,anti_symmetric_test.size(1)) return dataset From 068a5e3913aac39b52a5e4f3785c37fb3cde6078 Mon Sep 17 00:00:00 2001 From: Odd Gunnar Aspaas Date: Tue, 25 May 2021 14:25:11 +0200 Subject: [PATCH 05/12] make new folder each run --- data/FakeNewsNet/train_both.csv | 737 ++++++++++++++++++++++++++++++++ run_fn_detection.py | 31 +- trainer.py | 10 +- utils/misc.py | 7 + 4 files changed, 764 insertions(+), 21 deletions(-) create mode 100644 data/FakeNewsNet/train_both.csv create mode 100644 utils/misc.py diff --git a/data/FakeNewsNet/train_both.csv b/data/FakeNewsNet/train_both.csv new file mode 100644 index 0000000..2126487 --- /dev/null +++ b/data/FakeNewsNet/train_both.csv @@ -0,0 +1,737 @@ +,fb_head,fb_relation,fb_tail +0,/m/0157m,organization/role/leaders./organization/leadership/person,/m/0c0sl +1,/m/0157m,people/person/employment_history./business/employment_tenure/title,/m/02mjmr +2,/m/01fnkt,people/person/employment_history./business/employment_tenure/title,/m/02qkv0r +3,/m/0b23zg,organization/role/leaders./organization/leadership/person,/m/0p8q4 +4,/m/0d06m5,people/person/employment_history./business/employment_tenure/title,/m/07r9r0 +5,/m/04y7p0_,organization/role/leaders./organization/leadership/person,/m/04y7p0_ +6,/m/04y7p0_,people/person/employment_history./business/employment_tenure/title,/m/04gc2 +7,/m/0h7q0rq,people/person/employment_history./business/employment_tenure/title,/m/02mjmr +8,HES,organization/role/leaders./organization/leadership/person,Getty+ImagesJoe+Raedle +9,/m/0h7q0rq,organization/role/leaders./organization/leadership/person,/m/07t65 +10,/m/0h7q0rq,people/person/employment_history./business/employment_tenure/title,Alex+Pfeiffer +11,/m/0h7q0rq,organization/role/leaders./organization/leadership/person,/m/0h7q0rq +12,/m/01_vwx,people/person/employment_history./business/employment_tenure/title,/m/02mjmr +13,/m/0cb1tz,people/person/employment_history./business/employment_tenure/title,/m/0d8qb +14,/m/0h7q0rq,organization/organization/headquarters./location/mailing_address/country,/m/0v74 +15,/m/0fynw,organization/role/leaders./organization/leadership/person,/m/0d06m5 +16,/m/0d06m5,organization/role/leaders./organization/leadership/person,/m/0fynw +17,/m/0h7q0rq,people/person/places_lived./people/place_lived/location,/m/0g60z +18,/m/0fynw,organization/role/leaders./organization/leadership/person,/m/0h7q0rq +19,/m/0chw_,people/person/places_lived./people/place_lived/location,/m/03m7gdr +20,/m/0157m,people/person/employment_history./business/employment_tenure/title,/m/06slz5 +21,/m/0268g3w,people/person/employment_history./business/employment_tenure/title,/m/02g5lp +22,/m/07j6ty,people/person/employment_history./business/employment_tenure/title,/m/04c_3b +23,Michelle,people/person/employment_history./business/employment_tenure/title,/m/02xy5 +24,/m/01ndfjw,people/person/employment_history./business/employment_tenure/title,/m/01g2j2 +25,/m/04ykg,organization/role/leaders./organization/leadership/person,/m/017bdl +26,/m/05f9m8h,people/person/employment_history./business/employment_tenure/title,/m/03x_sks +27,/m/0gsksn,organization/role/leaders./organization/leadership/person,/m/04ykg +28,/m/012bzwng,people/person/religion,/m/01lp8 +29,/m/02wx77q,people/person/employment_history./business/employment_tenure/title,/m/02mwvv +30,/m/02wx77q,organization/role/leaders./organization/leadership/person,/m/09gd5v1 +31,/m/02x2tr8,organization/role/leaders./organization/leadership/person,/m/02_1m +32,/m/06r04p,people/person/employment_history./business/employment_tenure/title,/m/02mjmr +33,/m/06r04p,organization/role/leaders./organization/leadership/person,/m/02_1m +34,/m/0d06m5,people/person/employment_history./business/employment_tenure/title,/m/05_xks +35,/m/0dtj5,organization/role/leaders./organization/leadership/person,/m/01zty6 +36,/m/0d06m5,organization/role/leaders./organization/leadership/person,/m/0dtj5 +37,/m/0h7q0rq,organization/role/leaders./organization/leadership/person,/m/02_1m +38,/m/02mjmr,people/person/employment_history./business/employment_tenure/title,/m/02mjmr +39,/m/03mdbqx,people/person/places_lived./people/place_lived/location,/m/0d06m5 +40,/m/0157m,organization/role/leaders./organization/leadership/person,/m/02_1m +41,/m/0jl0g,people/person/employment_history./business/employment_tenure/title,/m/01fch0 +42,Jason+Falconer,people/person/employment_history./business/employment_tenure/title,Instructor +43,/m/01lk__r,people/person/employment_history./business/employment_tenure/title,/m/02mjmr +44,/m/01lk__r,organization/role/leaders./organization/leadership/person,/m/0138n5_r +45,/m/0157m,people/person/places_lived./people/place_lived/location,/m/08gb40 +46,/m/0d06m5,people/person/employment_history./business/employment_tenure/title,Jason+Falconer +47,/m/02x2tr8,people/person/employment_history./business/employment_tenure/title,Google Assistant +48,/m/021583,organization/role/leaders./organization/leadership/person,/m/03mdbqx +49,TwitterBernard+Sansaricq,organization/role/leaders./organization/leadership/person,SHARES+Facebook +50,TwitterBernard+Sansaricq,organization/role/leaders./organization/leadership/person,/m/07t58 +51,Adam+Gitlin,organization/role/leaders./organization/leadership/person,/m/02pjfvw +52,/m/02pjfvw,organization/role/leaders./organization/leadership/person,Adam+Gitlin +53,/m/0d8qb,people/person/employment_history./business/employment_tenure/title,/m/059yj +54,/m/0h7q0rq,organization/organization/headquarters./location/mailing_address/country,/m/07c5l +55,/m/0685rw,organization/role/leaders./organization/leadership/person,National+Border+Patrol+Council+Local+2455 +56,/m/0j5pc,people/person/employment_history./business/employment_tenure/title,/m/04c_3b +57,/m/094z8,location/hud_county_place/county,/m/0czf4m8 +58,/m/05sj2z,people/person/employment_history./business/employment_tenure/title,/m/06b1q +59,/m/0cqt90,organization/role/leaders./organization/leadership/person,/m/02z_b +60,/m/07c_l,organization/role/leaders./organization/leadership/person,/m/0cqt90 +61,/m/0cxslf,people/person/employment_history./business/employment_tenure/title,pollster +62,Greg+Pollowitz,organization/role/leaders./organization/leadership/person,/m/01dgg6 +63,/m/04glwxr,organization/role/leaders./organization/leadership/person,/m/03d5lmj +64,/m/04glwxr,people/person/employment_history./business/employment_tenure/title,/m/071rm +65,/m/04glwxr,people/person/employment_history./business/employment_tenure/title,/m/0377k9 +66,/m/04glwxr,organization/role/leaders./organization/leadership/person,Missiongathering+Christian+Church +67,/m/08193,people/person/employment_history./business/employment_tenure/title,/m/06bnz +68,/m/027nvsp,organization/role/leaders./organization/leadership/person,/m/04b6p +69,/m/0cqt90,people/person/employment_history./business/employment_tenure/title,/m/02mjmr +70,/m/0f8t6k,people/person/employment_history./business/employment_tenure/title,/m/0btx2g +71,/m/08cnqc,people/person/employment_history./business/employment_tenure/title,/m/0cfpc +72,/m/0cqt90,organization/role/leaders./organization/leadership/person,2020 Republican Party presidential primaries +73,/m/014df6,people/person/employment_history./business/employment_tenure/title,/m/0btx2g +74,/m/019x9z,people/person/employment_history./business/employment_tenure/title,/m/0btx2g +75,/m/0d05fv,people/person/employment_history./business/employment_tenure/title,/m/0btx2g +76,/m/02w8m6,organization/role/leaders./organization/leadership/person,/m/085srz +77,/m/02w8m6,people/person/employment_history./business/employment_tenure/title,/m/012t_z +78,/m/02w8m6,organization/role/leaders./organization/leadership/person,/m/06pwq +79,/m/09g6z41,organization/role/leaders./organization/leadership/person,/m/07gmmy +80,Lisa Gordon-Hagerty,people/person/employment_history./business/employment_tenure/title,/m/0bwkm08 +81,/m/0d06m5,people/person/employment_history./business/employment_tenure/title,State.Well +82,/m/0d06m5,people/person/employment_history./business/employment_tenure/title,/m/06slz5 +83,John Pfaff,organization/role/leaders./organization/leadership/person,/m/027kp3 +84,/m/0d06m5,people/person/employment_history./business/employment_tenure/title,/m/0bwl7t0 +85,/m/0bydx6,people/person/employment_history./business/employment_tenure/title,/m/06lvrr +86,/m/026swz9,people/person/employment_history./business/employment_tenure/title,/m/025whr1 +87,/m/02x2tr8,people/person/employment_history./business/employment_tenure/title,/m/0820qp +88,/m/0120w9bh,people/person/employment_history./business/employment_tenure/title,/m/0157m +89,/m/0120w9bh,people/person/employment_history./business/employment_tenure/title,/m/07g7bl +90,Kyle,people/person/employment_history./business/employment_tenure/title,/m/0ptt0 +91,/m/0343h,people/person/employment_history./business/employment_tenure/title,/m/03r8gp +92,Juanita Irizarry,people/person/employment_history./business/employment_tenure/title,/m/0cqf2_ +93,/m/05557v4,people/person/employment_history./business/employment_tenure/title,/m/0130xz +94,/m/07xg1,organization/organization/headquarters./location/mailing_address/country,/m/0jdd +95,/m/05557v4,people/person/employment_history./business/employment_tenure/title,/m/01gzh0 +96,/m/035rnz,people/person/employment_history./business/employment_tenure/title,/m/014g_s +97,/m/0257nv,organization/role/leaders./organization/leadership/person,/m/0bymv +98,/m/0257nv,organization/role/leaders./organization/leadership/person,/m/0cqt90 +99,/m/07rlz7,people/person/employment_history./business/employment_tenure/title,peaceout.com +100,/m/07rlz7,people/person/employment_history./business/employment_tenure/title,Organizer +101,O'Brien,organization/role/leaders./organization/leadership/person,/m/01wcxw +102,/m/05tjbz,people/person/employment_history./business/employment_tenure/title,/m/0btx2g +103,/m/06r04p,people/person/employment_history./business/employment_tenure/title,/m/02jknp +104,/m/045m16,organization/role/leaders./organization/leadership/person,/m/02_1m +105,Joseph diGenova,people/person/employment_history./business/employment_tenure/title,/m/04gc2 +106,/m/0h7q0rq,organization/organization/headquarters./location/mailing_address/country,/m/01z215 +107,USA+Politics+Today,organization/role/leaders./organization/leadership/person,/m/0157m +108,USA+Politics+Today,organization/role/leaders./organization/leadership/person,/m/0cqt90 +109,/m/0dsdp74,people/person/employment_history./business/employment_tenure/title,/m/0157m +110,/m/0dsdp74,people/person/employment_history./business/employment_tenure/title,/m/03mdbqx +111,/m/02pjpd,people/person/employment_history./business/employment_tenure/title,/m/0btx2g +112,/m/0h7q0rq,organization/role/leaders./organization/leadership/person,/m/07vp7 +113,/m/02607j,organization/organization/headquarters./location/mailing_address/country,/m/03s0c +114,Donald,people/person/employment_history./business/employment_tenure/title,/m/0h7q0rq +115,/m/03jmnt,people/person/employment_history./business/employment_tenure/title,/m/02mjmr +116,Jennifer Roberts,people/person/employment_history./business/employment_tenure/title,/m/03x_sks +117,/m/0nb_v,organization/organization/headquarters./location/mailing_address/country,/m/0gsg7 +118,/m/0y4zr4f,people/person/employment_history./business/employment_tenure/title,/m/0157m +119,/m/0y4zr4f,people/person/employment_history./business/employment_tenure/title,/m/03xk7t +120,/m/0y4zr4f,organization/role/leaders./organization/leadership/person,/m/09d5h +121,/m/0dq819,people/person/employment_history./business/employment_tenure/title,/m/03xk7t +122,/m/0dq819,people/person/employment_history./business/employment_tenure/title,/m/0cqt90 +123,/m/0157m,organization/role/leaders./organization/leadership/person,/m/085srz +124,/m/049_zz,organization/role/leaders./organization/leadership/person,/m/06jvj7 +125,/m/0fqmphx,people/person/employment_history./business/employment_tenure/title,/m/01pt0z +126,Corey Lewandowski,people/person/employment_history./business/employment_tenure/title,/m/0cqt90 +127,Corey Lewandowski,people/person/employment_history./business/employment_tenure/title,/m/03xk7t +128,Mook,organization/role/leaders./organization/leadership/person,NBCs+Today.Asked +129,/m/0b9ndp,people/person/employment_history./business/employment_tenure/title,/m/07r9r0 +130,/m/0bxhx,people/person/religion,/m/0q4tw +131,/m/0204x1,organization/role/leaders./organization/leadership/person,List of current United States senators +132,/m/05k7sb,organization/role/leaders./organization/leadership/person,/m/085srz +133,/m/0204x1,people/person/employment_history./business/employment_tenure/title,/m/0h7q0rq +134,/m/0c_md_,people/person/places_lived./people/place_lived/location,/m/06c0j +135,/m/0cqt90,organization/role/leaders./organization/leadership/person,/m/085srz +136,/m/01v6nf,organization/role/leaders./organization/leadership/person,/m/05k7sb +137,/m/02n8p2,organization/role/leaders./organization/leadership/person,/m/0gsgr +138,/m/0px38,organization/role/leaders./organization/leadership/person,/m/06v2j5 +139,/m/0wbh29p,people/person/employment_history./business/employment_tenure/title,CPL +140,/m/0wbh29p,people/person/places_lived./people/place_lived/location,/m/0nzlp +141,/m/0wbh29p,people/person/employment_history./business/employment_tenure/title,/m/0nzlp +142,/m/02qlgwq,organization/role/leaders./organization/leadership/person,/m/02z_b +143,/m/07j6ty,people/person/employment_history./business/employment_tenure/title,John+F.+Kennedy+Cruz +144,/m/0d3k14,people/person/employment_history./business/employment_tenure/title,/m/02mjmr +145,/m/0b1qpc,people/person/employment_history./business/employment_tenure/title,/m/02k13d +146,/m/0n3ll,people/person/places_lived./people/place_lived/location,/m/0157m +147,/m/0157m,organization/role/leaders./organization/leadership/person,/m/0cv_2 +148,/m/0d06m5,organization/role/leaders./organization/leadership/person,/m/0289n8t +149,/m/081yw,people/person/employment_history./business/employment_tenure/title,/m/0d3k14 +150,/m/0d3k14,people/person/employment_history./business/employment_tenure/title,/m/06hx2 +151,/m/06hx2,people/person/employment_history./business/employment_tenure/title,/m/0465sdr +152,/m/08193,people/person/employment_history./business/employment_tenure/title,/m/02mjmr +153,/m/0bphp,people/person/employment_history./business/employment_tenure/title,/m/0h7q0rq +154,/m/0f7fy,people/person/employment_history./business/employment_tenure/title,/m/02mjmr +155,/m/0f7fy,people/person/employment_history./business/employment_tenure/title,/m/01crd5 +156,/m/0cnyrfq,organization/role/leaders./organization/leadership/person,/m/0d06m5 +157,/m/06npx,location/country/languages_spoken,/m/0k7ny +158,/m/06npx,location/country/languages_spoken,/m/0d05w3 +159,/m/013y1y,people/person/employment_history./business/employment_tenure/title,/m/02j9z +160,/m/013y1y,people/person/employment_history./business/employment_tenure/title,/m/0c01v +161,Bryce+Heinlein,people/person/employment_history./business/employment_tenure/title,/m/01g2j2 +162,/m/03cm6b3,people/person/employment_history./business/employment_tenure/title,/m/06b1q +163,/m/05cljf,organization/role/leaders./organization/leadership/person,/m/0kc6x +164,/m/05yz6j,people/person/employment_history./business/employment_tenure/title,/m/014g_s +165,/m/03n1q0,organization/role/leaders./organization/leadership/person,/m/0cv_2 +166,/m/03n1q0,people/person/employment_history./business/employment_tenure/title,/m/05_xks +167,/m/01cy4y,organization/organization/headquarters./location/mailing_address/country,/m/07ssc +168,/m/012gx2,people/person/employment_history./business/employment_tenure/title,/m/0d05fv +169,/m/06c0j,people/person/employment_history./business/employment_tenure/title,/m/02mjmr +170,/m/05st_r,people/person/employment_history./business/employment_tenure/title,/m/07r9r0 +171,/m/05st_r,organization/role/leaders./organization/leadership/person,/m/07k5l +172,Evelyn+Arnold,people/person/employment_history./business/employment_tenure/title,/m/07s82n4 +173,W. Shannon Morris,people/person/employment_history./business/employment_tenure/title,/m/07s82n4 +174,Ahmad+Khan+RahamiHall,organization/role/leaders./organization/leadership/person,/m/06v0py +175,/m/0d06m5,people/person/employment_history./business/employment_tenure/title,/m/03xk7t +176,/m/0d06m5,organization/role/leaders./organization/leadership/person,/m/0gsgr +177,/m/02lhtf,people/person/employment_history./business/employment_tenure/title,/m/04wk4z +178,/m/0dq819,organization/role/leaders./organization/leadership/person,/m/0h7q0rq +179,/m/0dq819,people/person/employment_history./business/employment_tenure/title,/m/0dq819 +180,/m/0gsgr,organization/organization/headquarters./location/mailing_address/country,/m/0f8t6k +181,/m/0h7q0rq,organization/role/leaders./organization/leadership/person,United+Nations+CNN +182,/m/02_fg6,organization/role/leaders./organization/leadership/person,/m/02m1b +183,/m/0d3qd0,people/person/employment_history./business/employment_tenure/title,/m/0bwl7t0 +184,Margaret+Huang,organization/role/leaders./organization/leadership/person,Amnesty+Internationals+Interim+Executive +185,/m/02_fg6,people/person/employment_history./business/employment_tenure/title,/m/02mjmr +186,/m/0d06m5,people/person/religion,/m/0flw86 +187,/m/0b2xfq,organization/role/leaders./organization/leadership/person,/m/0bwfn +188,/m/0b2xfq,people/person/employment_history./business/employment_tenure/title,/m/016fly +189,/m/0dkr2r,organization/role/leaders./organization/leadership/person,/m/04tpcx +190,/m/023fbk,people/person/employment_history./business/employment_tenure/title,/m/02g9n +191,/m/015h4y,organization/role/leaders./organization/leadership/person,/m/0hhqj3p +192,Michael Oreskes,organization/role/leaders./organization/leadership/person,NPR+News +193,Michael Oreskes,people/person/employment_history./business/employment_tenure/title,/m/02jknp +194,/m/0_yglzq,people/person/employment_history./business/employment_tenure/title,/m/0_yglzq +195,/m/094z8,location/hud_county_place/county,/m/094z8 +196,Erik+J.+Heipt,people/person/employment_history./business/employment_tenure/title,/m/04gc2 +197,/m/0cqt90,organization/role/leaders./organization/leadership/person,AP+Photo+Trump +198,/m/0cqt90,organization/role/leaders./organization/leadership/person,/m/0gsgr +199,/m/02qg4z,people/person/employment_history./business/employment_tenure/title,/m/02mjmr +200,/m/05y6yms,people/person/employment_history./business/employment_tenure/title,Pack2Go+Europe +201,Kerr+Putney,people/person/employment_history./business/employment_tenure/title,/m/039h8_ +202,/m/02wsxy,people/person/employment_history./business/employment_tenure/title,/m/0htp +203,/m/02mjmr,people/person/employment_history./business/employment_tenure/title,/m/0cqt90 +204,Gabriel Gorenstein,people/person/employment_history./business/employment_tenure/title,/m/05b105r +205,/m/05b2r22,people/person/employment_history./business/employment_tenure/title,/m/0bscct +206,/m/01brfl,organization/role/leaders./organization/leadership/person,/m/05b2r22 +207,/m/0xnc3,people/person/places_lived./people/place_lived/location,/m/0j5g9 +208,/m/024p5b,organization/role/leaders./organization/leadership/person,RepublicanWashington+CNN +209,AP+Photo+Trump,organization/role/leaders./organization/leadership/person,/m/0157m +210,Monday.David+Wildstein,organization/role/leaders./organization/leadership/person,Interstate+Capital+Projects +211,/m/0803htk,people/person/employment_history./business/employment_tenure/title,/m/0820qp +212,Bob+Durando,people/person/employment_history./business/employment_tenure/title,/m/019hy +213,/m/01vswwx,organization/role/leaders./organization/leadership/person,/m/09d5h +214,Susan+MacManus,organization/role/leaders./organization/leadership/person,/m/09s5q8 +215,Susan+MacManus,people/person/employment_history./business/employment_tenure/title,/m/062z7 +216,/m/053f8h,people/person/employment_history./business/employment_tenure/title,/m/07r9r0 +217,Neema+Hakim,people/person/employment_history./business/employment_tenure/title,/m/03mdbqx +218,John P. Roth,people/person/employment_history./business/employment_tenure/title,/m/0kvg94 +219,/m/0j42f1z,organization/role/leaders./organization/leadership/person,Tim Baker +220,/m/05c3rfn,organization/role/leaders./organization/leadership/person,/m/02q03dy +221,/m/0cqt90,people/person/employment_history./business/employment_tenure/title,/m/01d74z +222,/m/0cqt90,people/person/employment_history./business/employment_tenure/title,/m/018gz8 +223,/m/0h7q0rq,people/person/employment_history./business/employment_tenure/title,/m/07t58 +224,David Martosko,organization/role/leaders./organization/leadership/person,/m/0180v2 +225,Jeremy Diamond,people/person/employment_history./business/employment_tenure/title,/m/0d8qb +226,Jeremy Diamond,organization/role/leaders./organization/leadership/person,/m/0gsgr +227,Ron+Bonjean,people/person/employment_history./business/employment_tenure/title,/m/03gq32j +228,ABC+News+Rahami,organization/organization/headquarters./location/mailing_address/country,/m/0b3wk +229,Cindy+Galli,organization/role/leaders./organization/leadership/person,2016 New York and New Jersey bombings +230,/m/0cqt90,people/person/employment_history./business/employment_tenure/title,/m/07r9r0 +231,/m/0cqt90,people/person/employment_history./business/employment_tenure/title,/m/037l9 +232,/m/03w9bnr,people/person/employment_history./business/employment_tenure/title,/m/0cqt90 +233,/m/03w9bnr,people/person/employment_history./business/employment_tenure/title,/m/07r9r0 +234,/m/01pjq21,people/person/employment_history./business/employment_tenure/title,/m/02qkv0r +235,Michael Cohen,people/person/employment_history./business/employment_tenure/title,/m/04gc2 +236,/m/09v12g0,people/person/employment_history./business/employment_tenure/title,/m/01cpjg +237,/m/0cqt90,organization/role/leaders./organization/leadership/person,/m/0h7q0rq +238,Getty+Joe+Arpaio+Trump,people/person/places_lived./people/place_lived/location,USI +239,/m/0h7q0rq,organization/role/leaders./organization/leadership/person,/m/0gsgr +240,/m/0fm2h,people/person/employment_history./business/employment_tenure/title,/m/060s9 +241,/m/02dgtb,people/person/employment_history./business/employment_tenure/title,/m/0btx2g +242,/m/0cqt90,people/person/nationality,/m/0bd931 +243,/m/042f3_,people/person/employment_history./business/employment_tenure/title,/m/04wk4z +244,/m/0cqt90,organization/role/leaders./organization/leadership/person,/m/0bd931 +245,/m/0b7rm2,organization/role/leaders./organization/leadership/person,/m/04s4_v +246,Igor+Sechinduring,organization/role/leaders./organization/leadership/person,/m/04s4_v +247,/m/0cqt90,organization/role/leaders./organization/leadership/person,/m/0cqt90 +248,/m/061xw_,organization/role/leaders./organization/leadership/person,/m/03cnqkd +249,Craig,organization/role/leaders./organization/leadership/person,/m/01mlrk +250,Sergei+Aleksashenko,people/person/employment_history./business/employment_tenure/title,/m/05_xks +251,Sergei+Aleksashenko,organization/role/leaders./organization/leadership/person,/m/0382np +252,/m/011l2bwd,organization/role/leaders./organization/leadership/person,/m/0bmgs01 +253,/m/05h3wxl,organization/role/leaders./organization/leadership/person,/m/02_1m +254,/m/02849v5,organization/role/leaders./organization/leadership/person,/m/037ppp +255,/m/05h3wxl,organization/role/leaders./organization/leadership/person,/m/0bhp2c +256,/m/02ybc5,organization/role/leaders./organization/leadership/person,/m/04s4_v +257,/m/09gd0s0,people/person/employment_history./business/employment_tenure/title,/m/0345h +258,/m/09gd0s0,organization/role/leaders./organization/leadership/person,/m/037ppp +259,/m/020yjg,people/person/employment_history./business/employment_tenure/title,/m/04c_3b +260,Rick Dearborn,people/person/employment_history./business/employment_tenure/title,/m/025whr1 +261,HES,organization/role/leaders./organization/leadership/person,/m/0cqt90 +262,/m/0296q2,people/person/employment_history./business/employment_tenure/title,/m/060s9 +263,/m/02qwnj2,people/person/employment_history./business/employment_tenure/title,/m/06bnz +264,Carter Page,people/person/employment_history./business/employment_tenure/title,/m/07r9r0 +265,/m/0px38,organization/role/leaders./organization/leadership/person,/m/0cqt90 +266,/m/02z2l74,people/person/employment_history./business/employment_tenure/title,/m/028fjr +267,/m/05c2_fd,people/person/employment_history./business/employment_tenure/title,/m/02mjmr +268,/m/0l8ps43,people/person/employment_history./business/employment_tenure/title,/m/03mdbqx +269,/m/0l8ps43,people/person/employment_history./business/employment_tenure/title,/m/01pt0z +270,/m/04c_3b,people/person/employment_history./business/employment_tenure/title,/m/0cqt90 +271,/m/07r9r0,people/person/employment_history./business/employment_tenure/title,/m/0d06m5 +272,/m/0cqt90,people/person/employment_history./business/employment_tenure/title,/m/07t58 +273,/m/0157m,people/person/employment_history./business/employment_tenure/title,/m/07r9r0 +274,Peter Baker,people/person/employment_history./business/employment_tenure/title,List of biographers +275,/m/0ds2ztj,people/person/employment_history./business/employment_tenure/title,/m/04l8cr +276,/m/024v2j,people/person/employment_history./business/employment_tenure/title,United States Speaker of the House +277,David Fahrenthold,organization/role/leaders./organization/leadership/person,/m/0px38 +278,David Fahrenthold,people/person/employment_history./business/employment_tenure/title,/m/0d8qb +279,/m/0lq3t,organization/role/leaders./organization/leadership/person,/m/0d06m5 +280,Saul+Florez,people/person/employment_history./business/employment_tenure/title,/m/0j28ksx +281,Andrew McCabe,people/person/employment_history./business/employment_tenure/title,Deputy Director +282,/m/0c3x3rz,people/person/employment_history./business/employment_tenure/title,/m/02qkv0r +283,Andrew McCabe,organization/role/leaders./organization/leadership/person,/m/02_1m +284,Minneapolis+Department+of+Civil+Rights,organization/role/leaders./organization/leadership/person,Velma+Korbel +285,Velma+Korbel,organization/role/leaders./organization/leadership/person,Minneapolis+Department+of+Civil+Rights +286,Velma+Korbel,organization/role/leaders./organization/leadership/person,/m/02jknp +287,/m/0cn4c3,people/person/employment_history./business/employment_tenure/title,/m/0h1vgc +288,/m/022r9r,people/person/employment_history./business/employment_tenure/title,/m/02mjmr +289,/m/0chghy,organization/organization/headquarters./location/mailing_address/country,/m/016bvz +290,/m/02p3gzs,people/person/employment_history./business/employment_tenure/title,/m/07ghq +291,Sean Spicer,organization/role/leaders./organization/leadership/person,/m/02r4jz +292,/m/0hx5v,organization/organization/headquarters./location/mailing_address/country,/m/07t58 +293,/m/06jm4y,organization/role/leaders./organization/leadership/person,/m/0k0symb +294,/m/0h7q0rq,people/person/employment_history./business/employment_tenure/title,/m/0chbx +295,Paul+Ryan+RWI,people/person/employment_history./business/employment_tenure/title,United States Speaker of the House +296,/m/05ghhl,people/person/employment_history./business/employment_tenure/title,/m/04wk4z +297,/m/05ghhl,organization/role/leaders./organization/leadership/person,/m/02z_b +298,/m/05ghhl,organization/role/leaders./organization/leadership/person,Ryan's World +299,/m/04s9n,people/person/employment_history./business/employment_tenure/title,/m/066dv +300,Walter Shaub,organization/role/leaders./organization/leadership/person,/m/05qwyl +301,/m/05qwyl,organization/role/leaders./organization/leadership/person,Walter Shaub +302,Lisa+Mead.Mead,people/person/employment_history./business/employment_tenure/title,/m/01pn0r +303,Lisa+Mead.Mead,organization/role/leaders./organization/leadership/person,/m/03mdbqx +304,/m/07t7hy,people/person/employment_history./business/employment_tenure/title,/m/060s9 +305,Lisa+Mead,people/person/employment_history./business/employment_tenure/title,/m/01pn0r +306,Carlson,organization/role/leaders./organization/leadership/person,/m/0k0symb +307,/m/0kw03,people/person/employment_history./business/employment_tenure/title,/m/02mjmr +308,/m/0kw03,people/person/employment_history./business/employment_tenure/title,/m/02rnkmh +309,/m/0kw03,people/person/employment_history./business/employment_tenure/title,/m/09l7n +310,/m/0kw03,organization/role/leaders./organization/leadership/person,/m/0289n8t +311,/m/0bymv,people/person/employment_history./business/employment_tenure/title,/m/04c_3b +312,/m/025cj4,people/person/employment_history./business/employment_tenure/title,Imran Awan +313,/m/04cmyw,people/person/employment_history./business/employment_tenure/title,/m/02qkv0r +314,/m/081sq,organization/role/leaders./organization/leadership/person,/m/04cmyw +315,/m/081sq,organization/organization/headquarters./location/mailing_address/country,/m/02mjmr +316,/m/0b3wk,organization/organization/headquarters./location/mailing_address/country,/m/0b3wk +317,/m/0p_pd,people/person/employment_history./business/employment_tenure/title,/m/014g_s +318,/m/035dk,people/person/employment_history./business/employment_tenure/title,/m/0cqt90 +319,/m/022r9r,people/person/employment_history./business/employment_tenure/title,/m/0btx2g +320,/m/0n8_swr,people/person/employment_history./business/employment_tenure/title,/m/02n9jv +321,/m/024v2j,people/person/employment_history./business/employment_tenure/title,/m/04l8cr +322,/m/039rwf,people/person/employment_history./business/employment_tenure/title,/m/0499g1 +323,/m/05_xks,organization/role/leaders./organization/leadership/person,/m/039rwf +324,/m/012v1t,organization/role/leaders./organization/leadership/person,/m/081sq +325,/m/0cqt90,organization/role/leaders./organization/leadership/person,/m/01p06z +326,Sean Spicer,people/person/employment_history./business/employment_tenure/title,/m/03mdbqx +327,Sean Spicer,people/person/nationality,/m/06nm1 +328,/m/0cqt90,people/person/nationality,/m/06nm1 +329,/m/094z8,location/hud_county_place/county,/m/01lbst +330,/m/065nfw,organization/role/leaders./organization/leadership/person,Heather +331,/m/0k0q7zz,people/person/employment_history./business/employment_tenure/title,/m/04zz1zx +332,/m/094z8,location/hud_county_place/county,/m/0cymp +333,/m/0bymv,organization/role/leaders./organization/leadership/person,/m/02kfv7 +334,/m/0g12y5,organization/role/leaders./organization/leadership/person,/m/0167_c +335,/m/0bymv,organization/role/leaders./organization/leadership/person,Meet +336,/m/0bymv,people/person/employment_history./business/employment_tenure/title,/m/05_xks +337,/m/01zq6b,people/person/employment_history./business/employment_tenure/title,Foreign Secretary +338,/m/0fgw19,people/person/employment_history./business/employment_tenure/title,/m/0h7q0rq +339,/m/047qlq,organization/role/leaders./organization/leadership/person,Cornerstone Church +340,/m/03dwt_,location/country/languages_spoken,/m/0hzlz +341,/m/036qv,organization/role/leaders./organization/leadership/person,James+Ben+Shahali +342,Jim Porter,people/person/employment_history./business/employment_tenure/title,/m/02mjmr +343,Jim Porter,organization/role/leaders./organization/leadership/person,List of presidents of the National Rifle Association +344,/m/02mjmr,people/person/employment_history./business/employment_tenure/title,/m/0465sdr +345,/m/04k0vk,people/person/employment_history./business/employment_tenure/title,/m/0d05fv +346,/m/0bqscsg,organization/role/leaders./organization/leadership/person,/m/0k0symb +347,/m/0bqscsg,organization/role/leaders./organization/leadership/person,/m/01c0z +348,/m/0bqscsg,organization/role/leaders./organization/leadership/person,/m/027_2w9 +349,/m/0bqscsg,people/person/employment_history./business/employment_tenure/title,/m/05_xks +350,/m/042kg,people/person/employment_history./business/employment_tenure/title,/m/02mjmr +351,Barack+Obama.Yesterday+Winters,people/person/employment_history./business/employment_tenure/title,/m/02mjmr +352,/m/0h7q0rq,people/person/places_lived./people/place_lived/location,/m/0cqt90 +353,Todd+McMartin,organization/role/leaders./organization/leadership/person,/m/02_1m +354,/m/0cqt90,organization/role/leaders./organization/leadership/person,/m/0d06m5 +355,/m/03p2hqb,people/person/employment_history./business/employment_tenure/title,/m/0d8qb +356,/m/05zbm4,organization/role/leaders./organization/leadership/person,/m/05gnf +357,/m/05zbm4,people/person/employment_history./business/employment_tenure/title,/m/03mdbqx +358,/m/05zbm4,organization/role/leaders./organization/leadership/person,/m/07wpbx +359,/m/052g5x,organization/role/leaders./organization/leadership/person,/m/03818y +360,Stephawn,people/person/employment_history./business/employment_tenure/title,Stephawn +361,/m/080kq,people/person/employment_history./business/employment_tenure/title,/m/04gc2 +362,/m/080kq,people/person/employment_history./business/employment_tenure/title,/m/0157m +363,/m/0c3x3rz,people/person/employment_history./business/employment_tenure/title,/m/06lvrr +364,James+McFitting,people/person/employment_history./business/employment_tenure/title,Col +365,/m/06gfj,transportation/road/end1./transportation/road_starting_point/location,/m/0k7ny +366,/m/06gfj,transportation/road/end1./transportation/road_starting_point/location,/m/0lrr8 +367,Kyle+Jennings,people/person/employment_history./business/employment_tenure/title,/m/015czt +368,"Leon County Court, Florida",organization/role/leaders./organization/leadership/person,/m/0d06m5 +369,/m/0159qc,people/person/employment_history./business/employment_tenure/title,/m/02qkv0r +370,/m/02zzm_,people/person/employment_history./business/employment_tenure/title,/m/0btx2g +371,/m/02zzm_,organization/role/leaders./organization/leadership/person,/m/01sv_b +372,/m/025n3p,organization/role/leaders./organization/leadership/person,/m/0glpjll +373,/m/025n3p,people/person/employment_history./business/employment_tenure/title,/m/014g_s +374,Clinton+Greenwood,people/person/employment_history./business/employment_tenure/title,/m/03yc4j2 +375,/m/0ftvg,people/person/employment_history./business/employment_tenure/title,/m/01fch0 +376,/m/0cqt90,people/person/employment_history./business/employment_tenure/title,/m/02nwq +377,/m/0cqt90,people/person/employment_history./business/employment_tenure/title,/m/05b7q +378,/m/0cqt90,people/person/employment_history./business/employment_tenure/title,/m/0d05w3 +379,/m/0cqt90,people/person/employment_history./business/employment_tenure/title,/m/02xry +380,/m/01zf6x,location/country/languages_spoken,/m/06bnz +381,/m/014j8_,location/country/languages_spoken,/m/06bnz +382,/m/0cqt90,location/country/languages_spoken,/m/01zf6x +383,Yuri+Ivanenko,people/person/employment_history./business/employment_tenure/title,/m/06lvrr +384,/m/022r9r,people/person/employment_history./business/employment_tenure/title,/m/0d05fv +385,/m/0hht8px,people/person/employment_history./business/employment_tenure/title,/m/06lvrr +386,/m/01z6ls,organization/role/leaders./organization/leadership/person,/m/07r9r0 +387,/m/020y8m,organization/role/leaders./organization/leadership/person,/m/02pxjcm +388,/m/020y8m,people/person/employment_history./business/employment_tenure/title,/m/02qkv0r +389,/m/04jprn,organization/role/leaders./organization/leadership/person,/m/01lvn4 +390,/m/03cqm2,organization/role/leaders./organization/leadership/person,/m/02z_b +391,/m/04jprn,organization/role/leaders./organization/leadership/person,/m/03mcdjf +392,Mizu+Tomazaki,people/person/employment_history./business/employment_tenure/title,/m/06lvrr +393,/m/0280k2k,people/person/employment_history./business/employment_tenure/title,/m/03mdbqx +394,/m/0280k2k,organization/role/leaders./organization/leadership/person,/m/021lry +395,/m/05xdp5,people/person/employment_history./business/employment_tenure/title,/m/03x_sks +396,Seth+RichWikiLeaks,organization/role/leaders./organization/leadership/person,/m/018y36 +397,Murder of Seth Rich,organization/role/leaders./organization/leadership/person,/m/018y36 +398,/m/0rlz,people/person/employment_history./business/employment_tenure/title,/m/02mjmr +399,/m/09b78,people/person/employment_history./business/employment_tenure/title,Col +400,Donald,organization/role/leaders./organization/leadership/person,/m/085srz +401,/m/01_gbv,people/person/employment_history./business/employment_tenure/title,Col +402,/m/0cqt90,organization/role/leaders./organization/leadership/person,/m/09b6t +403,/m/0c00p_n,people/person/employment_history./business/employment_tenure/title,/m/02qkv0r +404,/m/047blr4,organization/role/leaders./organization/leadership/person,House+Oversign+Committee+Chair +405,/m/047blr4,people/person/employment_history./business/employment_tenure/title,/m/02qkv0r +406,/m/0h3wc7,organization/role/leaders./organization/leadership/person,/m/0152x_ +407,/m/0h3wc7,people/person/employment_history./business/employment_tenure/title,/m/04wk4z +408,/m/05kldj,people/person/employment_history./business/employment_tenure/title,/m/057mg4 +409,Hillary+Clinton.Jason+Chaffetz,organization/role/leaders./organization/leadership/person,United+States+House+Committee+on+Oversight+and+Government+Reform +410,/m/0dpr5f,people/person/employment_history./business/employment_tenure/title,/m/07r9r0 +411,Louis+Leweigh,people/person/employment_history./business/employment_tenure/title,/m/02k13d +412,/m/025s8bs,location/country/languages_spoken,Mall+of+Claus +413,/m/0pv1y,location/country/languages_spoken,/m/07c5l +414,Mall+of+Claus,location/country/languages_spoken,/m/07c5l +415,Hal+Lindsay+DNJ,people/person/employment_history./business/employment_tenure/title,/m/04c_3b +416,Lindsay,people/person/employment_history./business/employment_tenure/title,/m/04c_3b +417,/m/02j2mf,organization/role/leaders./organization/leadership/person,/m/05f4p +418,/m/06sjd0,organization/role/leaders./organization/leadership/person,/m/0289n8t +419,/m/085srz,organization/role/leaders./organization/leadership/person,/m/01w74d +420,/m/0cqt90,organization/role/leaders./organization/leadership/person,/m/01q_cy +421,/m/06gfj,transportation/road/end1./transportation/road_starting_point/location,/m/0fn2g +422,/m/074fdq,people/person/employment_history./business/employment_tenure/title,/m/074fdq +423,Linda+Stanley,organization/role/leaders./organization/leadership/person,Franklin+County+Historical+Society +424,/m/094z8,location/hud_county_place/county,/m/019fz +425,/m/03cztnx,organization/role/leaders./organization/leadership/person,/m/03hpkp +426,/m/01tz2p,people/person/employment_history./business/employment_tenure/title,/m/04gc2 +427,/m/0dl567,organization/role/leaders./organization/leadership/person,/m/05ngqz +428,/m/05h59wt,people/person/employment_history./business/employment_tenure/title,/m/03x_sks +429,/m/043q0,people/person/employment_history./business/employment_tenure/title,/m/02mjmr +430,/m/01wl8_,organization/role/leaders./organization/leadership/person,Visit+Virginias+Blue+Ridge +431,/m/0d06m5,people/person/employment_history./business/employment_tenure/title,/m/02vy3_j +432,Josh+Hendler,people/person/employment_history./business/employment_tenure/title,/m/03xk7t +433,/m/0dpr5f,organization/role/leaders./organization/leadership/person,/m/09z___ +434,/m/0137g1,people/person/employment_history./business/employment_tenure/title,My First First Love +435,/m/0137g1,people/person/employment_history./business/employment_tenure/title,Jimmy Dimora +436,/m/02pjpd,organization/role/leaders./organization/leadership/person,/m/0gsgr +437,/m/0c3ntp,organization/role/leaders./organization/leadership/person,/m/0gsgr +438,2016 United States presidential election,organization/role/leaders./organization/leadership/person,/m/085srz +439,/m/0bwl7t0,people/person/employment_history./business/employment_tenure/title,2016 United States presidential election +440,/m/06c97,organization/role/leaders./organization/leadership/person,/m/028rk +441,/m/06c97,organization/role/leaders./organization/leadership/person,/m/03mdbqx +442,/m/0c_md_,people/person/employment_history./business/employment_tenure/title,/m/02mjmr +443,/m/0bl83,people/person/employment_history./business/employment_tenure/title,/m/0d05fv +444,Henry Trewhitt,people/person/employment_history./business/employment_tenure/title,/m/0d8qb +445,Henry Trewhitt,organization/role/leaders./organization/leadership/person,/m/01n0qv +446,/m/03cdg,organization/role/leaders./organization/leadership/person,/m/0gsgr +447,/m/0jw4v,people/person/employment_history./business/employment_tenure/title,/m/0btx2g +448,/m/0d05fv,people/person/employment_history./business/employment_tenure/title,/m/0d05fv +449,/m/0d3qd0,people/person/employment_history./business/employment_tenure/title,/m/04c_3b +450,/m/0d05fv,people/person/employment_history./business/employment_tenure/title,/m/02mjmr +451,/m/07t19q,organization/role/leaders./organization/leadership/person,/m/07tf8 +452,/m/0f7fy,organization/role/leaders./organization/leadership/person,/m/0d06m5 +453,/m/06c97,people/person/employment_history./business/employment_tenure/title,/m/0kyk +454,/m/064x_k,organization/role/leaders./organization/leadership/person,/m/0j75z +455,/m/064x_k,people/person/employment_history./business/employment_tenure/title,/m/06lvrr +456,/m/06hx2,organization/role/leaders./organization/leadership/person,/m/0cv_2 +457,/m/028rk,people/person/employment_history./business/employment_tenure/title,/m/02mjmr +458,Jim Justice,people/person/employment_history./business/employment_tenure/title,/m/081mh +459,Jim Justice,people/person/employment_history./business/employment_tenure/title,/m/0btx2g +460,/m/045k9,organization/role/leaders./organization/leadership/person,/m/0d06m5 +461,/m/06mg8q,people/person/employment_history./business/employment_tenure/title,/m/02mjmr +462,/m/045k9,organization/role/leaders./organization/leadership/person,/m/0157m +463,/m/06mg8q,organization/role/leaders./organization/leadership/person,/m/07r9r0 +464,/m/02r81cr,organization/role/leaders./organization/leadership/person,/m/0cqt90 +465,/m/0btx2g,people/person/employment_history./business/employment_tenure/title,/m/06mg8q +466,Kyle+Kondik,organization/role/leaders./organization/leadership/person,University of Virginia Center for Politics +467,Kyle+Kondik,people/person/employment_history./business/employment_tenure/title,/m/076xl9x +468,Scott+Crichlow,organization/role/leaders./organization/leadership/person,/m/019dwp +469,/m/01370l_y,people/person/employment_history./business/employment_tenure/title,/m/03mdbqx +470,/m/01370l_y,organization/role/leaders./organization/leadership/person,/m/045k9 +471,/m/02n9jv,people/person/employment_history./business/employment_tenure/title,David Saunders +472,Kerr+Putney,organization/role/leaders./organization/leadership/person,/m/03d5lmj +473,Rakeyia+Scott,people/person/places_lived./people/place_lived/location,Scotts +474,/m/05p0j_b,people/person/employment_history./business/employment_tenure/title,/m/0jdd +475,/m/05p0j_b,people/person/employment_history./business/employment_tenure/title,Superstar +476,/m/0ctllc,location/country/languages_spoken,/m/0v74 +477,Charles+Martlandfor,people/person/employment_history./business/employment_tenure/title,/m/01g2j2 +478,/m/0157m,organization/role/leaders./organization/leadership/person,Getty+Trump +479,/m/024tyk,people/person/employment_history./business/employment_tenure/title,/m/02qkv0r +480,Lynch,organization/role/leaders./organization/leadership/person,/m/021sj1 +481,Lynch,organization/role/leaders./organization/leadership/person,/m/0b9v8_ +482,Cedric+Richmond+DLa.,people/person/employment_history./business/employment_tenure/title,/m/02qkv0r +483,/m/04vrnl,people/person/nationality,/m/07t58 +484,/m/06cc_1,organization/role/leaders./organization/leadership/person,/m/0j_sncb +485,/m/03tk4w,people/person/employment_history./business/employment_tenure/title,/m/02qkv0r +486,/m/03w9bnr,organization/role/leaders./organization/leadership/person,/m/0152x_ +487,/m/0gjsd3,people/person/employment_history./business/employment_tenure/title,/m/03x_sks +488,/m/083rp7,organization/role/leaders./organization/leadership/person,/m/02rg_ +489,/m/03p2hqb,organization/role/leaders./organization/leadership/person,/m/02_1m +490,Timothy Evans,organization/role/leaders./organization/leadership/person,/m/0cv_2 +491,/m/027cmm9,people/person/employment_history./business/employment_tenure/title,/m/0btx2g +492,/m/0_ylz5n,people/person/employment_history./business/employment_tenure/title,/m/0btx2g +493,/m/0bb5tw,people/person/employment_history./business/employment_tenure/title,/m/0btx2g +494,/m/0h7q0rq,people/person/employment_history./business/employment_tenure/title,/m/05k7sb +495,Jean-Marc Puissesseau,people/person/employment_history./business/employment_tenure/title,/m/02mwvv +496,Friedrich,people/person/places_lived./people/place_lived/location,/m/017v_ +497,/m/0406vgn,organization/role/leaders./organization/leadership/person,/m/09pkf +498,Friedrich,people/person/places_lived./people/place_lived/location,/m/02rscps +499,/m/011x14ft,people/person/employment_history./business/employment_tenure/title,/m/02rscps +500,/m/0cx2r,organization/role/leaders./organization/leadership/person,/m/085srz +501,/m/025xzm,organization/role/leaders./organization/leadership/person,/m/0694qd +502,/m/025xzm,people/person/employment_history./business/employment_tenure/title,/m/07r9r0 +503,/m/0cx2r,organization/role/leaders./organization/leadership/person,/m/04mzmmw +504,/m/0cx2r,organization/role/leaders./organization/leadership/person,Strategy+Group+for+Media +505,/m/05tjbz,people/person/employment_history./business/employment_tenure/title,/m/0dln6mf +506,Etonde+Maloke,people/person/employment_history./business/employment_tenure/title,/m/014cnc +507,/m/06lwq6,organization/role/leaders./organization/leadership/person,/m/01lvn4 +508,/m/06gn7,people/person/employment_history./business/employment_tenure/title,/m/03x_sks +509,/m/06h90t,people/person/employment_history./business/employment_tenure/title,/m/02qkv0r +510,/m/025_74m,people/person/employment_history./business/employment_tenure/title,/m/02qkv0r +511,/m/0m_w6,organization/role/leaders./organization/leadership/person,/m/0cv_2 +512,/m/02n670,people/person/employment_history./business/employment_tenure/title,/m/01t_55 +513,/m/0d3qd0,people/person/employment_history./business/employment_tenure/title,/m/0d3qd0 +514,/m/0jl0yx0,people/person/employment_history./business/employment_tenure/title,/m/02mjmr +515,/m/02zs4,organization/organization/headquarters./location/mailing_address/country,/m/07t58 +516,/m/09k8g7,organization/role/leaders./organization/leadership/person,/m/02zs4 +517,/m/09k8g7,people/person/employment_history./business/employment_tenure/title,/m/02mwvv +518,/m/063xbk,organization/role/leaders./organization/leadership/person,Kendall Wall Band +519,/m/0zbssjv,people/person/employment_history./business/employment_tenure/title,/m/02mjmr +520,/m/0nbhk23,people/person/employment_history./business/employment_tenure/title,/m/01t_55 +521,/m/05b2r22,organization/role/leaders./organization/leadership/person,/m/01brfl +522,/m/0nbhk23,organization/role/leaders./organization/leadership/person,/m/07vp7 +523,/m/07t58,organization/organization/headquarters./location/mailing_address/country,/m/05b7q +524,/m/01vlpc,organization/organization/headquarters./location/mailing_address/country,/m/048fz +525,Bruce+Klingner,organization/role/leaders./organization/leadership/person,/m/01tw7q +526,/m/017ql,organization/organization/headquarters./location/mailing_address/country,/m/07t58 +527,/m/042f1,people/person/employment_history./business/employment_tenure/title,/m/02mjmr +528,/m/0k_2z,people/person/employment_history./business/employment_tenure/title,/m/04c_3b +529,/m/0k_2z,organization/role/leaders./organization/leadership/person,/m/085srz +530,/m/042dk,people/person/employment_history./business/employment_tenure/title,/m/02mjmr +531,/m/0d06m5,people/person/employment_history./business/employment_tenure/title,/m/02mjmr +532,Nate Smith,organization/role/leaders./organization/leadership/person,Texas secession movements +533,Peter+Kreko,organization/role/leaders./organization/leadership/person,/m/08qnnv +534,/m/0h7q0rq,people/person/employment_history./business/employment_tenure/title,/m/0h7q0rq +535,Jon+Alterman,organization/role/leaders./organization/leadership/person,/m/02n4mz +536,/m/0l8ps43,organization/role/leaders./organization/leadership/person,/m/03mdbqx +537,/m/0dnps,people/person/nationality,/m/02k54 +538,Fahad+Nazer,organization/role/leaders./organization/leadership/person,Saudi+Embassy +539,/m/01z6ls,people/person/employment_history./business/employment_tenure/title,/m/0h7q0rq +540,/m/0204x1,organization/role/leaders./organization/leadership/person,/m/07r9r0 +541,/m/01xcqs,people/person/employment_history./business/employment_tenure/title,/m/0250w0 +542,/m/0h7q0rq,people/person/nationality,/m/01z215 +543,/m/03dlf3,people/person/employment_history./business/employment_tenure/title,/m/02qkv0r +544,/m/0d8qb,people/person/employment_history./business/employment_tenure/title,/m/03dlf3 +545,/m/0h63jfq,organization/role/leaders./organization/leadership/person,Frank +546,/m/05tjbz,organization/role/leaders./organization/leadership/person,/m/0d06m5 +547,/m/0gg570x,people/person/employment_history./business/employment_tenure/title,/m/0465sdr +548,/m/03mdbqx,organization/role/leaders./organization/leadership/person,/m/0d06m5 +549,/m/0h7q0rq,people/person/employment_history./business/employment_tenure/title,/m/012t_z +550,/m/0cqt90,organization/role/leaders./organization/leadership/person,/m/0cjdk +551,/m/012v1t,people/person/employment_history./business/employment_tenure/title,/m/0499g1 +552,Ryan's World,people/person/employment_history./business/employment_tenure/title,/m/0cfpc +553,/m/012v1t,people/person/employment_history./business/employment_tenure/title,/m/0h7q0rq +554,/m/01xcd1,people/person/employment_history./business/employment_tenure/title,/m/0250w0 +555,/m/01xcd1,organization/role/leaders./organization/leadership/person,/m/07r9r0 +556,/m/025k5p,people/person/employment_history./business/employment_tenure/title,/m/0250w0 +557,/m/025k5p,people/person/employment_history./business/employment_tenure/title,/m/01xcd1 +558,/m/0286t7r,organization/role/leaders./organization/leadership/person,/m/085srz +559,/m/01tqr5,people/person/employment_history./business/employment_tenure/title,/m/07r9r0 +560,/m/0l8ps43,organization/role/leaders./organization/leadership/person,/m/081sq +561,/m/018fzs,people/person/employment_history./business/employment_tenure/title,United States Speaker of the House +562,Shooting of Terence Crutcher,people/person/employment_history./business/employment_tenure/title,/m/01t2ln +563,/m/0h7q0rq,people/person/employment_history./business/employment_tenure/title,Omran Daqneesh +564,/m/02pjhgt,people/person/employment_history./business/employment_tenure/title,/m/02h6676 +565,/m/0d06m5,organization/role/leaders./organization/leadership/person,/m/026wp +566,/m/012gx2,people/person/employment_history./business/employment_tenure/title,/m/02mjmr +567,/m/0l0cx,people/person/employment_history./business/employment_tenure/title,Lists of golfers +568,/m/06jvj7,organization/role/leaders./organization/leadership/person,/m/01bncw +569,/m/01b8g6,organization/role/leaders./organization/leadership/person,/m/059dn +570,/m/01b8g6,people/person/employment_history./business/employment_tenure/title,Secretary-General of the United Nations +571,/m/059dn,organization/role/leaders./organization/leadership/person,Jens +572,/m/0cqt90,people/person/places_lived./people/place_lived/location,/m/01n32 +573,Jeffrey DeLaurentis,people/person/employment_history./business/employment_tenure/title,/m/07c5l +574,Getty+Obama,people/person/employment_history./business/employment_tenure/title,/m/0d04z6 +575,Jeffrey DeLaurentis,people/person/employment_history./business/employment_tenure/title,/m/080ntlp +576,Getty+Obama,people/person/employment_history./business/employment_tenure/title,/m/0n46_gt +577,/m/0h7q0rq,people/person/employment_history./business/employment_tenure/title,/m/0n46_gt +578,/m/025s5v9,people/person/employment_history./business/employment_tenure/title,/m/02xy5 +579,/m/06k3_t,organization/role/leaders./organization/leadership/person,/m/0275kr +580,/m/06k3_t,people/person/employment_history./business/employment_tenure/title,/m/04wk4z +581,/m/0h7q0rq,organization/role/leaders./organization/leadership/person,/m/06k3_t +582,/m/01tqr5,organization/role/leaders./organization/leadership/person,/m/07w42 +583,/m/05st_r,organization/role/leaders./organization/leadership/person,/m/0kq5b +584,Stephen Anderson,people/person/employment_history./business/employment_tenure/title,/m/01fhsb +585,/m/0d0xkd,organization/role/leaders./organization/leadership/person,/m/09gbckv +586,/m/0cqt90,people/person/places_lived./people/place_lived/location,Umpa-Lumpaer +587,/m/03yjnwr,organization/role/leaders./organization/leadership/person,/m/01p06z +588,/m/03yjnwr,organization/role/leaders./organization/leadership/person,/m/0289n8t +589,/m/03yjnwr,people/person/employment_history./business/employment_tenure/title,/m/0d8qb +590,/m/0204x1,people/person/employment_history./business/employment_tenure/title,/m/0gs0g +591,Nicole+Mainor,people/person/employment_history./business/employment_tenure/title,/m/03mdbqx +592,Nicole+Mainor,organization/role/leaders./organization/leadership/person,/m/0fynw +593,Craig+Engle,people/person/employment_history./business/employment_tenure/title,/m/04cs6sm +594,/m/01w74d,organization/role/leaders./organization/leadership/person,/m/07r9r0 +595,/m/01w74d,people/person/employment_history./business/employment_tenure/title,/m/04c_3b +596,/m/03j740,organization/role/leaders./organization/leadership/person,/m/01w74d +597,/m/03j7vg,organization/role/leaders./organization/leadership/person,Reid +598,/m/0g92xz,organization/role/leaders./organization/leadership/person,/m/04vy0g +599,/m/01xcqs,organization/role/leaders./organization/leadership/person,/m/07r9r0 +600,/m/06qsh0,people/person/employment_history./business/employment_tenure/title,/m/0f5d2 +601,/m/08w60w,people/person/employment_history./business/employment_tenure/title,/m/02mjmr +602,/m/0g5vy,organization/role/leaders./organization/leadership/person,/m/07t7hy +603,/m/0g5vy,people/person/nationality,/m/07t7hy +604,/m/07zcdm,people/person/employment_history./business/employment_tenure/title,/m/02mjmr +605,/m/060nzt,people/person/employment_history./business/employment_tenure/title,/m/01_d4 +606,Sadiq+Khan+Trump,people/person/employment_history./business/employment_tenure/title,/m/01_d4 +607,/m/060nzt,people/person/employment_history./business/employment_tenure/title,/m/0cqt90 +608,/m/012gx2,people/person/employment_history./business/employment_tenure/title,/m/0cqt90 +609,/m/060nzt,people/person/employment_history./business/employment_tenure/title,/m/03x_sks +610,/m/06w3g7j,people/person/employment_history./business/employment_tenure/title,/m/060s9 +611,/m/0h7q0rq,people/person/employment_history./business/employment_tenure/title,/m/0bwl7t0 +612,/m/0h7q0rq,people/person/employment_history./business/employment_tenure/title,/m/06slz5 +613,/m/0bsfy,people/person/employment_history./business/employment_tenure/title,/m/02mjmr +614,/m/02wzck2,organization/role/leaders./organization/leadership/person,/m/0d42gh +615,/m/0bwh6,organization/role/leaders./organization/leadership/person,Facebook+TwitterHollywood +616,Mario+Macri,people/person/employment_history./business/employment_tenure/title,/m/02mjmr +617,/m/06l7l,organization/role/leaders./organization/leadership/person,/m/07q5n +618,/m/0gkh2,location/country/languages_spoken,/m/01z88t +619,Shooting of Jeremy Mardis,people/person/employment_history./business/employment_tenure/title,/m/03yc4j2 +620,/m/01fhz1,people/person/employment_history./business/employment_tenure/title,/m/06lvrr +621,/m/024v2j,organization/role/leaders./organization/leadership/person,CNN+House +622,Robert+Milichs,people/person/employment_history./business/employment_tenure/title,/m/06lvrr +623,/m/0781kl,organization/role/leaders./organization/leadership/person,/m/02jf15 +624,/m/0cqt90,people/person/employment_history./business/employment_tenure/title,Jimmy Dimora +625,/m/02_1m,organization/role/leaders./organization/leadership/person,/m/0157m +626,/m/0cqt90,people/person/employment_history./business/employment_tenure/title,/m/0t7g5 +627,/m/08193,people/person/employment_history./business/employment_tenure/title,/m/02bms +628,/m/0cqt90,people/person/employment_history./business/employment_tenure/title,/m/08193 +629,/m/0cqt90,people/person/nationality,/m/08193 +630,Bill+Turpin,people/person/places_lived./people/place_lived/location,/m/02542p +631,/m/0crpt8,people/person/employment_history./business/employment_tenure/title,/m/0ply0 +632,Thomas+Davidenko,organization/role/leaders./organization/leadership/person,/m/08g_yr +633,/m/06_w579,organization/role/leaders./organization/leadership/person,/m/08z129 +634,/m/01n4f8,organization/role/leaders./organization/leadership/person,/m/04sv_8 +635,/m/01n4f8,people/person/employment_history./business/employment_tenure/title,/m/04wk4z +636,/m/08r7gq,organization/role/leaders./organization/leadership/person,/m/0k__z +637,/m/08r7gq,people/person/employment_history./business/employment_tenure/title,/m/014cnc +638,/m/0cqt90,people/person/employment_history./business/employment_tenure/title,/m/03w9bnr +639,Jefferson Davis monument,/location/location/containedby,/m/0f2tj +640,/m/043q0,/government/government_office_category/officeholders./government/government_position_held/jurisdiction_of_office,/m/020d5 +641,/m/05xdp5,/government/government_office_category/officeholders./government/government_position_held/jurisdiction_of_office,/m/0f2tj +642,/m/05xdp5,/people/person/employment_history./business/employment_tenure/title,/m/0pqc5 +643,/m/02mjmr,/government/politician/government_positions_held./government/government_position_held/office_position_or_title,/m/060d2 +644,/m/02mjmr,/people/person/ethnicity,/m/0x67 +645,Beaureguard BillyBob Johnson,/base/popstra/organization/supporter./base/popstra/support/supporter,/m/020d5 +646,/m/02z_b,/base/newsevents/news_reporting_organisation/news_reports./base/newsevents/news_report/event,/m/012hdhkj +647,/m/0rlz,/government/politician/government_positions_held./government/government_position_held/office_position_or_title,/m/060d2 +648,/m/0cqt90,/base/newsevents/news_reporting_organisation/news_reports./base/newsevents/news_report/event,/m/0kbq +649,/m/0cqt90,/government/politician/party./government/political_party_tenure/party,/m/07wbk +650,/m/0rlz,/government/politician/party./government/political_party_tenure/party,/m/0d075m +651,/m/0cqt90,/food/diet_follower/follows_diet,/m/09b6t +652,The Red Shtick,/business/employer/employees./business/employment_tenure/person,/m/04lcmzy +653,/m/0kbq,/military/military_conflict/military_personnel_involved,/m/09b78 +654,/m/09b78,"/people/profession/specializations , /people/person/profession",/m/03kll +655,/m/0cqt90,/government/politician/government_positions_held./government/government_position_held/office_position_or_title,/m/060d2 +656,United States House Committee on Oversight and Government Reform,/organization/role/leaders./organization/leadership/person,/m/047blr4 +657,/m/02_1m,/organization/role/leaders./organization/leadership/person,/m/06r04p +658,/m/0d06m5,/base/fight/crime_type/people_convicted_of_this_crime./base/crime/criminal_conviction/guilty_of,/m/07ppd +659,Freedom Crossroads,/business/employer/employees./business/employment_tenure/person,Louis Leweigh +660,Louis Leweigh,/people/person/profession,/m/02k13d +661,/m/0509p,/celebrities/celebrity/sexual_relationships./celebrities/romantic_relationship/celebrity,/m/0157m +662,/m/0157m,/people/cause_of_death/people,/m/0509p +663,/m/0d06m5,/people/cause_of_death/people,/m/0509p +664,/m/0509p,/people/deceased_person/cause_of_death,/m/016x3s +665,/m/035y33,/people/profession/people_with_this_profession,/m/024tyk +666,/m/0cqt90,/base/popstra/celebrity/endorsements./base/popstra/paid_support/company,/m/06bnz +667,/m/024tyk,/medicine/notable_person_with_medical_condition/condition,/m/09klv +668,/m/024tyk,/government/politician/party./government/political_party_tenure/party,/m/0d075m +669,/m/024tyk,/people/appointed_role/appointment./people/appointment/appointed_by,/m/07t31 +670,/m/024tyk,/people/person/ethnicity,/m/0x67 +671,1tch,/business/employer/employees./business/employment_tenure/person,Teddy Stick +672,/m/03qk63k,/influence/influence_node/influenced,the poor +673,/m/01npnk3,/organization/membership_organization/members./organization/organization_membership/member,/m/0gg86 +674,/m/01npnk3,/award/hall_of_fame_inductee/hall_of_fame_inductions./award/hall_of_fame_induction/hall_of_fame,/m/0ggkm +675,/m/0ggkm,/award/hall_of_fame/discipline,/m/04rlf +676,/m/01npnk3,/medicine/notable_person_with_medical_condition/condition,/m/0g02vk +677,/m/01npnk3,/people/person/place_of_birth,/m/05mph +678,/m/01npnk3,/people/person/places_lived./people/place_lived/location,/m/0r80l +679,/m/01npnk3,/music/musical_group/member./music/group_membership/member,Melody Ranch Girls +680,/m/01k3fvw,/music/artist/genre,/m/01lyv +681,/m/09l65,/people/profession/people_with_this_profession,/m/01npnk3 +682,/m/0gbbt,/people/profession/people_with_this_profession,/m/01npnk3 +683,/m/053f8h,/government/politician/party./government/political_party_tenure/party,/m/0d075m +684,/m/0d06m5,/people/appointer/appointment_made./people/appointment/appointee,/m/053f8h +685,/m/053f8h,/people/person/employment_history./business/employment_tenure/title,/m/048zv9l +686,/m/053f8h,/people/person/places_lived./people/place_lived/location,/m/07z1m +687,/m/07z1m,/government/political_district/representatives./government/government_position_held/office_holder,/m/053f8h +688,/m/022r9r,/celebrities/celebrity/celebrity_friends./celebrities/friendship/friend,/m/0cqt90 +689,/m/02607j,/location/location/containedby,/m/02_286 +690,Not Who We Are campaign,/location/location/containedby,/m/02607j +691,/m/0cqt90,/base/fight/crime_type/people_convicted_of_this_crime./base/crime/criminal_conviction/guilty_of,/m/06d4h +692,Not Who We Are campaign,/organization/role/leaders./organization/leadership/person,Josh Hendler +693,Rob Bielunas,/people/person/education./education/education/institution,/m/02607j +694,/m/0cqt90,/government/politician/government_positions_held./government/government_position_held/office_position_or_title,/m/066q5c +695,/m/02mjmr,/people/person/place_of_birth,/m/03gh4 +696,/m/06jvj7,/people/person/profession,moderator +697,/m/0d06m5,/government/politician/government_positions_held./government/government_position_held/office_position_or_title,/m/066q5c +698,/m/01cw8w,/medicine/disease/causes,/m/0294j +699,/m/0157m,/base/popstra/celebrity/endorsements./base/popstra/paid_support/company,/m/0243xw +700,/m/0157m,/organization/organization/board_members./organization/organization_board_membership/member,Clinton Health Access Initiative +701,/m/0157m,/base/fight/crime_type/people_convicted_of_this_crime./base/crime/criminal_conviction/guilty_of,/m/09pngm +702,/m/0cqt90,/influence/influence_node/influenced,/m/0x2gl +703,/m/0cqt90,/fictional_universe/fictional_character/species,Oompa Loompa +704,/m/0cqt90,/base/popstra/celebrity/supporter./base/popstra/support/supported_organization,/m/01b5_d +705,/m/0d06m5,/influence/influence_node/influenced_by,/m/0gsgr +706,/m/0d06m5,/medicine/notable_person_with_medical_condition/condition,/m/0g02vk +707,/m/0d06m5,/influence/influence_node/influenced,/m/09pngm +708,/m/0d075m,/base/popstra/celebrity/supporter./base/popstra/support/supported_organization,/m/06z49 +709,/m/0d075m,/base/popstra/celebrity/supporter./base/popstra/support/supported_organization,/m/0dtbg9 +710,/m/0157m,/base/fight/crime_type/people_convicted_of_this_crime./base/crime/criminal_conviction/guilty_of,/m/01fdjk +711,/m/0d06m5,/base/fight/crime_type/people_convicted_of_this_crime./base/crime/criminal_conviction/guilty_of,/m/06d4h +712,/m/02mjmr,/organization/organization_founder/organizations_founded,/m/027x630 +713,/m/02j9z,/base/popstra/celebrity/supporter./base/popstra/support/supported_organization,/m/027x630 +714,/m/09c7w0,/base/popstra/celebrity/supporter./base/popstra/support/supported_organization,/m/027x630 +715,/m/027x630,/organization/organization/founders,/m/02mjmr +716,/m/02mjmr,/projects/project_role/projects./projects/project_participation/participant,/m/02h71z +717,/m/025s5v9,/medicine/notable_person_with_medical_condition/condition,/m/071d3 +718,/m/02mjmr,/people/person/profession,/m/07jq_ +719,/m/02mjmr,/people/person/place_of_birth,/m/019rg5 +720,/m/02mjmr,/people/person/religion,/m/0flw86 +721,/m/02mjmr,/base/fight/crime_type/people_convicted_of_this_crime./base/crime/criminal_conviction/guilty_of,/m/05cxz3 +722,/m/0d06m5,/base/fight/crime_type/people_convicted_of_this_crime./base/crime/criminal_conviction/guilty_of,/m/05cxz3 +723,/m/05mz656,/people/person/profession,/m/0pcq81q +724,/m/05mz656,/base/popstra/celebrity/supporter./base/popstra/support/supported_organization,/m/012hdhkj +725,/m/03cm6b3,/people/person/profession,/m/02h665k +726,/m/03cm6b3,/base/popstra/celebrity/supporter./base/popstra/support/supported_organization,/m/012hdhkj +727,/m/023fbk,/base/nobelprizes/nobel_subject_area/nobel_awards./base/nobelprizes/nobel_honor/nobel_prize_winner,/m/04sh3 +728,/m/0g60g,/law/inventor/inventions,/m/07__7 +729,Michael Oreskes,/people/person/profession,/m/014l7h +730,/m/0fzdz7,/people/person/profession,m/0mb31 +731,/m/0fm2h,/government/politician/government_positions_held./government/government_position_held/office_position_or_title,/m/060bp +732,/m/0fm2h,/people/person/place_of_birth,/m/03spz +733,/m/0d06m5,/government/politician/party./government/political_party_tenure/party,/m/0d075m +734,Ben LaBolt,/projects/project_role/projects./projects/project_participation/participant,/m/0gk_h41 +735,/m/0l8ps43,/projects/project_role/projects./projects/project_participation/participant,/m/081sq diff --git a/run_fn_detection.py b/run_fn_detection.py index 0768ff3..d5ce129 100644 --- a/run_fn_detection.py +++ b/run_fn_detection.py @@ -9,6 +9,7 @@ import utils.label_encode_dataset as led import utils.create_node_embedding as cne import utils.create_relation_embedding as cre +import utils.misc as misc from gensim.models import KeyedVectors from sklearn.preprocessing import LabelEncoder from torch_geometric.utils import structured_negative_sampling @@ -86,13 +87,18 @@ def load_data(news_dataset, kg_dataset, dataset_name, hidden=64, node_embedding_ print("Loading fake news dataset: " + news_dataset + "...") fn_test = pd.read_csv( - "./data/" + news_dataset + "/gs_train.csv", + "./data/" + news_dataset + "/train_both.csv", sep=",", header=0, engine="python", index_col=0, ) + # create model folder + save_folder = "model_" + misc.get_now() + print("creating model folder: " + save_folder + "...") + mkdir(save_folder) + # needed for embedding all nodes across datasets (will use edge_index to split datasets) all_data = pd.concat([train, valid]) all_data = pd.concat([all_data, test]) @@ -117,11 +123,9 @@ def load_data(news_dataset, kg_dataset, dataset_name, hidden=64, node_embedding_ le_relation = LabelEncoder() le_relation.fit(relations) - if not path.exists(embedding_path): - mkdir(embedding_path) - np.save(path.join(embedding_path, "le_relation_classes_" + dataset_name + ".npy"), le_relation.classes_) - np.save(path.join(embedding_path, "le_entity_classes_" + dataset_name + ".npy"), le_entity.classes_) + np.save(path.join(save_folder, "le_relation_classes_" + dataset_name + ".npy"), le_relation.classes_) + np.save(path.join(save_folder, "le_entity_classes_" + dataset_name + ".npy"), le_entity.classes_) train = led.label_encode_dataset(le_entity, le_relation, train, entity_ids) valid = led.label_encode_dataset(le_entity, le_relation, valid, entity_ids) @@ -133,15 +137,15 @@ def load_data(news_dataset, kg_dataset, dataset_name, hidden=64, node_embedding_ print('all_data.x.size: ', all_data.x.size()) # create node embeddings if none exists cne.create_node_embedding( - all_data, dataset_name, dimensions=node_embedding_size, workers=4, path=embedding_path + all_data, dataset_name, dimensions=node_embedding_size, workers=4, path=save_folder ) - cre.create_relation_embedding(relations, le_relation, dataset_name, dimensions=hidden, path=embedding_path) + cre.create_relation_embedding(relations, le_relation, dataset_name, dimensions=hidden, path=save_folder) embedded_nodes = KeyedVectors.load_word2vec_format( - "{}/node_embedding_{}_{}.kv".format(embedding_path, dataset_name, str(node_embedding_size)) + "{}/node_embedding_{}_{}.kv".format(save_folder, dataset_name, str(node_embedding_size)) ) embedded_relations = KeyedVectors.load_word2vec_format( - "{}/relation_embedding_le_{}_{}.bin".format(embedding_path, dataset_name, str(hidden)), + "{}/relation_embedding_le_{}_{}.bin".format(save_folder, dataset_name, str(hidden)), binary=True, ) # need to sort to get correct indexing @@ -170,7 +174,7 @@ def load_data(news_dataset, kg_dataset, dataset_name, hidden=64, node_embedding_ [train.edge_type, valid.edge_type, test.edge_type], dim=0 ) - return data, num_features, num_relations, embedded_relations + return data, num_features, num_relations, embedded_relations, save_folder def main(_args): @@ -181,13 +185,11 @@ def main(_args): ) args = gap.parser.parse_args(_args) dataset_name = args.dataset + '_' + args.news_dataset - data, num_features, num_relations, embedded_relations = load_data( + data, num_features, num_relations, embedded_relations, save_folder = load_data( hidden=args.hidden, kg_dataset=args.dataset, news_dataset=args.news_dataset, dataset_name=dataset_name ) - embedded_relations.to(args.device) - # python run_mr_lp.py --dropout=0 --hidden=128 --l2=5e-4 --num_layers=3 --cross_layer=False --patience=200 --residual=True --residual_star=True --dataset=FB15k_237 --device=cpu --epochs=2 - + embedded_relations.to(args.device) trainer.trainer( args, dataset_name, @@ -196,6 +198,7 @@ def main(_args): num_relations=num_relations, relation_embeddings=embedded_relations, num_epoch=args.epochs, + save_folder=save_folder ) diff --git a/trainer.py b/trainer.py index 18d9e14..dfec798 100644 --- a/trainer.py +++ b/trainer.py @@ -99,12 +99,13 @@ def trainer( dataset, relation_embeddings, num_relations, + save_folder, num_features=0, epochs_per_test=1, epoch_per_val=1, num_epoch=200, save_per_epoch=100, - cal_mrr_score=True, + cal_mrr_score=True ): # GPU cuDNN auto tuner @@ -143,11 +144,6 @@ def trainer( tw.init_writer(DATASET_NAME) tw.write_text("model/info", gap.args2string(args, sort_dict=True)) - # Create directory, if it doesn't already exists - out_path = "output" - if not path.exists(out_path): - mkdir(out_path) - optimizer = torch.optim.Adam(model.parameters(), lr=args.lr, weight_decay=args.l2) scheduler = ReduceLROnPlateau( optimizer, @@ -220,6 +216,6 @@ def trainer( scheduler.step(test_loss) if epoch == num_epoch or epoch % save_per_epoch == 0: - torch.save(model, path.join(out_path, DATASET_NAME + "_" + str(num_epoch) + 'epochs' + ".pkl")) + torch.save(model, path.join(save_folder, DATASET_NAME + "_" + str(num_epoch) + 'epochs' + ".pkl")) tw.writer.close() diff --git a/utils/misc.py b/utils/misc.py new file mode 100644 index 0000000..6f3c992 --- /dev/null +++ b/utils/misc.py @@ -0,0 +1,7 @@ +from datetime import datetime as dtime + +def get_now(): + now = dtime.now() + date = str(now.date()) + time = str(now.time()).split(".")[0] + return date + " " + time \ No newline at end of file From 6225f956676db52b8fa633efe175446f480cbb6f Mon Sep 17 00:00:00 2001 From: Odd Gunnar Aspaas Date: Wed, 26 May 2021 13:14:44 +0200 Subject: [PATCH 06/12] sigmoid on score --- module/graph_star.py | 14 +++++++++++--- run_fn_detection.py | 1 - 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/module/graph_star.py b/module/graph_star.py index 3d5bbeb..8e2d3a6 100644 --- a/module/graph_star.py +++ b/module/graph_star.py @@ -237,7 +237,7 @@ def lp_score(self, z, edge_index, edge_type): z = F.dropout(z, 0.5, training=self.training) pred = self.relation_score_function( z[edge_index[0]].unsqueeze(1), - self.RW[edge_type].unsqueeze(1), + self.RW[edge_type], z[edge_index[1]].unsqueeze(1), ) return pred @@ -276,9 +276,17 @@ def add_self_loop_edge(self, edge_index, edge_type, x_size): ) return edge_index, edge_type + def DistMult2(self, head, relation, tail): + score = head @ torch.diag_embed(relation,0,1) @ torch.transpose(tail,-2,-1) + score = score.view(-1,1).squeeze(1) + return torch.sigmoid(score) + def DistMult(self, head, relation, tail): - score = head * relation * tail - return score.sum(dim=2).squeeze(1) + score = head * relation.unsqueeze(1) * tail + score = score.sum(dim=2).squeeze(1) + #experimental + return torch.sigmoid(score) + def updateZ(self, z): self.z = z diff --git a/run_fn_detection.py b/run_fn_detection.py index d5ce129..8685e4f 100644 --- a/run_fn_detection.py +++ b/run_fn_detection.py @@ -52,7 +52,6 @@ def train_val_test_split(dataset, train, val, test): dataset.test_neg_edge_index = torch.tensor( [list(neg_test[0]), list(neg_test[2])], dtype=torch.long ).narrow(1,0,anti_symmetric_test.size(1)) - return dataset From 36c899bb8a3777864ecaacad087d568b8611539c Mon Sep 17 00:00:00 2001 From: oddgas Date: Fri, 28 May 2021 11:10:20 +0200 Subject: [PATCH 07/12] hotfix to dataset --- data/FakeNewsNet/gs_train.csv | 78 +++++++++++++---------------------- 1 file changed, 29 insertions(+), 49 deletions(-) diff --git a/data/FakeNewsNet/gs_train.csv b/data/FakeNewsNet/gs_train.csv index 06ddec7..0c369dd 100644 --- a/data/FakeNewsNet/gs_train.csv +++ b/data/FakeNewsNet/gs_train.csv @@ -8,8 +8,7 @@ text_id,head,relation,tail,fb_head,fb_relation,fb_tail,label 280,Beaureguard BillyBob Johnson,supports,Confederate States of America,Beaureguard BillyBob Johnson,/base/popstra/organization/supporter./base/popstra/support/supporter,/m/020d5,FALSE 280,Fox news,reports on ,Black Lives Matter,/m/02z_b,/base/newsevents/news_reporting_organisation/news_reports./base/newsevents/news_report/event,/m/012hdhkj,FALSE 282,Andrew Jackson,role,US president,/m/0rlz,/government/politician/government_positions_held./government/government_position_held/office_position_or_title,/m/060d2,FALSE -282,Donald Trump,reports on ?,American Civil War,/m/0cqt90,,/m/0kbq,FALSE -282,Donald Trump,interviewed in,The Red Shtick,/m/0cqt90,,The Red Shtick,FALSE +282,American Civil War,reports on ?,Donald Trump,/m/0cqt90,/base/newsevents/news_reporting_organisation/news_reports./base/newsevents/news_report/event,/m/0kbq,FALSE 282,Donald Trump,member of,Party of Lincoln (republican party),/m/0cqt90,/government/politician/party./government/political_party_tenure/party,/m/07wbk,FALSE 282,Andrew Jackson,asociated with,Democratic Party,/m/0rlz,/government/politician/party./government/political_party_tenure/party,/m/0d075m,FALSE 282,Donald Trump,fan of,Kentucky Fried Chicken,/m/0cqt90,/food/diet_follower/follows_diet,/m/09b6t,FALSE @@ -17,33 +16,27 @@ text_id,head,relation,tail,fb_head,fb_relation,fb_tail,label 282,American Civil War,involved person,Colonel Sanders,/m/0kbq,/military/military_conflict/military_personnel_involved,/m/09b78,FALSE 282,Colonel Sanders,is,hero,/m/09b78,"/people/profession/specializations , /people/person/profession",/m/03kll,FALSE 286,Donald Trump,role,US president,/m/0cqt90,/government/politician/government_positions_held./government/government_position_held/office_position_or_title,/m/060d2,FALSE -286,United States Congress,legal charges against,Hillary Clinton,/m/07t31,,/m/0d06m5,FALSE -286,United States House Committee on Oversight and Government Reform,chairman ,Jason Chaffetz,United States House Committee on Oversight and Government Reform,organization/role/leaders./organization/leadership/person,/m/047blr4,FALSE -286,Jason Chaffetz,iterviewed,Donald Trump,/m/047blr4,,/m/0cqt90,FALSE +286,United States House Committee on Oversight and Government Reform,chairman ,Jason Chaffetz,United States House Committee on Oversight and Government Reform,/organization/role/leaders./organization/leadership/person,/m/047blr4,FALSE 286,FBI,director (former),James Comey,/m/02_1m,/organization/role/leaders./organization/leadership/person,/m/06r04p,FALSE -286,Hillary Clinton,betray,United States of America,/m/0d06m5,,/m/09c7w0,FALSE +286,Hillary Clinton,commits,treason,/m/0d06m5,/base/fight/crime_type/people_convicted_of_this_crime./base/crime/criminal_conviction/guilty_of,/m/07ppd,FALSE 288,Freedom Crossroads,employee,Louis Leweigh,Freedom Crossroads,/business/employer/employees./business/employment_tenure/person,Louis Leweigh,FALSE 288,Louis Leweigh,occupation,Correspondent,Louis Leweigh,/people/person/profession,/m/02k13d,FALSE 288,Monica Lewinsky,affair,Bill Clinton,/m/0509p,/celebrities/celebrity/sexual_relationships./celebrities/romantic_relationship/celebrity,/m/0157m,FALSE -288,Bill Clinton,suspected murder involvement,Monica Lewinsky,/m/0509p,,/m/0509p,FALSE +288,Bill Clinton,involved person,Monica Lewinsky,/m/0157m,/people/cause_of_death/people,/m/0509p,FALSE +288,Hillary Clinton,involved person,Monica Lewinsky,/m/0d06m5,/people/cause_of_death/people,/m/0509p,FALSE +288,Monica Lewinsky,murdered in,burglary,/m/0509p,/people/deceased_person/cause_of_death,/m/016x3s,FALSE 295,public servant (government official),people with role,Maxine Waters,/m/035y33,/people/profession/people_with_this_profession,/m/024tyk,FALSE 295,Donald Trump,role,US president,/m/0cqt90,/government/politician/government_positions_held./government/government_position_held/office_position_or_title,/m/060d2,FALSE -295,Donald Trump,asociated with,Russia,/m/0cqt90,,/m/06bnz,FALSE -295,Maxine Waters,condemn,White House,/m/024tyk,,/m/081sq,FALSE +295,Donald Trump,asociated with,Russia,/m/0cqt90,/base/popstra/celebrity/endorsements./base/popstra/paid_support/company,/m/06bnz,FALSE 295,Maxine Waters,medical condition,Dementia,/m/024tyk,/medicine/notable_person_with_medical_condition/condition,/m/09klv,FALSE 295,Maxine Waters,asociated with,Democratic Party,/m/024tyk,/government/politician/party./government/political_party_tenure/party,/m/0d075m,FALSE 295,Maxine Waters,part of,United States Congress,/m/024tyk,/people/appointed_role/appointment./people/appointment/appointed_by,/m/07t31,FALSE 295,Maxine Waters,ethnicity,African American,/m/024tyk,/people/person/ethnicity,/m/0x67,FALSE 295,1tch,employee,Teddy Stick,1tch,/business/employer/employees./business/employment_tenure/person,Teddy Stick,FALSE -300,Donald Trump,cheered for,crisis (Housing Crisis),/m/0cqt90,,/m/02gyy_,TRUE -300,Hillary Clinton,critiqued,Donald Trump,/m/0d06m5,,/m/0cqt90,TRUE -300,Donald Trump,interviewed in,The Globe and Mail,/m/0cqt90,,/m/01j1gk,TRUE -300,crisis (housing),asociated with,Great Recession,/m/02gyy_,,/m/03qk63k,TRUE 300,Great Recession,affected negatively,the poor,/m/03qk63k,/influence/influence_node/influenced,the poor,TRUE 303,Jean Shepard,member of,Grand Ole Opry,/m/01npnk3,/organization/membership_organization/members./organization/organization_membership/member,/m/0gg86,TRUE 303,Jean Shepard,inducted,Country Music Hall of Fame,/m/01npnk3,/award/hall_of_fame_inductee/hall_of_fame_inductions./award/hall_of_fame_induction/hall_of_fame,/m/0ggkm,TRUE 303,Country Music Hall of Fame,hall_of_fame_dicipline,music,/m/0ggkm,/award/hall_of_fame/discipline,/m/04rlf,TRUE -303,Jean Shepard,interviewed in,Women Of Country,/m/01npnk3,,Women Of Country,TRUE 303,Jean Shepard,medical condition,Parkinsons,/m/01npnk3,/medicine/notable_person_with_medical_condition/condition,/m/0g02vk,TRUE 303,Jean Shepard,born in,Oklahoma,/m/01npnk3,/people/person/place_of_birth,/m/05mph,TRUE 303,Jean Shepard,lived in,"Visalia, California",/m/01npnk3,/people/person/places_lived./people/place_lived/location,/m/0r80l,TRUE @@ -52,61 +45,49 @@ text_id,head,relation,tail,fb_head,fb_relation,fb_tail,label 303,singer,people with profession,Jean Shepard,/m/09l65,/people/profession/people_with_this_profession,/m/01npnk3,TRUE 303,bassist,musical profession,Jean Shepard,/m/0gbbt,/people/profession/people_with_this_profession,/m/01npnk3,TRUE 307,Tim Kaine,associated with,Democratic Party,/m/053f8h,/government/politician/party./government/political_party_tenure/party,/m/0d075m,TRUE -307,Tim Kaine,"""running mate""",Hillary Clinton,/m/053f8h,,/m/0d06m5,TRUE +307,Hillary Clinton,appoint,Tim Kaine,/m/0d06m5,/people/appointer/appointment_made./people/appointment/appointee,/m/053f8h,TRUE 307,Tim Kaine,role,Senator,/m/053f8h,/people/person/employment_history./business/employment_tenure/title,/m/048zv9l,TRUE 307,Tim Kaine,lives in ,Virginia,/m/053f8h,/people/person/places_lived./people/place_lived/location,/m/07z1m,TRUE -307,Tim Kaine,works in,Virginia,/m/053f8h,,/m/07z1m,TRUE -307,Tim Kaine,interviewed in,The Ellen DeGeneres Show,/m/053f8h,,/m/0528kc,TRUE -307,Tim Kaine,nominee,Vice President of the U.S,/m/053f8h,,/m/080v2,TRUE -307,Mike Pence,asociated with ,Donald Trump,/m/022r9r,,/m/0cqt90,TRUE -307,Tim Kaine,grew up as,Republican Party,/m/053f8h,,/m/07wbk,TRUE -307,Hillary Clinton,grew up as,Republican Party,/m/0d06m5,,/m/07wbk,TRUE +307,Virginia,officeholder,Tim Kaine,/m/07z1m,/government/political_district/representatives./government/government_position_held/office_holder,/m/053f8h,TRUE +307,Mike Pence,asociated with ,Donald Trump,/m/022r9r,/celebrities/celebrity/celebrity_friends./celebrities/friendship/friend,/m/0cqt90,TRUE 308,Hofstra University,located in,New York,/m/02607j,/location/location/containedby,/m/02_286,TRUE -308,Not Who We Are campaign,is against,Donald Trump,Not Who We Are campaign,,/m/0cqt90,TRUE 308,Not Who We Are campaign,campain at location,Hofstra University,Not Who We Are campaign,/location/location/containedby,/m/02607j,TRUE -308,Donald Trump,is part of ,racism,/m/0cqt90,,/m/06d4h,TRUE -308,Not Who We Are campaign,manager of,Josh Hendler,Not Who We Are campaign,organization/role/leaders./organization/leadership/person,Josh Hendler,TRUE +308,Donald Trump,is part of ,racism,/m/0cqt90,/base/fight/crime_type/people_convicted_of_this_crime./base/crime/criminal_conviction/guilty_of,/m/06d4h,TRUE +308,Not Who We Are campaign,manager of,Josh Hendler,Not Who We Are campaign,/organization/role/leaders./organization/leadership/person,Josh Hendler,TRUE 308,Rob Bielunas,student at,Hofstra University,Rob Bielunas,/people/person/education./education/education/institution,/m/02607j,TRUE 308,Donald Trump,role,presidential candidate,/m/0cqt90,/government/politician/government_positions_held./government/government_position_held/office_position_or_title,/m/066q5c,TRUE -328,Donald Trump,is part of,racism,/m/0cqt90,,racist,TRUE +328,Donald Trump,is part of,racism,/m/0cqt90,/base/fight/crime_type/people_convicted_of_this_crime./base/crime/criminal_conviction/guilty_of,/m/06d4h,TRUE 328,Barack Obama,role,US president,/m/02mjmr,/government/politician/government_positions_held./government/government_position_held/office_position_or_title,/m/060d2,TRUE 328,Barack Obama,born in,Hawaii,/m/02mjmr,/people/person/place_of_birth,/m/03gh4,TRUE -328,Lester Holt,profession,moderator,/m/06jvj7,,moderator,TRUE +328,Lester Holt,profession,moderator,/m/06jvj7,/people/person/profession,moderator,TRUE 328,Donald Trump,role,presidential candidate,/m/0cqt90,/government/politician/government_positions_held./government/government_position_held/office_position_or_title,/m/066q5c,TRUE 328,Donald Trump,associated with,Republican Party,/m/0cqt90,/government/politician/party./government/political_party_tenure/party,/m/07wbk,TRUE 328,Donald Trump,role,presidential candidate,/m/0cqt90,/government/politician/government_positions_held./government/government_position_held/office_position_or_title,/m/066q5c,TRUE 328,Hillary Clinton,role,presidential candidate,/m/0d06m5,/government/politician/government_positions_held./government/government_position_held/office_position_or_title,/m/066q5c,TRUE 2,HIVAIDs drug,cause,death,/m/01cw8w,/medicine/disease/causes,/m/0294j,FALSE 2,Bill Clinton,associates with,Ranbaxy,/m/0157m,/base/popstra/celebrity/endorsements./base/popstra/paid_support/company,/m/0243xw,FALSE -2,Ranbaxy,banned from,The united states of america,/m/0243xw,,/m/09c7w0,FALSE 2,Bill Clinton,Chairman,Clinton Health Access Initiative,/m/0157m,/organization/organization/board_members./organization/organization_board_membership/member,Clinton Health Access Initiative,FALSE -2,Bill Clinton,involved with,corruption,/m/0157m,,/m/09pngm,FALSE +2,Bill Clinton,involved with,corruption,/m/0157m,/base/fight/crime_type/people_convicted_of_this_crime./base/crime/criminal_conviction/guilty_of,/m/09pngm,FALSE 3,Donald Trump,encourage ,manslaugther,/m/0cqt90,/influence/influence_node/influenced,/m/0x2gl,FALSE -3,Hillary Clinton,supports,open border,/m/0d06m5,,/m/043l_lj,FALSE -3,Syria,will destory,The united states of america,/m/06vbd,,/m/09c7w0,FALSE 3,Donald Trump,ethnicity,Oompa Loompa,/m/0cqt90,/fictional_universe/fictional_character/species,Oompa Loompa,FALSE -3,Donald Trump,supports,concentration camps,/m/0cqt90,,/m/01b5_d,FALSE +3,Donald Trump,supports,concentration camps,/m/0cqt90,/base/popstra/celebrity/supporter./base/popstra/support/supported_organization,/m/01b5_d,FALSE 8,Hillary Clinton,collaboration,CNN,/m/0d06m5,/influence/influence_node/influenced_by,/m/0gsgr,FALSE 8,Hillary Clinton,medical condition,Parkinsons disease,/m/0d06m5,/medicine/notable_person_with_medical_condition/condition,/m/0g02vk,FALSE -8,Hillary Clinton,involved with,corruption,/m/0d06m5,,/m/09pngm,FALSE -8,Democratic Party,oppose,1964 Civil Rights Act,/m/0d075m,,/m/019twb,FALSE -8,Democratic Party,fought to maintain,Slavery,/m/0d075m,,/m/06z49,FALSE -8,Republican Party,oppose ,Slavery,/m/0d075m,,/m/06z49,FALSE +8,Hillary Clinton,involved with,corruption,/m/0d06m5,/influence/influence_node/influenced,/m/09pngm,FALSE +8,Democratic Party,supports,slavery,/m/0d075m,/base/popstra/celebrity/supporter./base/popstra/support/supported_organization,/m/06z49,FALSE +8,Republican Party,supports,emancipation,/m/0d075m,/base/popstra/celebrity/supporter./base/popstra/support/supported_organization,/m/0dtbg9,FALSE 8,Bill Clinton,is,sexual predator,/m/0157m,/base/fight/crime_type/people_convicted_of_this_crime./base/crime/criminal_conviction/guilty_of,/m/01fdjk,FALSE -8,Hillary Clinton,is ,crime,/m/0d06m5,,/m/01rbb,FALSE 8,Hillary Clinton,is,racism,/m/0d06m5,/base/fight/crime_type/people_convicted_of_this_crime./base/crime/criminal_conviction/guilty_of,/m/06d4h,FALSE -11,Barrack Obama,created,ISIS,/m/02mjmr,,/m/027x630,FALSE +11,Barack Obama,created,ISIS,/m/02mjmr,/organization/organization_founder/organizations_founded,/m/027x630,FALSE 11,Europe,supports,ISIS,/m/02j9z,/base/popstra/celebrity/supporter./base/popstra/support/supported_organization,/m/027x630,FALSE 11,The United States,supports,ISIS,/m/09c7w0,/base/popstra/celebrity/supporter./base/popstra/support/supported_organization,/m/027x630,FALSE -14,ISIS,creator,Barrack Obama,/m/027x630,/organization/organization/founders,/m/02mjmr,FALSE -14,Barrack Obama,appear at,Coachella,/m/02mjmr,,/m/02h71z,FALSE +14,ISIS,creator,Barack Obama,/m/027x630,/organization/organization/founders,/m/02mjmr,FALSE +14,Barack Obama,appear at,Coachella,/m/02mjmr,/projects/project_role/projects./projects/project_participation/participant,/m/02h71z,FALSE 14,Michelle Obama,medical condition,sleep apnea,/m/025s5v9,/medicine/notable_person_with_medical_condition/condition,/m/071d3,FALSE -14,Barrack Obama,loves,electronic music,/m/02mjmr,,/m/02lkt,FALSE -14,Barrack Obama,uses,ecstasy,/m/02mjmr,,/m/02qpq,FALSE -14,Barrack Obama,profession,terrorism,/m/02mjmr,/people/person/profession,/m/07jq_,FALSE -14,Barrack Obama,place of birth,Kenya,/m/02mjmr,/people/person/place_of_birth,/m/019rg5,FALSE -14,Barrack Obama,religion,Islam,/m/02mjmr,/people/person/religion,/m/0flw86,FALSE -14,Barrack Obama,guilty of,crime of possessing illegal drugs,/m/02mjmr,/base/fight/crime_type/people_convicted_of_this_crime./base/crime/criminal_conviction/guilty_of,/m/05cxz3,FALSE +14,Barack Obama,profession,terrorism,/m/02mjmr,/people/person/profession,/m/07jq_,FALSE +14,Barack Obama,place of birth,Kenya,/m/02mjmr,/people/person/place_of_birth,/m/019rg5,FALSE +14,Barack Obama,religion,Islam,/m/02mjmr,/people/person/religion,/m/0flw86,FALSE +14,Barack Obama,guilty of,crime of possessing illegal drugs,/m/02mjmr,/base/fight/crime_type/people_convicted_of_this_crime./base/crime/criminal_conviction/guilty_of,/m/05cxz3,FALSE 14,Hillary Clinton,guilty of,crime of possessing illegal drugs,/m/0d06m5,/base/fight/crime_type/people_convicted_of_this_crime./base/crime/criminal_conviction/guilty_of,/m/05cxz3,FALSE 119,Megan Rapinoe,profession,soccer player,/m/05mz656,/people/person/profession,/m/0pcq81q,TRUE 119,Megan Rapinoe,supports,BLM,/m/05mz656,/base/popstra/celebrity/supporter./base/popstra/support/supported_organization,/m/012hdhkj,TRUE @@ -121,13 +102,12 @@ text_id,head,relation,tail,fb_head,fb_relation,fb_tail,label 173,Barack Obama,born in,Hawaii,/m/02mjmr,/people/person/place_of_birth,/m/03gh4,TRUE 174,Benjamin Netanyahu,profession,prime minister,/m/0fm2h,/government/politician/government_positions_held./government/government_position_held/office_position_or_title,/m/060bp,TRUE 174,Benjamin Netanyahu,place of birth,Israel,/m/0fm2h,/people/person/place_of_birth,/m/03spz,TRUE -174,The United States,alliance,Israel,/m/09c7w0,,/m/03spz,TRUE 174,Hillary Clinton,associated with,Democratic Party,/m/0d06m5,/government/politician/party./government/political_party_tenure/party,/m/0d075m,TRUE 174,Hillary Clinton,role,presidential candidate,/m/0d06m5,/government/politician/government_positions_held./government/government_position_held/office_position_or_title,/m/066q5c,TRUE -178,Ben LaBolt,press secretary,Obamas 2012 reelection campaign,Ben LaBolt,,/m/0gk_h41,TRUE -178,Josh Ernest,press secretary,The White House,/m/0l8ps43,,/m/081sq,TRUE +178,Ben LaBolt,press secretary,Obamas 2012 reelection campaign,Ben LaBolt,/projects/project_role/projects./projects/project_participation/participant,/m/0gk_h41,TRUE +178,Josh Ernest,press secretary,Obamas 2012 reelection campaign,/m/0l8ps43,/projects/project_role/projects./projects/project_participation/participant,/m/081sq,TRUE 178,Hillary Clinton,associated with,Democratic Party,/m/0d06m5,/government/politician/party./government/political_party_tenure/party,/m/0d075m,TRUE 178,Hillary Clinton,role,presidential candidate,/m/0d06m5,/government/politician/government_positions_held./government/government_position_held/office_position_or_title,/m/066q5c,TRUE -178,Barrack Obama,profession,president,/m/02mjmr,/government/politician/government_positions_held./government/government_position_held/office_position_or_title,/m/060d2,TRUE +178,Barack Obama,profession,president,/m/02mjmr,/government/politician/government_positions_held./government/government_position_held/office_position_or_title,/m/060d2,TRUE 178,Donald Trump,associated with,Republican Party,/m/0cqt90,/government/politician/party./government/political_party_tenure/party,/m/07wbk,TRUE 178,Donald Trump,role,presidential candidate,/m/0cqt90,/government/politician/government_positions_held./government/government_position_held/office_position_or_title,/m/066q5c,TRUE \ No newline at end of file From 668cfde3c8896c70c8f7e9e6494b21049875a611 Mon Sep 17 00:00:00 2001 From: oddgas Date: Fri, 28 May 2021 19:39:55 +0200 Subject: [PATCH 08/12] fix bug in tensorboard --- utils/misc.py | 4 ++-- utils/tensorboard_writer.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/utils/misc.py b/utils/misc.py index 6f3c992..48cba8c 100644 --- a/utils/misc.py +++ b/utils/misc.py @@ -1,7 +1,7 @@ from datetime import datetime as dtime - +import numpy as np def get_now(): now = dtime.now() date = str(now.date()) time = str(now.time()).split(".")[0] - return date + " " + time \ No newline at end of file + return date + "_" + time + str(np.random.randint(0,100)) \ No newline at end of file diff --git a/utils/tensorboard_writer.py b/utils/tensorboard_writer.py index 0624ebc..eabfbdf 100644 --- a/utils/tensorboard_writer.py +++ b/utils/tensorboard_writer.py @@ -53,7 +53,7 @@ def log_epoch( def write_text(path, text): - writer.add_text(path, text, epochs) + writer.add_text(path, text, steps) def init_writer(name): From 71d18989847c55c56a6bf2e3fc23c44e448b15a1 Mon Sep 17 00:00:00 2001 From: oddgas Date: Sat, 5 Jun 2021 13:41:57 +0200 Subject: [PATCH 09/12] fix sigmoid --- module/graph_star.py | 2 +- trainer.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/module/graph_star.py b/module/graph_star.py index 8e2d3a6..d7e48fb 100644 --- a/module/graph_star.py +++ b/module/graph_star.py @@ -285,7 +285,7 @@ def DistMult(self, head, relation, tail): score = head * relation.unsqueeze(1) * tail score = score.sum(dim=2).squeeze(1) #experimental - return torch.sigmoid(score) + return score def updateZ(self, z): diff --git a/trainer.py b/trainer.py index dfec798..b0048cc 100644 --- a/trainer.py +++ b/trainer.py @@ -81,6 +81,7 @@ def train_transductive( ) loss = model.lp_loss(pred, y) + pred = torch.sigmoid(pred) lp_auc, lp_ap = model.lp_test(pred, y) if ((mode == "test")) and cal_mrr_score: From 96600c6c151cc404fd03b3e8eda0463f39578d76 Mon Sep 17 00:00:00 2001 From: oddgas Date: Wed, 9 Jun 2021 17:18:41 +0200 Subject: [PATCH 10/12] new loss function --- module/graph_star.py | 14 ++++++++++---- trainer.py | 1 - 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/module/graph_star.py b/module/graph_star.py index d7e48fb..7f8130b 100644 --- a/module/graph_star.py +++ b/module/graph_star.py @@ -167,7 +167,7 @@ def __init__( ) self.rl = nn.Linear(num_relations, hid) self.RW = nn.Parameter(relation_embeddings) - self.LP_loss = nn.BCEWithLogitsLoss() + self.LP_loss = nn.BCELoss() def forward(self, x, edge_index, batch, star=None, y=None, edge_type=None): if self.training: @@ -285,7 +285,7 @@ def DistMult(self, head, relation, tail): score = head * relation.unsqueeze(1) * tail score = score.sum(dim=2).squeeze(1) #experimental - return score + return torch.sigmoid(score) def updateZ(self, z): @@ -318,6 +318,8 @@ def lp_log_ranks( relation = torch.full((z.size(0),), pos_edge_type[i], dtype=dt, device=dev) # Currently distmult function score pred = self.lp_score(z, head_pred_ei, relation) + # Round to 3 decimalplaces + pred = torch.round(pred * 10**4) / (10**4) # score of actual triple target = pred[pos_edge_index[0][i]] @@ -341,6 +343,7 @@ def lp_log_ranks( # tail batch < h, r, ?> # for all triples + for i in tqdm(range(pos_edge_index.size(1)), desc="tail prediction"): # For the head from this triple (pos_edge_index[0][i]) to all tails @@ -357,6 +360,9 @@ def lp_log_ranks( relation = torch.full((z.size(0),), pos_edge_type[i], dtype=dt, device=dev) # Currently distmult function score pred = self.lp_score(z, tail_pred_ei, relation) + + # Round to 3 decimalplaces + pred = torch.round(pred * 10**4) / (10**4) # score of actual triple target = pred[pos_edge_index[1][i]] @@ -381,8 +387,8 @@ def lp_log_ranks( ranks = np.array(ranks) res = { - "MRR": (1 / ranks).sum() / len(ranks), - "MR": (ranks).sum() / len(ranks), + "MRR": ((1 / ranks).sum().float() / len(ranks)).item(), + "MR": ((ranks).sum().float() / len(ranks)).item(), "HIT@1": (ranks <= 1).sum() / len(ranks), "HIT@3": (ranks <= 3).sum() / len(ranks), "HIT@10": (ranks <= 10).sum() / len(ranks), diff --git a/trainer.py b/trainer.py index b0048cc..dfec798 100644 --- a/trainer.py +++ b/trainer.py @@ -81,7 +81,6 @@ def train_transductive( ) loss = model.lp_loss(pred, y) - pred = torch.sigmoid(pred) lp_auc, lp_ap = model.lp_test(pred, y) if ((mode == "test")) and cal_mrr_score: From 6d9a361b89cf61a08478c82b6bfc1928b75ab0b0 Mon Sep 17 00:00:00 2001 From: oddgas Date: Wed, 9 Jun 2021 17:20:12 +0200 Subject: [PATCH 11/12] assert training --- run_fn_detection.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/run_fn_detection.py b/run_fn_detection.py index 8685e4f..c342bce 100644 --- a/run_fn_detection.py +++ b/run_fn_detection.py @@ -96,7 +96,7 @@ def load_data(news_dataset, kg_dataset, dataset_name, hidden=64, node_embedding_ # create model folder save_folder = "model_" + misc.get_now() print("creating model folder: " + save_folder + "...") - mkdir(save_folder) + #mkdir(save_folder) # needed for embedding all nodes across datasets (will use edge_index to split datasets) all_data = pd.concat([train, valid]) @@ -136,7 +136,7 @@ def load_data(news_dataset, kg_dataset, dataset_name, hidden=64, node_embedding_ print('all_data.x.size: ', all_data.x.size()) # create node embeddings if none exists cne.create_node_embedding( - all_data, dataset_name, dimensions=node_embedding_size, workers=4, path=save_folder + all_data, dataset_name, dimensions=node_embedding_size, workers=1, path=save_folder ) cre.create_relation_embedding(relations, le_relation, dataset_name, dimensions=hidden, path=save_folder) embedded_nodes = KeyedVectors.load_word2vec_format( From 431d2fe3983370439f50c59371172522d63c7d4e Mon Sep 17 00:00:00 2001 From: oddgas Date: Wed, 9 Jun 2021 17:21:32 +0200 Subject: [PATCH 12/12] hot fix --- run_fn_detection.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/run_fn_detection.py b/run_fn_detection.py index c342bce..8685e4f 100644 --- a/run_fn_detection.py +++ b/run_fn_detection.py @@ -96,7 +96,7 @@ def load_data(news_dataset, kg_dataset, dataset_name, hidden=64, node_embedding_ # create model folder save_folder = "model_" + misc.get_now() print("creating model folder: " + save_folder + "...") - #mkdir(save_folder) + mkdir(save_folder) # needed for embedding all nodes across datasets (will use edge_index to split datasets) all_data = pd.concat([train, valid]) @@ -136,7 +136,7 @@ def load_data(news_dataset, kg_dataset, dataset_name, hidden=64, node_embedding_ print('all_data.x.size: ', all_data.x.size()) # create node embeddings if none exists cne.create_node_embedding( - all_data, dataset_name, dimensions=node_embedding_size, workers=1, path=save_folder + all_data, dataset_name, dimensions=node_embedding_size, workers=4, path=save_folder ) cre.create_relation_embedding(relations, le_relation, dataset_name, dimensions=hidden, path=save_folder) embedded_nodes = KeyedVectors.load_word2vec_format(