From 89e0b9288ae31b7ee914d46f84472a3ca859f2b5 Mon Sep 17 00:00:00 2001 From: xxhu Date: Fri, 9 Jul 2021 01:10:36 +0800 Subject: [PATCH] Initialize the stanford corenlp pipelines in NLPUtil at one time; Add the newly generated query logs for lcquad and qald --- ...ery_log_test_lc-quad_2021_07_08 23_48.txt} | 4002 ++++++++--------- ...uery_log_test_qald-9_2021_07_09 00_51.txt} | 902 ++-- .../cn/edu/nju/ws/edgqa/domain/edg/EDG.java | 4 +- .../java/cn/edu/nju/ws/edgqa/main/EDGQA.java | 1 + .../cn/edu/nju/ws/edgqa/utils/NLPUtil.java | 432 +- 5 files changed, 2541 insertions(+), 2800 deletions(-) rename query_logs/{query_log_test_lc-quad_2021_07_04 16_19.txt => query_log_test_lc-quad_2021_07_08 23_48.txt} (93%) rename query_logs/{query_log_test_qald-9_2021_07_04 17_50.txt => query_log_test_qald-9_2021_07_09 00_51.txt} (95%) diff --git a/query_logs/query_log_test_lc-quad_2021_07_04 16_19.txt b/query_logs/query_log_test_lc-quad_2021_07_08 23_48.txt similarity index 93% rename from query_logs/query_log_test_lc-quad_2021_07_04 16_19.txt rename to query_logs/query_log_test_lc-quad_2021_07_08 23_48.txt index d4e3280..3598a51 100644 --- a/query_logs/query_log_test_lc-quad_2021_07_04 16_19.txt +++ b/query_logs/query_log_test_lc-quad_2021_07_08 23_48.txt @@ -7,8 +7,8 @@ Question 0: Which architect of Marine Corps Air Station Kaneohe Bay was also ten [INFO] Golden Answer: [http://dbpedia.org/resource/United_States_Navy] [INFO] Predicted Answer: [http://dbpedia.org/resource/United_States_Navy] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 1, P: 1.000, R: 1.000, macro F1: 1.000, macro F1*: 1.000 -[INFO] Current time: 5531 ms, average: 5555.00 ms/ques, entity detection: 400.00 ms/ques (linking tool: 240.00 ms/ques), relation detection: 901.00 ms/ques (similarity: 668.00 ms/ques), EDG generation: 2603.00 ms/ques +[INFO] Cumulative metrics, sample: 1, P: 1.000, R: 1.000, macro F1: 1.000 +[INFO] Current question consumed time: 8293 ms, average: 8317.00 ms/ques, entity detection: 298.00 ms/ques (linking tool: 183.00 ms/ques), relation detection: 947.00 ms/ques (similarity: 767.00 ms/ques), EDG generation: 5577.00 ms/ques [INFO] Entity Linking P: 1.0, R: 1.0, micro F1: 1.0, macro F1: 1.0 [INFO] Relation Linking P: 1.0, R: 1.0, micro F1: 1.0, macro F1: 1.0 [INFO] Type Linking P: 1.0, R: 1.0, micro F1: 1.0, macro F1: 1.0 @@ -21,8 +21,8 @@ Question 1: Which sects people are part of local political parties which fall un [INFO] Golden Answer: [http://dbpedia.org/resource/Sunni_Islam, http://dbpedia.org/resource/Islam, http://dbpedia.org/resource/Wahhabism] [INFO] Predicted Answer: [http://dbpedia.org/resource/Sunni_Islam, http://dbpedia.org/resource/Islam, http://dbpedia.org/resource/Wahhabism, http://dbpedia.org/resource/Columbus_School_for_Girls, http://dbpedia.org/resource/Soviet_Union, http://dbpedia.org/resource/Jawahar_Navodaya_Vidyalaya,_Sitapur, http://dbpedia.org/resource/Culbert_Olson, http://dbpedia.org/resource/Miloš_Zeman, http://dbpedia.org/resource/Ibrahim_Alizade, http://dbpedia.org/resource/Vaso_Čubrilović, http://dbpedia.org/resource/East_Bay_Innovation_Academy, Islam@en] [INFO] P: 0.2727272727272727, R: 1.0, F: 0.42857142857142855 -[INFO] Cumulative metrics, sample: 2, P: 0.636, R: 1.000, macro F1: 0.714, macro F1*: 0.778 -[INFO] Current time: 4709 ms, average: 5132.00 ms/ques, entity detection: 404.50 ms/ques (linking tool: 293.00 ms/ques), relation detection: 1344.00 ms/ques (similarity: 1140.50 ms/ques), EDG generation: 1367.50 ms/ques +[INFO] Cumulative metrics, sample: 2, P: 0.636, R: 1.000, macro F1: 0.778 +[INFO] Current question consumed time: 4528 ms, average: 6423.00 ms/ques, entity detection: 344.00 ms/ques (linking tool: 258.50 ms/ques), relation detection: 1349.00 ms/ques (similarity: 1133.00 ms/ques), EDG generation: 2855.50 ms/ques [INFO] Entity Linking P: 1.0, R: 1.0, micro F1: 1.0, macro F1: 1.0 [INFO] Relation Linking P: 1.0, R: 1.0, micro F1: 1.0, macro F1: 1.0 [INFO] Type Linking P: 0.5, R: 0.5, micro F1: 0.5, macro F1: 0.5 @@ -35,8 +35,8 @@ Question 2: List common systems are developed by of the Google Videos and writte [INFO] Golden Answer: [http://dbpedia.org/resource/Google] [INFO] Predicted Answer: [http://dbpedia.org/resource/Google] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 3, P: 0.758, R: 1.000, macro F1: 0.810, macro F1*: 0.862 -[INFO] Current time: 2046 ms, average: 4103.33 ms/ques, entity detection: 434.67 ms/ques (linking tool: 317.00 ms/ques), relation detection: 1045.33 ms/ques (similarity: 915.33 ms/ques), EDG generation: 937.33 ms/ques +[INFO] Cumulative metrics, sample: 3, P: 0.758, R: 1.000, macro F1: 0.862 +[INFO] Current question consumed time: 2109 ms, average: 4985.00 ms/ques, entity detection: 406.33 ms/ques (linking tool: 304.67 ms/ques), relation detection: 1066.00 ms/ques (similarity: 898.33 ms/ques), EDG generation: 1924.33 ms/ques [INFO] Entity Linking P: 1.0, R: 1.0, micro F1: 1.0, macro F1: 1.0 [INFO] Relation Linking P: 1.0, R: 1.0, micro F1: 1.0, macro F1: 1.0 [INFO] Type Linking P: 0.6666666666666666, R: 0.6666666666666666, micro F1: 0.6666666666666666, macro F1: 0.6666666666666666 @@ -49,8 +49,8 @@ Question 3: For which games are Sam Loyd and Eric Schiller both famous? [INFO] Golden Answer: [http://dbpedia.org/resource/Chess] [INFO] Predicted Answer: [http://dbpedia.org/resource/Chess] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 4, P: 0.818, R: 1.000, macro F1: 0.857, macro F1*: 0.900 -[INFO] Current time: 1305 ms, average: 3403.75 ms/ques, entity detection: 385.50 ms/ques (linking tool: 283.75 ms/ques), relation detection: 860.25 ms/ques (similarity: 766.25 ms/ques), EDG generation: 710.75 ms/ques +[INFO] Cumulative metrics, sample: 4, P: 0.818, R: 1.000, macro F1: 0.900 +[INFO] Current question consumed time: 1250 ms, average: 4051.25 ms/ques, entity detection: 363.00 ms/ques (linking tool: 274.50 ms/ques), relation detection: 877.50 ms/ques (similarity: 756.00 ms/ques), EDG generation: 1455.00 ms/ques [INFO] Entity Linking P: 1.0, R: 1.0, micro F1: 1.0, macro F1: 1.0 [INFO] Relation Linking P: 1.0, R: 1.0, micro F1: 1.0, macro F1: 1.0 [INFO] Type Linking P: 0.5, R: 0.5, micro F1: 0.5, macro F1: 0.5 @@ -63,8 +63,8 @@ Question 4: Is Peter Piper Pizza in the pizza industry? [INFO] Golden Answer: [true] [INFO] Predicted Answer: [true] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 5, P: 0.855, R: 1.000, macro F1: 0.886, macro F1*: 0.922 -[INFO] Current time: 1279 ms, average: 2979.00 ms/ques, entity detection: 360.20 ms/ques (linking tool: 261.20 ms/ques), relation detection: 792.60 ms/ques (similarity: 687.60 ms/ques), EDG generation: 571.80 ms/ques +[INFO] Cumulative metrics, sample: 5, P: 0.855, R: 1.000, macro F1: 0.922 +[INFO] Current question consumed time: 1270 ms, average: 3495.00 ms/ques, entity detection: 338.00 ms/ques (linking tool: 245.40 ms/ques), relation detection: 811.40 ms/ques (similarity: 673.60 ms/ques), EDG generation: 1167.20 ms/ques [INFO] Entity Linking P: 1.0, R: 1.0, micro F1: 1.0, macro F1: 1.0 [INFO] Relation Linking P: 1.0, R: 1.0, micro F1: 1.0, macro F1: 1.0 [INFO] Type Linking P: 0.6, R: 0.6, micro F1: 0.6, macro F1: 0.6 @@ -77,8 +77,8 @@ Question 5: Name the artist who made Dream Dancing and is often associated with [INFO] Golden Answer: [http://dbpedia.org/resource/Ella_Fitzgerald] [INFO] Predicted Answer: [http://dbpedia.org/resource/Ella_Fitzgerald] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 6, P: 0.879, R: 1.000, macro F1: 0.905, macro F1*: 0.935 -[INFO] Current time: 3052 ms, average: 2991.17 ms/ques, entity detection: 368.50 ms/ques (linking tool: 258.50 ms/ques), relation detection: 767.50 ms/ques (similarity: 689.67 ms/ques), EDG generation: 484.33 ms/ques +[INFO] Cumulative metrics, sample: 6, P: 0.879, R: 1.000, macro F1: 0.935 +[INFO] Current question consumed time: 2777 ms, average: 3375.33 ms/ques, entity detection: 352.67 ms/ques (linking tool: 251.50 ms/ques), relation detection: 771.17 ms/ques (similarity: 677.33 ms/ques), EDG generation: 979.33 ms/ques [INFO] Entity Linking P: 1.0, R: 1.0, micro F1: 1.0, macro F1: 1.0 [INFO] Relation Linking P: 1.0, R: 1.0, micro F1: 1.0, macro F1: 1.0 [INFO] Type Linking P: 0.6666666666666666, R: 0.6666666666666666, micro F1: 0.6666666666666666, macro F1: 0.6666666666666666 @@ -91,8 +91,8 @@ Question 6: Who is the office holder with deputies as Neil Brown and Andrew Peac [INFO] Golden Answer: [http://dbpedia.org/resource/John_Howard] [INFO] Predicted Answer: [http://dbpedia.org/resource/John_Howard] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 7, P: 0.896, R: 1.000, macro F1: 0.918, macro F1*: 0.945 -[INFO] Current time: 3021 ms, average: 2995.43 ms/ques, entity detection: 358.14 ms/ques (linking tool: 248.71 ms/ques), relation detection: 808.71 ms/ques (similarity: 765.00 ms/ques), EDG generation: 432.29 ms/ques +[INFO] Cumulative metrics, sample: 7, P: 0.896, R: 1.000, macro F1: 0.945 +[INFO] Current question consumed time: 3062 ms, average: 3330.57 ms/ques, entity detection: 344.86 ms/ques (linking tool: 242.71 ms/ques), relation detection: 815.00 ms/ques (similarity: 756.57 ms/ques), EDG generation: 848.43 ms/ques [INFO] Entity Linking P: 1.0, R: 1.0, micro F1: 1.0, macro F1: 1.0 [INFO] Relation Linking P: 1.0, R: 1.0, micro F1: 1.0, macro F1: 1.0 [INFO] Type Linking P: 0.5714285714285714, R: 0.5714285714285714, micro F1: 0.5714285714285714, macro F1: 0.5714285714285714 @@ -105,8 +105,8 @@ Question 7: Where did these popular aeroplanes - Focke Wulf 260 and Start+Flug H [INFO] Golden Answer: [http://dbpedia.org/resource/West_Germany] [INFO] Predicted Answer: [http://dbpedia.org/resource/West_Germany] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 8, P: 0.909, R: 1.000, macro F1: 0.929, macro F1*: 0.952 -[INFO] Current time: 2975 ms, average: 2992.88 ms/ques, entity detection: 352.62 ms/ques (linking tool: 240.38 ms/ques), relation detection: 852.12 ms/ques (similarity: 756.88 ms/ques), EDG generation: 389.75 ms/ques +[INFO] Cumulative metrics, sample: 8, P: 0.909, R: 1.000, macro F1: 0.952 +[INFO] Current question consumed time: 3027 ms, average: 3292.62 ms/ques, entity detection: 336.50 ms/ques (linking tool: 233.62 ms/ques), relation detection: 852.62 ms/ques (similarity: 748.88 ms/ques), EDG generation: 750.25 ms/ques [INFO] Entity Linking P: 1.0, R: 1.0, micro F1: 1.0, macro F1: 1.0 [INFO] Relation Linking P: 0.875, R: 0.875, micro F1: 0.875, macro F1: 0.875 [INFO] Type Linking P: 0.625, R: 0.625, micro F1: 0.625, macro F1: 0.625 @@ -119,8 +119,8 @@ Question 8: Which television show 's producer is Stephen E. Ambrose and written [INFO] Golden Answer: [http://dbpedia.org/resource/Band_of_Brothers_(miniseries)] [INFO] Predicted Answer: [http://dbpedia.org/resource/Band_of_Brothers_(miniseries)] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 9, P: 0.919, R: 1.000, macro F1: 0.937, macro F1*: 0.958 -[INFO] Current time: 1900 ms, average: 2871.44 ms/ques, entity detection: 350.11 ms/ques (linking tool: 235.67 ms/ques), relation detection: 811.56 ms/ques (similarity: 734.22 ms/ques), EDG generation: 353.33 ms/ques +[INFO] Cumulative metrics, sample: 9, P: 0.919, R: 1.000, macro F1: 0.958 +[INFO] Current question consumed time: 1912 ms, average: 3139.22 ms/ques, entity detection: 334.56 ms/ques (linking tool: 229.11 ms/ques), relation detection: 813.78 ms/ques (similarity: 726.78 ms/ques), EDG generation: 673.00 ms/ques [INFO] Entity Linking P: 1.0, R: 1.0, micro F1: 1.0, macro F1: 1.0 [INFO] Relation Linking P: 0.8888888888888888, R: 0.8888888888888888, micro F1: 0.8888888888888888, macro F1: 0.8888888888888888 [INFO] Type Linking P: 0.5555555555555556, R: 0.5555555555555556, micro F1: 0.5555555555555556, macro F1: 0.5555555555555556 @@ -133,8 +133,8 @@ Question 9: Who created the Women in the Garden and also the L'Enfant a la tasse [INFO] Golden Answer: [http://dbpedia.org/resource/Claude_Monet] [INFO] Predicted Answer: [http://dbpedia.org/resource/Claude_Monet] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 10, P: 0.927, R: 1.000, macro F1: 0.943, macro F1*: 0.962 -[INFO] Current time: 2277 ms, average: 2812.00 ms/ques, entity detection: 341.50 ms/ques (linking tool: 234.30 ms/ques), relation detection: 751.20 ms/ques (similarity: 671.90 ms/ques), EDG generation: 409.60 ms/ques +[INFO] Cumulative metrics, sample: 10, P: 0.927, R: 1.000, macro F1: 0.962 +[INFO] Current question consumed time: 2201 ms, average: 3045.40 ms/ques, entity detection: 325.40 ms/ques (linking tool: 226.30 ms/ques), relation detection: 749.60 ms/ques (similarity: 665.90 ms/ques), EDG generation: 698.20 ms/ques [INFO] Entity Linking P: 1.0, R: 1.0, micro F1: 1.0, macro F1: 1.0 [INFO] Relation Linking P: 0.9, R: 0.9, micro F1: 0.9, macro F1: 0.9 [INFO] Type Linking P: 0.6, R: 0.6, micro F1: 0.6, macro F1: 0.6 @@ -147,8 +147,8 @@ Question 10: Which famous horserace has seen both Fergie sutherland and the illu [INFO] Golden Answer: [http://dbpedia.org/resource/Cheltenham_Gold_Cup] [INFO] Predicted Answer: [http://dbpedia.org/resource/Cheltenham_Gold_Cup, http://dbpedia.org/resource/Queen_Mary_Stakes, http://dbpedia.org/resource/Clonmel_Oil_Chase, http://dbpedia.org/resource/Falmouth_Stakes, http://dbpedia.org/resource/Irish_Gold_Cup, http://dbpedia.org/resource/Nas_Na_Riogh_Novice_Chase, Falmouth Stakes@en] [INFO] P: 0.16666666666666666, R: 1.0, F: 0.2857142857142857 -[INFO] Cumulative metrics, sample: 11, P: 0.858, R: 1.000, macro F1: 0.883, macro F1*: 0.924 -[INFO] Current time: 2365 ms, average: 2771.36 ms/ques, entity detection: 348.82 ms/ques (linking tool: 236.91 ms/ques), relation detection: 737.64 ms/ques (similarity: 645.09 ms/ques), EDG generation: 378.73 ms/ques +[INFO] Cumulative metrics, sample: 11, P: 0.858, R: 1.000, macro F1: 0.924 +[INFO] Current question consumed time: 2173 ms, average: 2966.09 ms/ques, entity detection: 335.45 ms/ques (linking tool: 233.36 ms/ques), relation detection: 739.73 ms/ques (similarity: 641.64 ms/ques), EDG generation: 642.00 ms/ques [INFO] Entity Linking P: 1.0, R: 0.9545454545454546, micro F1: 0.9696969696969696, macro F1: 0.9767441860465117 [INFO] Relation Linking P: 0.9090909090909091, R: 0.9090909090909091, micro F1: 0.9090909090909091, macro F1: 0.9090909090909091 [INFO] Type Linking P: 0.6363636363636364, R: 0.6363636363636364, micro F1: 0.6363636363636364, macro F1: 0.6363636363636364 @@ -161,8 +161,8 @@ Question 11: Tell me the name of the Prime Time Entertainment Network's TV show [INFO] Golden Answer: [http://dbpedia.org/resource/Babylon_5] [INFO] Predicted Answer: [http://dbpedia.org/resource/Babylon_5] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 12, P: 0.870, R: 1.000, macro F1: 0.893, macro F1*: 0.930 -[INFO] Current time: 1567 ms, average: 2671.00 ms/ques, entity detection: 354.42 ms/ques (linking tool: 237.25 ms/ques), relation detection: 709.83 ms/ques (similarity: 612.75 ms/ques), EDG generation: 351.50 ms/ques +[INFO] Cumulative metrics, sample: 12, P: 0.870, R: 1.000, macro F1: 0.930 +[INFO] Current question consumed time: 1480 ms, average: 2842.25 ms/ques, entity detection: 338.58 ms/ques (linking tool: 232.58 ms/ques), relation detection: 711.50 ms/ques (similarity: 609.33 ms/ques), EDG generation: 592.42 ms/ques [INFO] Entity Linking P: 1.0, R: 0.9583333333333334, micro F1: 0.9722222222222222, macro F1: 0.9787234042553191 [INFO] Relation Linking P: 0.9166666666666666, R: 0.9166666666666666, micro F1: 0.9166666666666666, macro F1: 0.9166666666666666 [INFO] Type Linking P: 0.5833333333333334, R: 0.5833333333333334, micro F1: 0.5833333333333334, macro F1: 0.5833333333333334 @@ -175,8 +175,8 @@ Question 12: How many other battles have the military person fought whose one of [INFO] Golden Answer: [2459] [INFO] Predicted Answer: [60, 1] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 13, P: 0.803, R: 0.923, macro F1: 0.824, macro F1*: 0.859 -[INFO] Current time: 18039 ms, average: 3853.15 ms/ques, entity detection: 818.92 ms/ques (linking tool: 707.08 ms/ques), relation detection: 971.92 ms/ques (similarity: 732.62 ms/ques), EDG generation: 330.85 ms/ques +[INFO] Cumulative metrics, sample: 13, P: 0.803, R: 0.923, macro F1: 0.859 +[INFO] Current question consumed time: 18637 ms, average: 4057.23 ms/ques, entity detection: 819.31 ms/ques (linking tool: 717.85 ms/ques), relation detection: 987.69 ms/ques (similarity: 736.31 ms/ques), EDG generation: 550.46 ms/ques [INFO] Entity Linking P: 1.0, R: 0.9615384615384616, micro F1: 0.9743589743589743, macro F1: 0.9803921568627451 [INFO] Relation Linking P: 0.8461538461538461, R: 0.8461538461538461, micro F1: 0.8461538461538461, macro F1: 0.8461538461538461 [INFO] Type Linking P: 0.5384615384615384, R: 0.5384615384615384, micro F1: 0.5384615384615384, macro F1: 0.5384615384615384 @@ -189,8 +189,8 @@ Question 13: Whose network's parent organisation is Comcast? [INFO] Golden Answer: [http://dbpedia.org/resource/Cheers_(season_1), http://dbpedia.org/resource/Cheers_(season_2), http://dbpedia.org/resource/Cheers_(season_3), http://dbpedia.org/resource/Cheers_(season_4), http://dbpedia.org/resource/Cheers_(season_5), http://dbpedia.org/resource/Cheers_(season_6), http://dbpedia.org/resource/Cheers_(season_7), http://dbpedia.org/resource/Frasier_(season_9), http://dbpedia.org/resource/Friends, http://dbpedia.org/resource/It's_Punky_Brewster, http://dbpedia.org/resource/Scrubs_(TV_series), http://dbpedia.org/resource/St._Elsewhere, http://dbpedia.org/resource/The_Cosby_Show, http://dbpedia.org/resource/Alvin_and_the_Chipmunks_(1983_TV_series), http://dbpedia.org/resource/Dolly_Parton's_Coat_of_Many_Colors, http://dbpedia.org/resource/E-Ring, http://dbpedia.org/resource/Father_of_the_Pride, http://dbpedia.org/resource/High_School_U.S.A., http://dbpedia.org/resource/Information_Please, http://dbpedia.org/resource/My_Two_Dads, http://dbpedia.org/resource/Queen_for_a_Day, http://dbpedia.org/resource/Remington_Steele, http://dbpedia.org/resource/Rowan_&_Martin's_Laugh-In, http://dbpedia.org/resource/Saturday_Night_Live_(season_1), http://dbpedia.org/resource/The_Man_from_U.N.C.L.E., http://dbpedia.org/resource/The_Single_Guy, http://dbpedia.org/resource/The_Steve_Allen_Show, http://dbpedia.org/resource/The_Virginian_(TV_series), http://dbpedia.org/resource/Victoria_Beckham:_Coming_to_America, http://dbpedia.org/resource/Walt_Disney_anthology_television_series, http://dbpedia.org/resource/Banacek, http://dbpedia.org/resource/Buck_Rogers_in_the_25th_Century_(TV_series), http://dbpedia.org/resource/Guilty_or_Innocent:_The_Sam_Sheppard_Murder_Case, http://dbpedia.org/resource/Howdy_Doody, http://dbpedia.org/resource/I_Spy_(1965_TV_series), http://dbpedia.org/resource/Magnavox_Presents_Frank_Sinatra, http://dbpedia.org/resource/Suspicion_(TV_series), http://dbpedia.org/resource/The_Funny_Side, http://dbpedia.org/resource/The_Gary_Coleman_Show, http://dbpedia.org/resource/The_Ghost_&_Mrs._Muir_(TV_series), http://dbpedia.org/resource/The_High_Chaparral, http://dbpedia.org/resource/The_Kaiser_Aluminum_Hour, http://dbpedia.org/resource/The_New_Show, http://dbpedia.org/resource/The_Questor_Tapes, http://dbpedia.org/resource/To_Tell_the_Truth, http://dbpedia.org/resource/Exiled:_A_Law_&_Order_Movie, http://dbpedia.org/resource/Fame_(1982_TV_series), http://dbpedia.org/resource/Late_Night_with_Seth_Meyers, http://dbpedia.org/resource/Law_&_Order:_Special_Victims_Unit_(season_7), http://dbpedia.org/resource/On_the_Edge_of_Innocence, http://dbpedia.org/resource/Spider-Man_and_His_Amazing_Friends, http://dbpedia.org/resource/The_Jay_Leno_Show, http://dbpedia.org/resource/L.A._Takedown, http://dbpedia.org/resource/Rudolph_the_Red-Nosed_Reindeer_(TV_special), http://dbpedia.org/resource/The_Rocky_and_Bullwinkle_Show, http://dbpedia.org/resource/Quincy,_M.E., http://dbpedia.org/resource/The_First_Olympics:_Athens_1896, http://dbpedia.org/resource/The_Golden_Girls_(season_4), http://dbpedia.org/resource/A_Man_Called_Sloane, http://dbpedia.org/resource/Please_Don't_Eat_the_Daisies_(TV_series), http://dbpedia.org/resource/The_Parent_Trap_III, http://dbpedia.org/resource/The_Parent_Trap_IV:_Hawaiian_Honeymoon, http://dbpedia.org/resource/AFL_on_NBC, http://dbpedia.org/resource/The_Jensen_Project, http://dbpedia.org/resource/Three_Sisters_(TV_series), http://dbpedia.org/resource/50_Grand_Slam, http://dbpedia.org/resource/Against_the_Grain_(TV_series), http://dbpedia.org/resource/Eerie,_Indiana, http://dbpedia.org/resource/The_Dinah_Shore_Chevy_Show, http://dbpedia.org/resource/Halloween_with_the_New_Addams_Family, http://dbpedia.org/resource/It's_Worth_What%3F, http://dbpedia.org/resource/Lipshitz_Saves_the_World, http://dbpedia.org/resource/Little_Mo_(1978_film), http://dbpedia.org/resource/Saturday_Night_Live_(season_40), http://dbpedia.org/resource/The_Addams_Family_(1973_animated_series), http://dbpedia.org/resource/The_Bob_Cummings_Show, http://dbpedia.org/resource/The_Bold_Ones:_The_Protectors, http://dbpedia.org/resource/The_Cosby_Mysteries, http://dbpedia.org/resource/Whoopi, http://dbpedia.org/resource/Your_First_Impression, http://dbpedia.org/resource/Fire_on_the_Mountain_(1981_film), http://dbpedia.org/resource/Five_Fingers_(TV_series), http://dbpedia.org/resource/Liza_with_a_Z, http://dbpedia.org/resource/Rainbow_(1978_film), http://dbpedia.org/resource/Return_to_Mayberry, http://dbpedia.org/resource/The_Bold_Ones, http://dbpedia.org/resource/1_vs._100_(U.S._game_show), http://dbpedia.org/resource/Average_Joe_(TV_series), http://dbpedia.org/resource/Chain_Reaction_(game_show), http://dbpedia.org/resource/Jackpot_(game_show), http://dbpedia.org/resource/Time_Machine_(game_show), http://dbpedia.org/resource/Treasure_Hunt_(U.S._game_show), http://dbpedia.org/resource/Win,_Lose_or_Draw, http://dbpedia.org/resource/Banyon, http://dbpedia.org/resource/Café_Americain, http://dbpedia.org/resource/Camel_News_Caravan, http://dbpedia.org/resource/Celebrity_Family_Feud, http://dbpedia.org/resource/City_of_Angels_(1976_TV_series), http://dbpedia.org/resource/Dawn:_Portrait_of_a_Teenage_Runaway, http://dbpedia.org/resource/Fraggle_Rock:_The_Animated_Series, http://dbpedia.org/resource/I_Dream_of_Jeannie..._Fifteen_Years_Later, http://dbpedia.org/resource/Return_to_Peyton_Place_(TV_series), http://dbpedia.org/resource/Runaround_(game_show), http://dbpedia.org/resource/Surface_(TV_series), http://dbpedia.org/resource/Bonino_(TV_series), http://dbpedia.org/resource/Bret_Maverick, http://dbpedia.org/resource/Cursed_(TV_series), http://dbpedia.org/resource/LateLine, http://dbpedia.org/resource/Baggy_Pants_and_the_Nitwits, http://dbpedia.org/resource/Quantum_Leap_(season_2), http://dbpedia.org/resource/Quarterlife, http://dbpedia.org/resource/Something_for_a_Lonely_Man, http://dbpedia.org/resource/Then_Came_Bronson, http://dbpedia.org/resource/Wild_Kingdom, http://dbpedia.org/resource/Free_Agents_(U.S._TV_series), http://dbpedia.org/resource/Hitler's_SS:_Portrait_in_Evil, http://dbpedia.org/resource/Murder_in_Texas, http://dbpedia.org/resource/Revenge_of_the_Stepford_Wives, http://dbpedia.org/resource/The_Bob_Newhart_Show_(1961_TV_series), http://dbpedia.org/resource/Redigo_(TV_series), http://dbpedia.org/resource/Rockin'_Through_the_Decades, http://dbpedia.org/resource/Saturday_Night_Live_(season_39), http://dbpedia.org/resource/Ding_Dong_School, http://dbpedia.org/resource/M.Y.O.B._(TV_series), http://dbpedia.org/resource/Boone_(TV_series), http://dbpedia.org/resource/Kingpin_(TV_series), http://dbpedia.org/resource/Emergency_+4, http://dbpedia.org/resource/The_Snoop_Sisters, http://dbpedia.org/resource/30_Rock_(season_7), http://dbpedia.org/resource/33⅓_Revolutions_per_Monkee, http://dbpedia.org/resource/Battlestars_(game_show), http://dbpedia.org/resource/Joe_Forrester_(TV_series), http://dbpedia.org/resource/Law_&_Order_(season_9), http://dbpedia.org/resource/Little_Richard_(film), http://dbpedia.org/resource/Love,_Sidney, http://dbpedia.org/resource/Northwest_Passage_(TV_series), http://dbpedia.org/resource/The_More_You_Know, http://dbpedia.org/resource/Movin'_with_Nancy, http://dbpedia.org/resource/Scrubs_(season_4), http://dbpedia.org/resource/The_Spike_Jones_Show_(series), http://dbpedia.org/resource/Danger_Island_(film), http://dbpedia.org/resource/Dragnet_(1951_TV_series), http://dbpedia.org/resource/Fashion_Star_(season_1), http://dbpedia.org/resource/Living_Proof:_The_Hank_Williams,_Jr._Story, http://dbpedia.org/resource/Thank_God_You're_Here_(U.S._TV_series), http://dbpedia.org/resource/The_Real_Wedding_Crashers, http://dbpedia.org/resource/Fatal_Vision_(miniseries), http://dbpedia.org/resource/Call_My_Bluff_(U.S._game_show), http://dbpedia.org/resource/Trinity_(U.S._TV_series), http://dbpedia.org/resource/The_Biggest_Loser:_Families, http://dbpedia.org/resource/Saturday_Night_Live_(season_34), http://dbpedia.org/resource/The_Restaurant_(U.S._TV_series), http://dbpedia.org/resource/World's_Most_Amazing_Videos, http://dbpedia.org/resource/Your_Mother_Wears_Combat_Boots, http://dbpedia.org/resource/Parks_and_Recreation_(season_2), http://dbpedia.org/resource/Saturday_Night_Live_(season_10), http://dbpedia.org/resource/Saturday_Night_Live_(season_32), http://dbpedia.org/resource/Reach_for_the_Stars_(game_show), http://dbpedia.org/resource/The_Rockford_Files_(season_2), http://dbpedia.org/resource/Jigsaw_John_(TV_series), http://dbpedia.org/resource/Someone_Like_Me_(TV_series), http://dbpedia.org/resource/Winnetka_Road, http://dbpedia.org/resource/Combat_Academy, http://dbpedia.org/resource/Kate's_Secret, http://dbpedia.org/resource/Class_Cruise, http://dbpedia.org/resource/Science_of_Love, http://dbpedia.org/resource/Silent_Witness_(1985_film), http://dbpedia.org/resource/Moment_of_Truth:_Why_My_Daughter%3F, http://dbpedia.org/resource/Strong_(TV_series), http://dbpedia.org/resource/My_Name_Is_Earl_(season_4), http://dbpedia.org/resource/Chicago_Fire_(season_1), http://dbpedia.org/resource/Chicago_Fire_(season_2), http://dbpedia.org/resource/Chicago_Fire_(season_3), http://dbpedia.org/resource/Chicago_Fire_(season_4), http://dbpedia.org/resource/Chicago_P.D._(season_1), http://dbpedia.org/resource/Chicago_P.D._(season_2), http://dbpedia.org/resource/Chicago_P.D._(season_3), http://dbpedia.org/resource/Make_Your_Own_Kind_of_Music_(TV_series), http://dbpedia.org/resource/Knight_Rider_(season_4), http://dbpedia.org/resource/Fashion_Star_(season_2), http://dbpedia.org/resource/The_Six_of_Us, http://dbpedia.org/resource/The_Ted_Steele_Show, http://dbpedia.org/resource/Friday_Night_Lights_(season_2), http://dbpedia.org/resource/Take_a_Chance_(U.S._game_show), http://dbpedia.org/resource/Parenthood_(season_1), http://dbpedia.org/resource/Parenthood_(season_3), http://dbpedia.org/resource/Parenthood_(season_4), http://dbpedia.org/resource/Parenthood_(season_5), http://dbpedia.org/resource/Parenthood_(season_6), http://dbpedia.org/resource/Where_the_Girls_Are_(TV_program), http://dbpedia.org/resource/The_A-Team_(season_2), http://dbpedia.org/resource/The_A-Team_(season_3), http://dbpedia.org/resource/The_A-Team_(season_5), http://dbpedia.org/resource/Adam-12_(season_2), http://dbpedia.org/resource/Dragnet_(1967_series)_(season_1), http://dbpedia.org/resource/Dragnet_(1967_series)_(season_3), http://dbpedia.org/resource/Dragnet_(1967_series)_(season_4), http://dbpedia.org/resource/Ironside_(season_1), http://dbpedia.org/resource/Ironside_(season_2), http://dbpedia.org/resource/Ironside_(season_4), http://dbpedia.org/resource/Ironside_(season_5), http://dbpedia.org/resource/Ironside_(season_6), http://dbpedia.org/resource/Ironside_(season_8), http://dbpedia.org/resource/Quincy,_M.E._(season_1), http://dbpedia.org/resource/Quincy,_M.E._(season_2), http://dbpedia.org/resource/Quincy,_M.E._(season_3), http://dbpedia.org/resource/Quincy,_M.E._(season_4), http://dbpedia.org/resource/Quincy,_M.E._(season_5), http://dbpedia.org/resource/Quincy,_M.E._(season_6), http://dbpedia.org/resource/Quincy,_M.E._(season_7), http://dbpedia.org/resource/Quincy,_M.E._(season_8), http://dbpedia.org/resource/3rd_Rock_from_the_Sun, http://dbpedia.org/resource/Alfred_Hitchcock_Presents, http://dbpedia.org/resource/Alfred_Hitchcock_Presents_(1985_TV_series), http://dbpedia.org/resource/Jeopardy!, http://dbpedia.org/resource/Star_Trek:_The_Original_Series, http://dbpedia.org/resource/30_Rock_(season_3), http://dbpedia.org/resource/Crime_Story_(TV_series), http://dbpedia.org/resource/Frasier_(season_10), http://dbpedia.org/resource/Knight_Rider_(1982_TV_series), http://dbpedia.org/resource/Studio_60_on_the_Sunset_Strip, http://dbpedia.org/resource/The_Tonight_Show_Starring_Jimmy_Fallon, http://dbpedia.org/resource/Alice_in_Wonderland_(1999_film), http://dbpedia.org/resource/Caesar's_Hour, http://dbpedia.org/resource/College_Basketball_on_NBC, http://dbpedia.org/resource/Eischied, http://dbpedia.org/resource/Major_League_Baseball:_An_Inside_Look, http://dbpedia.org/resource/Thoroughbred_Racing_on_NBC, http://dbpedia.org/resource/WWE_Tribute_to_the_Troops, http://dbpedia.org/resource/American_Gladiators_(2008_TV_series), http://dbpedia.org/resource/Mann_&_Machine, http://dbpedia.org/resource/National_Heads-Up_Poker_Championship, http://dbpedia.org/resource/Ringo_(1978_film), http://dbpedia.org/resource/Space_Sentinels, http://dbpedia.org/resource/The_Golden_Girls_(season_1), http://dbpedia.org/resource/The_Rogues_(TV_series), http://dbpedia.org/resource/The_Yellow_Rose, http://dbpedia.org/resource/Suddenly_Susan, http://dbpedia.org/resource/A_Flintstone_Christmas, http://dbpedia.org/resource/Fred_and_Barney_Meet_The_Thing, http://dbpedia.org/resource/Fred_and_Barney_Meet_the_Shmoo, http://dbpedia.org/resource/Galaxy_Goof-Ups, http://dbpedia.org/resource/Serving_in_Silence:_The_Margarethe_Cammermeyer_Story, http://dbpedia.org/resource/The_Flintstone_Funnies, http://dbpedia.org/resource/The_New_Fred_and_Barney_Show, http://dbpedia.org/resource/You,_Me_and_the_Apocalypse, http://dbpedia.org/resource/Nearly_Departed, http://dbpedia.org/resource/Parks_and_Recreation_(season_7), http://dbpedia.org/resource/Sealab_2020, http://dbpedia.org/resource/The_Flintstones'_New_Neighbors, http://dbpedia.org/resource/The_Flintstones:_Fred's_Final_Fling, http://dbpedia.org/resource/The_Flintstones:_Jogging_Fever, http://dbpedia.org/resource/The_Flintstones:_Little_Big_League, http://dbpedia.org/resource/The_Flintstones:_Wind-Up_Wilma, http://dbpedia.org/resource/The_Flintstones_Meet_Rockula_and_Frankenstone, http://dbpedia.org/resource/Identity_(game_show), http://dbpedia.org/resource/On_Wings_of_Eagles_(miniseries), http://dbpedia.org/resource/Outlaws_(1960_TV_series), http://dbpedia.org/resource/Who_Said_That%3F, http://dbpedia.org/resource/The_Biggest_Loser:_No_Excuses, http://dbpedia.org/resource/The_Burning_Bed, http://dbpedia.org/resource/Awake_(TV_series), http://dbpedia.org/resource/The_Biggest_Loser_(season_3), http://dbpedia.org/resource/The_Book_of_Daniel_(TV_series), http://dbpedia.org/resource/Run,_Joe,_Run, http://dbpedia.org/resource/Sigmund_and_the_Sea_Monsters, http://dbpedia.org/resource/The_Bold_Ones:_The_Lawyers, http://dbpedia.org/resource/Alexander:_The_Other_Side_of_Dawn, http://dbpedia.org/resource/Law_&_Order:_Special_Victims_Unit_(season_10), http://dbpedia.org/resource/The_Gabby_Hayes_Show, http://dbpedia.org/resource/Deadline_(American_TV_series), http://dbpedia.org/resource/In_Love_and_War_(1987_film), http://dbpedia.org/resource/Saturday_Night_Live_(season_2), http://dbpedia.org/resource/She_Spies, http://dbpedia.org/resource/Adam_(1983_film), http://dbpedia.org/resource/The_Ruff_and_Reddy_Show, http://dbpedia.org/resource/McKeever_and_the_Colonel, http://dbpedia.org/resource/Sierra_(TV_series), http://dbpedia.org/resource/Columbo_(season_4), http://dbpedia.org/resource/Columbo_(season_5), http://dbpedia.org/resource/Jewels_(miniseries), http://dbpedia.org/resource/Kristin_(TV_series), http://dbpedia.org/resource/Quantum_Leap_(season_5), http://dbpedia.org/resource/The_Fresh_Prince_of_Bel-Air_(season_3), http://dbpedia.org/resource/The_Secret_Life_of_Kathy_McCormick, http://dbpedia.org/resource/Casper's_First_Christmas, http://dbpedia.org/resource/Doug_Henning's_World_of_Magic, http://dbpedia.org/resource/Most_Outrageous_Moments, http://dbpedia.org/resource/Betty_White's_Off_Their_Rockers, http://dbpedia.org/resource/Half_Nelson_(TV_series), http://dbpedia.org/resource/Jack_Frost_(TV_special), http://dbpedia.org/resource/Lookwell, http://dbpedia.org/resource/Saturday_Night_Live_(season_18), http://dbpedia.org/resource/Homeland_Security_(film), http://dbpedia.org/resource/Skunked_TV, http://dbpedia.org/resource/The_Man_and_the_Challenge, http://dbpedia.org/resource/Sweet_Justice, http://dbpedia.org/resource/A_Howling_in_the_Woods, http://dbpedia.org/resource/I_Still_Dream_of_Jeannie, http://dbpedia.org/resource/McDuff,_the_Talking_Dog, http://dbpedia.org/resource/In_the_Line_of_Duty:_The_F.B.I._Murders, http://dbpedia.org/resource/Legmen, http://dbpedia.org/resource/Revelations_(miniseries), http://dbpedia.org/resource/The_Innocent_(1994_film), http://dbpedia.org/resource/A_Chipmunk_Reunion, http://dbpedia.org/resource/Opposites_Attract_(1990_film), http://dbpedia.org/resource/Those_She_Left_Behind, http://dbpedia.org/resource/Popeye_Doyle_(film), http://dbpedia.org/resource/A_Child's_Cry_for_Help, http://dbpedia.org/resource/Frasier_(season_4), http://dbpedia.org/resource/Miracle_Child_(1993_film), http://dbpedia.org/resource/Snowbeast, http://dbpedia.org/resource/Grimm_(season_1), http://dbpedia.org/resource/Dragnet_(1967_series)_(season_2), http://dbpedia.org/resource/Come_to_Papa_(TV_series), http://dbpedia.org/resource/Lazer_Tag_Academy, http://dbpedia.org/resource/My_Son_Jeep, http://dbpedia.org/resource/Community_(season_2), http://dbpedia.org/resource/The_Jane_Whitney_Show, http://dbpedia.org/resource/The_Kids_From_C.A.P.E.R., http://dbpedia.org/resource/The_World's_Most_Dangerous_Magic, http://dbpedia.org/resource/My_Name_Is_Earl_(season_2), http://dbpedia.org/resource/The_Office_(U.S._season_2), http://dbpedia.org/resource/Amnesia_(game_show), http://dbpedia.org/resource/3rd_Rock_from_the_Sun_(season_2), http://dbpedia.org/resource/3rd_Rock_from_the_Sun_(season_6), http://dbpedia.org/resource/List_of_The_Tonight_Show_with_Jay_Leno_episodes_(2000–09), http://dbpedia.org/resource/The_Jack_Benny_Program_(season_15), http://dbpedia.org/resource/List_of_Late_Night_with_Jimmy_Fallon_episodes_(2010), http://dbpedia.org/resource/Harry's_Law_(season_1), http://dbpedia.org/resource/The_Fresh_Prince_of_Bel-Air_(season_5), http://dbpedia.org/resource/List_of_The_Tonight_Show_Starring_Johnny_Carson_episodes_(1974), http://dbpedia.org/resource/List_of_The_Tonight_Show_Starring_Johnny_Carson_episodes_(1975), http://dbpedia.org/resource/List_of_The_Tonight_Show_Starring_Johnny_Carson_episodes_(1976), http://dbpedia.org/resource/List_of_The_Tonight_Show_Starring_Johnny_Carson_episodes_(1978), http://dbpedia.org/resource/List_of_The_Tonight_Show_Starring_Johnny_Carson_episodes_(1979), http://dbpedia.org/resource/List_of_The_Tonight_Show_Starring_Johnny_Carson_episodes_(1980), http://dbpedia.org/resource/List_of_The_Tonight_Show_Starring_Johnny_Carson_episodes_(1981), http://dbpedia.org/resource/List_of_The_Tonight_Show_Starring_Johnny_Carson_episodes_(1982), http://dbpedia.org/resource/List_of_The_Tonight_Show_Starring_Johnny_Carson_episodes_(1983), http://dbpedia.org/resource/List_of_The_Tonight_Show_Starring_Johnny_Carson_episodes_(1984), http://dbpedia.org/resource/List_of_The_Tonight_Show_Starring_Johnny_Carson_episodes_(1985), http://dbpedia.org/resource/List_of_The_Tonight_Show_Starring_Johnny_Carson_episodes_(1986), http://dbpedia.org/resource/List_of_The_Tonight_Show_Starring_Johnny_Carson_episodes_(1987), http://dbpedia.org/resource/List_of_The_Tonight_Show_Starring_Johnny_Carson_episodes_(1988), http://dbpedia.org/resource/List_of_The_Tonight_Show_Starring_Johnny_Carson_episodes_(1989), http://dbpedia.org/resource/List_of_The_Tonight_Show_Starring_Johnny_Carson_episodes_(1990), http://dbpedia.org/resource/List_of_The_Tonight_Show_Starring_Johnny_Carson_episodes_(1991), http://dbpedia.org/resource/List_of_The_Tonight_Show_Starring_Johnny_Carson_episodes_(1992), http://dbpedia.org/resource/List_of_Late_Night_with_Jimmy_Fallon_episodes_(2009), http://dbpedia.org/resource/List_of_Late_Night_with_Jimmy_Fallon_episodes_(2011), http://dbpedia.org/resource/List_of_The_Tonight_Show_Starring_Jimmy_Fallon_episodes_(2014), http://dbpedia.org/resource/List_of_The_Tonight_Show_Starring_Jimmy_Fallon_episodes_(2016), http://dbpedia.org/resource/Dateline_NBC, http://dbpedia.org/resource/I_Dream_of_Jeannie, http://dbpedia.org/resource/Kings_(U.S._TV_series), http://dbpedia.org/resource/Medium_(TV_series), http://dbpedia.org/resource/Miami_Vice, http://dbpedia.org/resource/SeaQuest_DSV, http://dbpedia.org/resource/The_Celebrity_Apprentice, http://dbpedia.org/resource/The_Incredible_Hulk_(1978_TV_series), http://dbpedia.org/resource/The_Tonight_Show, http://dbpedia.org/resource/The_Tonight_Show_with_Jay_Leno, http://dbpedia.org/resource/Wheel_of_Fortune_(U.S._game_show), http://dbpedia.org/resource/Bionic_Woman_(2007_TV_series), http://dbpedia.org/resource/Homicide:_Life_on_the_Street_(season_6), http://dbpedia.org/resource/L.A._Law, http://dbpedia.org/resource/Law_&_Order:_Special_Victims_Unit, http://dbpedia.org/resource/Law_&_Order:_Special_Victims_Unit_(season_8), http://dbpedia.org/resource/Love,_Lies_and_Murder, http://dbpedia.org/resource/Misfits_of_Science, http://dbpedia.org/resource/Taxi_(TV_series), http://dbpedia.org/resource/The_Bionic_Woman, http://dbpedia.org/resource/The_Year_Without_a_Santa_Claus_(2006_film), http://dbpedia.org/resource/Earth_2_(TV_series), http://dbpedia.org/resource/Riverboat_(TV_series), http://dbpedia.org/resource/Underdog_(TV_series), http://dbpedia.org/resource/The_Apprentice_(TV_series), http://dbpedia.org/resource/Windfall_(TV_series), http://dbpedia.org/resource/Ellery_Queen_(TV_series), http://dbpedia.org/resource/Seinfeld_(season_1), http://dbpedia.org/resource/Seinfeld_(season_2), http://dbpedia.org/resource/Seinfeld_(season_3), http://dbpedia.org/resource/Seinfeld_(season_4), http://dbpedia.org/resource/Seinfeld_(season_5), http://dbpedia.org/resource/Seinfeld_(season_6), http://dbpedia.org/resource/Seinfeld_(season_7), http://dbpedia.org/resource/Seinfeld_(season_8), http://dbpedia.org/resource/Seinfeld_(season_9), http://dbpedia.org/resource/The_John_Larroquette_Show, http://dbpedia.org/resource/The_Michael_Richards_Show, http://dbpedia.org/resource/Watching_Ellie, http://dbpedia.org/resource/Backstairs_at_the_White_House, http://dbpedia.org/resource/Love_in_the_Wild, http://dbpedia.org/resource/The_Contender_(TV_series), http://dbpedia.org/resource/Choose_Up_Sides, http://dbpedia.org/resource/Dough_Re_Mi, http://dbpedia.org/resource/Make_the_Connection, http://dbpedia.org/resource/Sale_of_the_Century_(U.S._game_show), http://dbpedia.org/resource/The_Two_Mrs._Grenvilles, http://dbpedia.org/resource/3rd_Rock_from_the_Sun_(season_1), http://dbpedia.org/resource/3rd_Rock_from_the_Sun_(season_5), http://dbpedia.org/resource/America's_Most_Talented_Kid, http://dbpedia.org/resource/From_Here_to_Eternity_(miniseries), http://dbpedia.org/resource/I'll_Fly_Away_(TV_series), http://dbpedia.org/resource/I_Married_Joan, http://dbpedia.org/resource/Something_So_Right_(TV_series), http://dbpedia.org/resource/Bare_Essence, http://dbpedia.org/resource/C.P.O._Sharkey, http://dbpedia.org/resource/Freaks_and_Geeks, http://dbpedia.org/resource/Harper_Valley_PTA_(TV_series), http://dbpedia.org/resource/Heist_(TV_series), http://dbpedia.org/resource/Kath_&_Kim_(U.S._TV_series), http://dbpedia.org/resource/Lovers_and_Friends, http://dbpedia.org/resource/My_Hero_(U.S._TV_series), http://dbpedia.org/resource/Needles_and_Pins_(TV_series), http://dbpedia.org/resource/Rescue_from_Gilligan's_Island, http://dbpedia.org/resource/Saturday_Night_Live_(season_21), http://dbpedia.org/resource/Saturday_Night_Live_(season_36), http://dbpedia.org/resource/Saturday_Night_Live_(season_6), http://dbpedia.org/resource/Sirota's_Court, http://dbpedia.org/resource/Teachers_(2006_U.S._TV_series), http://dbpedia.org/resource/The_Bill_Cosby_Show, http://dbpedia.org/resource/The_Castaways_on_Gilligan's_Island, http://dbpedia.org/resource/The_Duck_Factory, http://dbpedia.org/resource/The_Others_(TV_series), http://dbpedia.org/resource/The_Pursuit_of_Happiness_(TV_series), http://dbpedia.org/resource/Twenty_Good_Years, http://dbpedia.org/resource/My_Name_Is_Earl_(season_1), http://dbpedia.org/resource/Raines, http://dbpedia.org/resource/Journeyman_(TV_series), http://dbpedia.org/resource/List_of_The_Tonight_Show_Starring_Johnny_Carson_episodes_(1977), http://dbpedia.org/resource/Born_Innocent_(film), http://dbpedia.org/resource/Polly_(1989_film), http://dbpedia.org/resource/How_to_Break_Up_a_Happy_Divorce, http://dbpedia.org/resource/Leave_It_to_the_Girls, http://dbpedia.org/resource/The_Greatest_Gift_(film), http://dbpedia.org/resource/America's_Next_Great_Restaurant, http://dbpedia.org/resource/A_Cry_for_Help:_The_Tracey_Thurman_Story, http://dbpedia.org/resource/Columbo_(season_1), http://dbpedia.org/resource/Columbo_(season_6), http://dbpedia.org/resource/Columbo_(season_7), http://dbpedia.org/resource/The_Hero_(1966_TV_series), http://dbpedia.org/resource/100_Questions, http://dbpedia.org/resource/Face_the_Ace, http://dbpedia.org/resource/Hank_(1965_TV_series), http://dbpedia.org/resource/Hawaii_(TV_series), http://dbpedia.org/resource/Inconceivable, http://dbpedia.org/resource/Karen_(1964_TV_series), http://dbpedia.org/resource/Meet_My_Folks, http://dbpedia.org/resource/Mr._Rhodes, http://dbpedia.org/resource/Pride_&_Joy_(TV_series), http://dbpedia.org/resource/The_Great_American_Road_Trip, http://dbpedia.org/resource/The_Night_They_Took_Miss_Beautiful, http://dbpedia.org/resource/Punch_and_Jody, http://dbpedia.org/resource/Somerset_Maugham_TV_Theatre, http://dbpedia.org/resource/Undercovers_(TV_series), http://dbpedia.org/resource/Growing_Up_Brady_(film), http://dbpedia.org/resource/How_to_Eat_Like_a_Child, http://dbpedia.org/resource/In_the_Beginning_(miniseries), http://dbpedia.org/resource/Homicide:_Life_on_the_Street_(season_1), http://dbpedia.org/resource/Homicide:_Life_on_the_Street_(season_2), http://dbpedia.org/resource/Homicide:_Life_on_the_Street_(season_3), http://dbpedia.org/resource/Homicide:_Life_on_the_Street_(season_4), http://dbpedia.org/resource/Mindreaders, http://dbpedia.org/resource/Crash:_The_Mystery_of_Flight_1501, http://dbpedia.org/resource/My_Dad_Is_Better_Than_Your_Dad, http://dbpedia.org/resource/Doomsday_Virus, http://dbpedia.org/resource/Law_&_Order:_Criminal_Intent_(season_5), http://dbpedia.org/resource/Sportsworld_(U.S._TV_series), http://dbpedia.org/resource/The_Home_Court, http://dbpedia.org/resource/Parks_and_Recreation_(season_4), http://dbpedia.org/resource/Parks_and_Recreation_(season_6), http://dbpedia.org/resource/In_the_Loop_with_iVillage, http://dbpedia.org/resource/Law_&_Order:_Criminal_Intent_(season_1), http://dbpedia.org/resource/Law_&_Order:_Criminal_Intent_(season_2), http://dbpedia.org/resource/Law_&_Order:_Criminal_Intent_(season_4), http://dbpedia.org/resource/Law_&_Order:_Criminal_Intent_(season_6), http://dbpedia.org/resource/Saturday_Night_Live_(season_14), http://dbpedia.org/resource/Saturday_Night_Live_(season_15), http://dbpedia.org/resource/Saturday_Night_Live_(season_17), http://dbpedia.org/resource/Saturday_Night_Live_(season_22), http://dbpedia.org/resource/Saturday_Night_Live_(season_25), http://dbpedia.org/resource/Saturday_Night_Live_(season_33), http://dbpedia.org/resource/Saturday_Night_Live_(season_41), http://dbpedia.org/resource/Saturday_Night_Live_(season_8), http://dbpedia.org/resource/Community_(season_1), http://dbpedia.org/resource/Eyes_of_Terror, http://dbpedia.org/resource/Stonestreet:_Who_Killed_the_Centerfold_Model%3F, http://dbpedia.org/resource/The_West_Wing_(season_3), http://dbpedia.org/resource/Money_Talks_News, http://dbpedia.org/resource/Lost_(game_show), http://dbpedia.org/resource/The_Biggest_Loser:_Battle_of_the_Ages, http://dbpedia.org/resource/The_Biggest_Loser:_Challenge_America, http://dbpedia.org/resource/The_Biggest_Loser:_Pay_It_Forward, http://dbpedia.org/resource/The_Biggest_Loser:_Second_Chances, http://dbpedia.org/resource/The_Biggest_Loser_(season_1), http://dbpedia.org/resource/The_Biggest_Loser_(season_2), http://dbpedia.org/resource/The_Biggest_Loser_(season_4), http://dbpedia.org/resource/Harry's_Law_(season_2), http://dbpedia.org/resource/Grimm_(season_3), http://dbpedia.org/resource/Community_(season_4), http://dbpedia.org/resource/Boss_Lady, http://dbpedia.org/resource/Frasier_(season_7), http://dbpedia.org/resource/Casper's_Halloween_Special, http://dbpedia.org/resource/Son-Rise:_A_Miracle_of_Love, http://dbpedia.org/resource/The_Golden_Girls_(season_7), http://dbpedia.org/resource/Quantum_Leap_(season_1), http://dbpedia.org/resource/Quantum_Leap_(season_4), http://dbpedia.org/resource/Three_Wishes_(U.S._TV_series), http://dbpedia.org/resource/List_of_Late_Night_with_Jimmy_Fallon_episodes_(2013–14), http://dbpedia.org/resource/List_of_The_Tonight_Show_Starring_Jimmy_Fallon_episodes_(2015), http://dbpedia.org/resource/List_of_Late_Night_with_Seth_Meyers_episodes_(2014), http://dbpedia.org/resource/List_of_Late_Night_with_Seth_Meyers_episodes_(2016), http://dbpedia.org/resource/30_Rock, http://dbpedia.org/resource/A.D._The_Bible_Continues, http://dbpedia.org/resource/ALF:_The_Animated_Series, http://dbpedia.org/resource/Another_World_(TV_series), http://dbpedia.org/resource/Candid_Camera, http://dbpedia.org/resource/Days_of_Our_Lives, http://dbpedia.org/resource/Hallmark_Hall_of_Fame, http://dbpedia.org/resource/Heidi_(1968_film), http://dbpedia.org/resource/Law_&_Order, http://dbpedia.org/resource/Law_&_Order_(season_5), http://dbpedia.org/resource/Meet_the_Press, http://dbpedia.org/resource/Seinfeld, http://dbpedia.org/resource/Treasure_Hunters_(TV_series), http://dbpedia.org/resource/Wild_Texas_Wind, http://dbpedia.org/resource/Amazing_Stories_(TV_series), http://dbpedia.org/resource/Dr._Kildare_(TV_series), http://dbpedia.org/resource/J.J._Starbuck, http://dbpedia.org/resource/Land_of_the_Lost_(1974_TV_series), http://dbpedia.org/resource/Late_Night_with_Jimmy_Fallon, http://dbpedia.org/resource/Man_of_the_People_(TV_series), http://dbpedia.org/resource/Nashville_Star, http://dbpedia.org/resource/The_Danny_Thomas_Hour, http://dbpedia.org/resource/The_Death_of_the_Incredible_Hulk, http://dbpedia.org/resource/The_Incredible_Hulk_Returns, http://dbpedia.org/resource/The_Lieutenant, http://dbpedia.org/resource/The_Magician_(U.S._TV_series), http://dbpedia.org/resource/The_Oregon_Trail_(TV_series), http://dbpedia.org/resource/The_Trial_of_the_Incredible_Hulk, http://dbpedia.org/resource/You_Bet_Your_Life, http://dbpedia.org/resource/A_Killer_Among_Us, http://dbpedia.org/resource/Bitter_Harvest_(1981_film), http://dbpedia.org/resource/CHiPs, http://dbpedia.org/resource/Centennial_(miniseries), http://dbpedia.org/resource/Ed_(TV_series), http://dbpedia.org/resource/Friday_Night_Lights_(TV_series), http://dbpedia.org/resource/Hollywood_Squares, http://dbpedia.org/resource/Kidnapped_(TV_series), http://dbpedia.org/resource/Late_Night_(NBC), http://dbpedia.org/resource/NASCAR_on_NBC, http://dbpedia.org/resource/Police_Story_(TV_series), http://dbpedia.org/resource/The_A-Team, http://dbpedia.org/resource/The_David_Letterman_Show, http://dbpedia.org/resource/The_Dick_Powell_Show, http://dbpedia.org/resource/The_Gong_Show, http://dbpedia.org/resource/The_Tonight_Show_with_Conan_O'Brien, http://dbpedia.org/resource/Babes_in_Toyland_(1986_film), http://dbpedia.org/resource/Caroline_in_the_City, http://dbpedia.org/resource/Cocaine:_One_Man's_Seduction, http://dbpedia.org/resource/David_Copperfield_(1993_film), http://dbpedia.org/resource/Elf:_Buddy's_Musical_Christmas, http://dbpedia.org/resource/Friday_Night_Videos, http://dbpedia.org/resource/Hollywood_Game_Night, http://dbpedia.org/resource/Homicide:_Life_on_the_Street_(season_5), http://dbpedia.org/resource/How_Murray_Saved_Christmas, http://dbpedia.org/resource/Light's_Diamond_Jubilee, http://dbpedia.org/resource/McMillan_&_Wife, http://dbpedia.org/resource/Mr._Krueger's_Christmas, http://dbpedia.org/resource/My_World_and_Welcome_to_It, http://dbpedia.org/resource/Passions, http://dbpedia.org/resource/Phenomenon_(TV_series), http://dbpedia.org/resource/Poker_After_Dark, http://dbpedia.org/resource/Providence_(TV_series), http://dbpedia.org/resource/Saturday_Night_Live_(season_7), http://dbpedia.org/resource/Saved_by_the_Bell, http://dbpedia.org/resource/Shannon's_Deal, http://dbpedia.org/resource/The_Family_Tree, http://dbpedia.org/resource/The_Jimmy_Stewart_Show, http://dbpedia.org/resource/The_Magic_Clown, http://dbpedia.org/resource/The_New_Fantastic_Four, http://dbpedia.org/resource/The_Office_(U.S._TV_series), http://dbpedia.org/resource/The_Office_(U.S._season_7), http://dbpedia.org/resource/Thundarr_the_Barbarian, http://dbpedia.org/resource/Tonight_Starring_Jack_Paar, http://dbpedia.org/resource/Veronica's_Closet, http://dbpedia.org/resource/Death_of_a_Centerfold:_The_Dorothy_Stratten_Story, http://dbpedia.org/resource/Ensign_O'Toole, http://dbpedia.org/resource/Grandpa_Goes_to_Washington, http://dbpedia.org/resource/Poison_Ivy_(1985_film), http://dbpedia.org/resource/The_Harlem_Globetrotters_on_Gilligan's_Island, http://dbpedia.org/resource/The_Restless_Gun, http://dbpedia.org/resource/The_Black_Donnellys, http://dbpedia.org/resource/Law_&_Order:_Special_Victims_Unit_(season_14), http://dbpedia.org/resource/Undateable, http://dbpedia.org/resource/Fame_Is_the_Name_of_the_Game, http://dbpedia.org/resource/Saturday_Night_Live_(season_28), http://dbpedia.org/resource/Saturday_Night_Live_(season_29), http://dbpedia.org/resource/The_Incredible_Hulk_(1982_TV_series), http://dbpedia.org/resource/The_Odyssey_(miniseries), http://dbpedia.org/resource/Brand_New_Life_(TV_series), http://dbpedia.org/resource/Everything's_Relative_(1999_TV_series), http://dbpedia.org/resource/30_Rock_(season_4), http://dbpedia.org/resource/Snap_Judgment_(game_show), http://dbpedia.org/resource/Will_&_Grace_(season_1), http://dbpedia.org/resource/Will_&_Grace_(season_2), http://dbpedia.org/resource/Will_&_Grace_(season_3), http://dbpedia.org/resource/Will_&_Grace_(season_4), http://dbpedia.org/resource/Will_&_Grace_(season_5), http://dbpedia.org/resource/Will_&_Grace_(season_6), http://dbpedia.org/resource/Will_&_Grace_(season_7), http://dbpedia.org/resource/Will_&_Grace_(season_8), http://dbpedia.org/resource/Child_Bride_of_Short_Creek, http://dbpedia.org/resource/Sarah_T._–_Portrait_of_a_Teenage_Alcoholic, http://dbpedia.org/resource/The_Rockford_Files_(season_3), http://dbpedia.org/resource/Star_Trek:_The_Original_Series_(season_1), http://dbpedia.org/resource/Star_Trek:_The_Original_Series_(season_2), http://dbpedia.org/resource/The_Kid_with_the_200_I.Q., http://dbpedia.org/resource/The_Possessed_(1977_film), http://dbpedia.org/resource/One_Happy_Family, http://dbpedia.org/resource/Say_When!!, http://dbpedia.org/resource/Space_Cats, http://dbpedia.org/resource/The_Red_Pony_(1973_film), http://dbpedia.org/resource/Mama's_Boy_(TV_series), http://dbpedia.org/resource/Crusoe_(TV_series), http://dbpedia.org/resource/She_Cried_No, http://dbpedia.org/resource/30_Rock_(season_6), http://dbpedia.org/resource/The_Little_Rascals_Christmas_Special, http://dbpedia.org/resource/The_Rockford_Files_(season_1), http://dbpedia.org/resource/Meteor_(miniseries), http://dbpedia.org/resource/Rosetti_and_Ryan, http://dbpedia.org/resource/Who's_Watching_the_Kids%3F, http://dbpedia.org/resource/Diana:_Her_True_Story, http://dbpedia.org/resource/Great_Ghost_Tales, http://dbpedia.org/resource/The_Rockford_Files_(season_4), http://dbpedia.org/resource/The_A-Team_(season_4), http://dbpedia.org/resource/Norby_(TV_series), http://dbpedia.org/resource/The_Armed_Forces_Hour, http://dbpedia.org/resource/Friday_Night_Lights_(season_4), http://dbpedia.org/resource/My_Name_Is_Earl_(season_3), http://dbpedia.org/resource/List_of_The_Tonight_Show_with_Jay_Leno_episodes_(1992–99), http://dbpedia.org/resource/List_of_The_Tonight_Show_with_Jay_Leno_episodes_(2010), http://dbpedia.org/resource/The_Winds_of_Whoopie, http://dbpedia.org/resource/Cheers_(season_10), http://dbpedia.org/resource/The_Biggest_Loser_(Dutch_TV_series), http://dbpedia.org/resource/Baywatch, http://dbpedia.org/resource/Cheers, http://dbpedia.org/resource/Fear_Factor, http://dbpedia.org/resource/Gambit_(game_show), http://dbpedia.org/resource/Major_League_Baseball_on_NBC, http://dbpedia.org/resource/NBC_Nightly_News, http://dbpedia.org/resource/The_Baseball_Network, http://dbpedia.org/resource/The_Fresh_Prince_of_Bel-Air, http://dbpedia.org/resource/The_Revenge_of_Al_Capone, http://dbpedia.org/resource/The_Tonight_Show_Starring_Johnny_Carson, http://dbpedia.org/resource/The_West_Wing, http://dbpedia.org/resource/This_Is_Your_Life, http://dbpedia.org/resource/Today_(U.S._TV_program), http://dbpedia.org/resource/Will_&_Grace, http://dbpedia.org/resource/Your_Show_of_Shows, http://dbpedia.org/resource/A_Different_World, http://dbpedia.org/resource/Hannibal_(TV_series), http://dbpedia.org/resource/Law_&_Order:_Special_Victims_Unit_(season_13), http://dbpedia.org/resource/Let's_Make_a_Deal, http://dbpedia.org/resource/My_Name_Is_Earl, http://dbpedia.org/resource/Night_Court, http://dbpedia.org/resource/Star_Trek:_The_Animated_Series, http://dbpedia.org/resource/The_Price_Is_Right_(1956_U.S._game_show), http://dbpedia.org/resource/This_Is_Show_Business, http://dbpedia.org/resource/Truth_or_Consequences, http://dbpedia.org/resource/Twenty_One_(game_show), http://dbpedia.org/resource/Unsolved_Mysteries, http://dbpedia.org/resource/The_Hobbit_(1977_film), http://dbpedia.org/resource/The_Murder_of_Mary_Phagan, http://dbpedia.org/resource/Bates_Motel_(film), http://dbpedia.org/resource/Dick_Clark's_New_Year's_Rockin'_Eve, http://dbpedia.org/resource/Law_of_the_Plainsman, http://dbpedia.org/resource/Lipstick_Jungle_(TV_series), http://dbpedia.org/resource/Scared_Shrekless, http://dbpedia.org/resource/Gavilan_(TV_series), http://dbpedia.org/resource/Name_That_Tune, http://dbpedia.org/resource/Profiler_(TV_series), http://dbpedia.org/resource/American_Dreams, http://dbpedia.org/resource/Crusader_Rabbit, http://dbpedia.org/resource/Dante_(TV_series), http://dbpedia.org/resource/King_Leonardo_and_His_Short_Subjects, http://dbpedia.org/resource/Meet_McGraw, http://dbpedia.org/resource/The_Loneliest_Runner, http://dbpedia.org/resource/Laura_Lansing_Slept_Here, http://dbpedia.org/resource/One_Christmas_(film), http://dbpedia.org/resource/Parenthood_(1990_TV_series), http://dbpedia.org/resource/The_Rebels_(TV_series), http://dbpedia.org/resource/The_Troubleshooters_(1959_TV_series), http://dbpedia.org/resource/Law_&_Order:_Special_Victims_Unit_(season_1), http://dbpedia.org/resource/Law_&_Order:_Special_Victims_Unit_(season_2), http://dbpedia.org/resource/Sunday_Night_(American_TV_program), http://dbpedia.org/resource/CB_Bears, http://dbpedia.org/resource/Casper_and_the_Angels, http://dbpedia.org/resource/Foofur, http://dbpedia.org/resource/Going_Bananas_(U.S._series), http://dbpedia.org/resource/Hey,_Landlord, http://dbpedia.org/resource/Jokebook, http://dbpedia.org/resource/Match_Game-Hollywood_Squares_Hour, http://dbpedia.org/resource/Shirley_(TV_series), http://dbpedia.org/resource/Shirt_Tales, http://dbpedia.org/resource/Snorks, http://dbpedia.org/resource/Space_Stars, http://dbpedia.org/resource/The_Space_Kidettes, http://dbpedia.org/resource/Wheelie_and_the_Chopper_Bunch, http://dbpedia.org/resource/Yo_Yogi!, http://dbpedia.org/resource/Yogi's_Space_Race, http://dbpedia.org/resource/Danielle_Steel's_Star, http://dbpedia.org/resource/Family_Album_(miniseries), http://dbpedia.org/resource/Mister_Sterling, http://dbpedia.org/resource/Perry_Mason_(TV_movies), http://dbpedia.org/resource/Saturday_Night's_Main_Event, http://dbpedia.org/resource/Saturday_Night_Live_(season_9), http://dbpedia.org/resource/Tate_(TV_series), http://dbpedia.org/resource/The_Firm_(2012_TV_series), http://dbpedia.org/resource/Michael_Shayne, http://dbpedia.org/resource/My_Sweet_Charlie, http://dbpedia.org/resource/The_Californians_(TV_series), http://dbpedia.org/resource/For_Your_Love_(TV_series), http://dbpedia.org/resource/Tennis_on_NBC, http://dbpedia.org/resource/The_Gangster_Chronicles, http://dbpedia.org/resource/The_Great_Escape_II:_The_Untold_Story, http://dbpedia.org/resource/The_Temptations_(miniseries), http://dbpedia.org/resource/A_Year_in_the_Life, http://dbpedia.org/resource/Blind_Date_(1950s_game_show), http://dbpedia.org/resource/Brotherly_Love_(1995_TV_series), http://dbpedia.org/resource/Fall_into_Darkness, http://dbpedia.org/resource/Hidden_Faces, http://dbpedia.org/resource/How_to_Survive_a_Marriage, http://dbpedia.org/resource/My_Friend_Tony, http://dbpedia.org/resource/Our_Five_Daughters, http://dbpedia.org/resource/Saber_of_London, http://dbpedia.org/resource/Sister,_Sister_(1982_film), http://dbpedia.org/resource/The_Bold_Ones:_The_Senator, http://dbpedia.org/resource/The_Brighter_Day, http://dbpedia.org/resource/The_John_Forsythe_Show, http://dbpedia.org/resource/The_Round_Table_(TV_series), http://dbpedia.org/resource/The_Sophisticated_Gents, http://dbpedia.org/resource/A_Reason_to_Live_(1985_film), http://dbpedia.org/resource/Little_House_on_the_Prairie_(film), http://dbpedia.org/resource/Law_&_Order:_Special_Victims_Unit_(season_11), http://dbpedia.org/resource/Law_&_Order:_Special_Victims_Unit_(season_12), http://dbpedia.org/resource/Law_&_Order:_Special_Victims_Unit_(season_15), http://dbpedia.org/resource/Law_&_Order:_Special_Victims_Unit_(season_16), http://dbpedia.org/resource/Law_&_Order:_Special_Victims_Unit_(season_17), http://dbpedia.org/resource/Law_&_Order:_Special_Victims_Unit_(season_3), http://dbpedia.org/resource/Law_&_Order:_Special_Victims_Unit_(season_4), http://dbpedia.org/resource/Law_&_Order:_Special_Victims_Unit_(season_5), http://dbpedia.org/resource/Law_&_Order:_Special_Victims_Unit_(season_6), http://dbpedia.org/resource/Andy_Barker,_P.I., http://dbpedia.org/resource/Saturday_Night_Live_(season_26), http://dbpedia.org/resource/Saturday_Night_Live_(season_27), http://dbpedia.org/resource/Saturday_Night_Live_(season_30), http://dbpedia.org/resource/Scrubs_(season_1), http://dbpedia.org/resource/Scrubs_(season_2), http://dbpedia.org/resource/Scrubs_(season_3), http://dbpedia.org/resource/Scrubs_(season_5), http://dbpedia.org/resource/Scrubs_(season_6), http://dbpedia.org/resource/Scrubs_(season_7), http://dbpedia.org/resource/The_Golden_Girls_(season_2), http://dbpedia.org/resource/Saturday_Night_Live_(season_38), http://dbpedia.org/resource/The_Berenstain_Bears_Meet_Bigpaw, http://dbpedia.org/resource/ALF_Tales, http://dbpedia.org/resource/Private_Eye_(TV_series), http://dbpedia.org/resource/Age_of_Love_(TV_series), http://dbpedia.org/resource/Clash_of_the_Choirs, http://dbpedia.org/resource/A.U.S.A., http://dbpedia.org/resource/Frasier_(season_5), http://dbpedia.org/resource/The_Gambler_(film_series), http://dbpedia.org/resource/The_Jake_Effect, http://dbpedia.org/resource/The_Second_Half, http://dbpedia.org/resource/Crisis_Center, http://dbpedia.org/resource/PDQ_(game_show), http://dbpedia.org/resource/What's_This_Song%3F, http://dbpedia.org/resource/The_Stepford_Children, http://dbpedia.org/resource/Built_to_Last_(TV_series), http://dbpedia.org/resource/House_Rules_(1998_TV_series), http://dbpedia.org/resource/Mr._and_Mrs._North, http://dbpedia.org/resource/Black_Tie_Affair_(TV_series), http://dbpedia.org/resource/Chicago_Sons, http://dbpedia.org/resource/The_World_of_Mr._Sweeney, http://dbpedia.org/resource/Friday_Night_Lights_(season_1), http://dbpedia.org/resource/Snip_(TV_series), http://dbpedia.org/resource/Her_Hidden_Truth, http://dbpedia.org/resource/If_Someone_Had_Known, http://dbpedia.org/resource/Police_Story:_Confessions_of_a_Lady_Cop, http://dbpedia.org/resource/Parks_and_Recreation_(season_1), http://dbpedia.org/resource/Parks_and_Recreation_(season_3), http://dbpedia.org/resource/Parks_and_Recreation_(season_5), http://dbpedia.org/resource/The_West_Wing_(season_6), http://dbpedia.org/resource/The_Biggest_Loser:_Glory_Days, http://dbpedia.org/resource/Valley_Forge_(film), http://dbpedia.org/resource/The_Fresh_Prince_of_Bel-Air_(season_4), http://dbpedia.org/resource/Rewind_(2013_film), http://dbpedia.org/resource/Community_(season_5), http://dbpedia.org/resource/A_Burning_Passion:_The_Margaret_Mitchell_Story, http://dbpedia.org/resource/The_Crisis_(TV_series), http://dbpedia.org/resource/In_Wine_Country, http://dbpedia.org/resource/Friday_Night_Lights_(season_3), http://dbpedia.org/resource/People_Will_Talk_(game_show), http://dbpedia.org/resource/The_Office:_The_Accountants, http://dbpedia.org/resource/The_Swift_Show, http://dbpedia.org/resource/Frasier_(season_11), http://dbpedia.org/resource/The_Blacklist_(season_3), http://dbpedia.org/resource/Chain_Letter_(U.S._game_show), http://dbpedia.org/resource/3rd_Rock_from_the_Sun_(season_3), http://dbpedia.org/resource/Just_Shoot_Me!_(season_1), http://dbpedia.org/resource/Just_Shoot_Me!_(season_2), http://dbpedia.org/resource/Just_Shoot_Me!_(season_3), http://dbpedia.org/resource/Just_Shoot_Me!_(season_4), http://dbpedia.org/resource/Just_Shoot_Me!_(season_5), http://dbpedia.org/resource/Just_Shoot_Me!_(season_6), http://dbpedia.org/resource/Just_Shoot_Me!_(season_7), http://dbpedia.org/resource/The_Rockford_Files_(season_5), http://dbpedia.org/resource/The_Rockford_Files_(season_6), http://dbpedia.org/resource/All_Star_Secrets, http://dbpedia.org/resource/Columbo, http://dbpedia.org/resource/Crossing_Jordan, http://dbpedia.org/resource/Cruel_Doubt, http://dbpedia.org/resource/ER_(TV_series), http://dbpedia.org/resource/Empire_(1962_TV_series), http://dbpedia.org/resource/Football_Night_in_America, http://dbpedia.org/resource/Frasier, http://dbpedia.org/resource/Generations_(U.S._TV_series), http://dbpedia.org/resource/Golf_Channel_on_NBC, http://dbpedia.org/resource/Hill_Street_Blues, http://dbpedia.org/resource/Homicide:_Life_on_the_Street, http://dbpedia.org/resource/Las_Vegas_(TV_series), http://dbpedia.org/resource/Late_Night_with_Conan_O'Brien, http://dbpedia.org/resource/Late_Night_with_David_Letterman, http://dbpedia.org/resource/Later_(talk_show), http://dbpedia.org/resource/Matlock_(TV_series), http://dbpedia.org/resource/NBA_on_NBC, http://dbpedia.org/resource/NBC_Sunday_Night_Football, http://dbpedia.org/resource/NFL_on_NBC, http://dbpedia.org/resource/NHL_on_NBC, http://dbpedia.org/resource/NewsRadio, http://dbpedia.org/resource/Olympics_on_NBC, http://dbpedia.org/resource/Quantum_Leap, http://dbpedia.org/resource/Rock_Center_with_Brian_Williams, http://dbpedia.org/resource/Second_City_Television, http://dbpedia.org/resource/Sunshine_(U.S._TV_series), http://dbpedia.org/resource/Tattingers, http://dbpedia.org/resource/The_Rockford_Files, http://dbpedia.org/resource/Thursday_Night_Football, http://dbpedia.org/resource/Up_All_Night_(TV_series), http://dbpedia.org/resource/A_Man_and_His_Music_+_Ella_+_Jobim, http://dbpedia.org/resource/Baa_Baa_Black_Sheep_(TV_series), http://dbpedia.org/resource/Frank_Sinatra:_A_Man_and_His_Music, http://dbpedia.org/resource/Friends_(season_1), http://dbpedia.org/resource/Friends_(season_10), http://dbpedia.org/resource/Friends_(season_2), http://dbpedia.org/resource/Friends_(season_3), http://dbpedia.org/resource/Friends_(season_4), http://dbpedia.org/resource/Friends_(season_5), http://dbpedia.org/resource/Friends_(season_6), http://dbpedia.org/resource/Friends_(season_7), http://dbpedia.org/resource/Friends_(season_8), http://dbpedia.org/resource/Friends_(season_9), http://dbpedia.org/resource/Get_Smart, http://dbpedia.org/resource/Grimm_(season_4), http://dbpedia.org/resource/Joey_(TV_series), http://dbpedia.org/resource/Mad_About_You, http://dbpedia.org/resource/Wichita_Town, http://dbpedia.org/resource/Wings_(1990_TV_series), http://dbpedia.org/resource/Cartoon_All-Stars_to_the_Rescue, http://dbpedia.org/resource/Just_Shoot_Me!, http://dbpedia.org/resource/Law_&_Order:_Criminal_Intent, http://dbpedia.org/resource/The_New_Adventures_of_Huckleberry_Finn, http://dbpedia.org/resource/A_Muppets_Christmas:_Letters_to_Santa, http://dbpedia.org/resource/America's_Got_Talent, http://dbpedia.org/resource/Captain_N:_The_Game_Master, http://dbpedia.org/resource/Chuck_(TV_series), http://dbpedia.org/resource/ER_(season_3), http://dbpedia.org/resource/Emergency!, http://dbpedia.org/resource/Fired_Up_(TV_series), http://dbpedia.org/resource/Hunter_(U.S._TV_series), http://dbpedia.org/resource/It's_a_Very_Merry_Muppet_Christmas_Movie, http://dbpedia.org/resource/Lucas_Tanner, http://dbpedia.org/resource/Macy's_Thanksgiving_Day_Parade, http://dbpedia.org/resource/Man_from_Atlantis, http://dbpedia.org/resource/Mickey's_60th_Birthday, http://dbpedia.org/resource/Police_Woman_(TV_series), http://dbpedia.org/resource/Sisters_(TV_series), http://dbpedia.org/resource/Stark_Raving_Mad_(TV_series), http://dbpedia.org/resource/The_Bold_Ones:_The_New_Doctors, http://dbpedia.org/resource/The_Highwayman_(TV_series), http://dbpedia.org/resource/Third_Watch, http://dbpedia.org/resource/Westwind_(TV_series), http://dbpedia.org/resource/1st_Look, http://dbpedia.org/resource/3-2-1_Penguins!, http://dbpedia.org/resource/A_Man_and_His_Music_–_Part_II, http://dbpedia.org/resource/Adam-12, http://dbpedia.org/resource/Buffalo_Bill_(TV_series), http://dbpedia.org/resource/California_Dreams, http://dbpedia.org/resource/Card_Sharks, http://dbpedia.org/resource/City_Guys, http://dbpedia.org/resource/Concentration_(game_show), http://dbpedia.org/resource/Deal_or_No_Deal_(U.S._game_show), http://dbpedia.org/resource/Early_Today, http://dbpedia.org/resource/Endurance_(TV_series), http://dbpedia.org/resource/Got_Talent, http://dbpedia.org/resource/H.R._Pufnstuf, http://dbpedia.org/resource/Hang_Time_(TV_series), http://dbpedia.org/resource/High_Rollers, http://dbpedia.org/resource/Highway_to_Heaven, http://dbpedia.org/resource/Huntley-Brinkley_Report, http://dbpedia.org/resource/In_the_Heat_of_the_Night_(TV_series), http://dbpedia.org/resource/Knight_Rider_(2008_TV_series), http://dbpedia.org/resource/Leeza, http://dbpedia.org/resource/Minute_to_Win_It, http://dbpedia.org/resource/NBC_News_at_Sunrise, http://dbpedia.org/resource/NBC_Nightside, http://dbpedia.org/resource/Nero_Wolfe_(1981_TV_series), http://dbpedia.org/resource/Notre_Dame_Football_on_NBC, http://dbpedia.org/resource/One_World_(TV_series), http://dbpedia.org/resource/One_to_Grow_On, http://dbpedia.org/resource/Password_Plus_and_Super_Password, http://dbpedia.org/resource/Santa_Barbara_(TV_series), http://dbpedia.org/resource/Saved_by_the_Bell:_The_New_Class, http://dbpedia.org/resource/Scout's_Safari, http://dbpedia.org/resource/Somerset_(TV_series), http://dbpedia.org/resource/Sunset_Beach_(TV_series), http://dbpedia.org/resource/The_Apprentice_(U.S._TV_series), http://dbpedia.org/resource/The_Banana_Splits, http://dbpedia.org/resource/The_Biggest_Loser_(U.S._TV_series), http://dbpedia.org/resource/The_Bugaloos, http://dbpedia.org/resource/The_Flip_Wilson_Show, http://dbpedia.org/resource/The_Hogan_Family, http://dbpedia.org/resource/The_Krofft_Superstar_Hour, http://dbpedia.org/resource/The_Marriage_Ref_(U.S._TV_series), http://dbpedia.org/resource/The_Smurfs_(TV_series), http://dbpedia.org/resource/The_Waverly_Wonders, http://dbpedia.org/resource/Time_Warp_Trio, http://dbpedia.org/resource/Tomorrow_(TV_series), http://dbpedia.org/resource/Tonight_Starring_Steve_Allen, http://dbpedia.org/resource/Weekend_Today, http://dbpedia.org/resource/Fabulous_Funnies, http://dbpedia.org/resource/God,_the_Devil_and_Bob, http://dbpedia.org/resource/Star_Trek:_The_Original_Series_(season_3), http://dbpedia.org/resource/The_Kid_Super_Power_Hour_with_Shazam!, http://dbpedia.org/resource/The_New_Adventures_of_Flash_Gordon, http://dbpedia.org/resource/The_Secret_Lives_of_Waldo_Kitty, http://dbpedia.org/resource/A_Tour_of_the_White_House_with_Mrs._John_F._Kennedy, http://dbpedia.org/resource/ProStars, http://dbpedia.org/resource/Boomtown_(2002_TV_series), http://dbpedia.org/resource/Frasier_(season_6), http://dbpedia.org/resource/Knight_Rider_2000, http://dbpedia.org/resource/Team_Knight_Rider, http://dbpedia.org/resource/Evil_Roy_Slade, http://dbpedia.org/resource/Jackpot_Bowling, http://dbpedia.org/resource/San_Francisco_International_Airport_(TV_series), http://dbpedia.org/resource/The_Paul_Winchell_Show, http://dbpedia.org/resource/The_Pink_Panther_Show, http://dbpedia.org/resource/An_Evening_with_Fred_Astaire, http://dbpedia.org/resource/Choices_of_the_Heart, http://dbpedia.org/resource/Down_You_Go, http://dbpedia.org/resource/Father_Dowling_Mysteries, http://dbpedia.org/resource/Fear_Itself_(TV_series), http://dbpedia.org/resource/Godzilla_(animated_series), http://dbpedia.org/resource/Hec_Ramsey, http://dbpedia.org/resource/Monster_Squad, http://dbpedia.org/resource/Poor_Little_Rich_Girl:_The_Barbara_Hutton_Story, http://dbpedia.org/resource/Prime_Suspect_(U.S._TV_series), http://dbpedia.org/resource/The_Psychiatrist_(TV_series), http://dbpedia.org/resource/Law_&_Order:_LA, http://dbpedia.org/resource/Law_&_Order:_Trial_by_Jury, http://dbpedia.org/resource/Witness_to_the_Mob, http://dbpedia.org/resource/Camp_Candy, http://dbpedia.org/resource/Saturday_Night_Live_(season_13), http://dbpedia.org/resource/Witch's_Night_Out, http://dbpedia.org/resource/Columbo_(season_3), http://dbpedia.org/resource/Committed_(2005_TV_series), http://dbpedia.org/resource/Legends_of_the_Superheroes, http://dbpedia.org/resource/List_of_The_Alfred_Hitchcock_Hour_episodes, http://dbpedia.org/resource/Saturday_Night_Live_(season_20), http://dbpedia.org/resource/Stars_Earn_Stripes, http://dbpedia.org/resource/The_Court_of_Last_Resort, http://dbpedia.org/resource/$25_Million_Dollar_Hoax, http://dbpedia.org/resource/Cold_Feet_(U.S._TV_series), http://dbpedia.org/resource/Coupling_(U.S._TV_series), http://dbpedia.org/resource/I'm_a_Celebrity...Get_Me_Out_of_Here!_(U.S._TV_series), http://dbpedia.org/resource/Men_Behaving_Badly_(U.S._TV_series), http://dbpedia.org/resource/Return_to_the_Planet_of_the_Apes, http://dbpedia.org/resource/America's_Toughest_Jobs, http://dbpedia.org/resource/Amy_Prentiss, http://dbpedia.org/resource/Are_You_There,_Chelsea%3F, http://dbpedia.org/resource/Baby_Boom_(U.S._TV_series), http://dbpedia.org/resource/Battery_Park_(TV_series), http://dbpedia.org/resource/Blank_Check_(game_show), http://dbpedia.org/resource/Buford_and_the_Galloping_Ghost, http://dbpedia.org/resource/C._Everett_Koop,_M.D., http://dbpedia.org/resource/Campus_Hoopla, http://dbpedia.org/resource/Chicago_Story]... [INFO] Predicted Answer: [http://dbpedia.org/resource/Cheers_(season_1), http://dbpedia.org/resource/Cheers_(season_2), http://dbpedia.org/resource/Cheers_(season_3), http://dbpedia.org/resource/Cheers_(season_4), http://dbpedia.org/resource/Cheers_(season_5), http://dbpedia.org/resource/Cheers_(season_6), http://dbpedia.org/resource/Cheers_(season_7), http://dbpedia.org/resource/Frasier_(season_9), http://dbpedia.org/resource/Friends, http://dbpedia.org/resource/It's_Punky_Brewster, http://dbpedia.org/resource/Scrubs_(TV_series), http://dbpedia.org/resource/St._Elsewhere, http://dbpedia.org/resource/The_Cosby_Show, http://dbpedia.org/resource/Alvin_and_the_Chipmunks_(1983_TV_series), http://dbpedia.org/resource/Dolly_Parton's_Coat_of_Many_Colors, http://dbpedia.org/resource/E-Ring, http://dbpedia.org/resource/Father_of_the_Pride, http://dbpedia.org/resource/High_School_U.S.A., http://dbpedia.org/resource/Information_Please, http://dbpedia.org/resource/My_Two_Dads, http://dbpedia.org/resource/Queen_for_a_Day, http://dbpedia.org/resource/Remington_Steele, http://dbpedia.org/resource/Rowan_&_Martin's_Laugh-In, http://dbpedia.org/resource/Saturday_Night_Live_(season_1), http://dbpedia.org/resource/The_Man_from_U.N.C.L.E., http://dbpedia.org/resource/The_Single_Guy, http://dbpedia.org/resource/The_Steve_Allen_Show, http://dbpedia.org/resource/The_Virginian_(TV_series), http://dbpedia.org/resource/Victoria_Beckham:_Coming_to_America, http://dbpedia.org/resource/Walt_Disney_anthology_television_series, http://dbpedia.org/resource/Banacek, http://dbpedia.org/resource/Buck_Rogers_in_the_25th_Century_(TV_series), http://dbpedia.org/resource/Guilty_or_Innocent:_The_Sam_Sheppard_Murder_Case, http://dbpedia.org/resource/Howdy_Doody, http://dbpedia.org/resource/I_Spy_(1965_TV_series), http://dbpedia.org/resource/Magnavox_Presents_Frank_Sinatra, http://dbpedia.org/resource/Suspicion_(TV_series), http://dbpedia.org/resource/The_Funny_Side, http://dbpedia.org/resource/The_Gary_Coleman_Show, http://dbpedia.org/resource/The_Ghost_&_Mrs._Muir_(TV_series), http://dbpedia.org/resource/The_High_Chaparral, http://dbpedia.org/resource/The_Kaiser_Aluminum_Hour, http://dbpedia.org/resource/The_New_Show, http://dbpedia.org/resource/The_Questor_Tapes, http://dbpedia.org/resource/To_Tell_the_Truth, http://dbpedia.org/resource/Exiled:_A_Law_&_Order_Movie, http://dbpedia.org/resource/Fame_(1982_TV_series), http://dbpedia.org/resource/Late_Night_with_Seth_Meyers, http://dbpedia.org/resource/Law_&_Order:_Special_Victims_Unit_(season_7), http://dbpedia.org/resource/On_the_Edge_of_Innocence, http://dbpedia.org/resource/Spider-Man_and_His_Amazing_Friends, http://dbpedia.org/resource/The_Jay_Leno_Show, http://dbpedia.org/resource/L.A._Takedown, http://dbpedia.org/resource/Rudolph_the_Red-Nosed_Reindeer_(TV_special), http://dbpedia.org/resource/The_Rocky_and_Bullwinkle_Show, http://dbpedia.org/resource/Quincy,_M.E., http://dbpedia.org/resource/The_First_Olympics:_Athens_1896, http://dbpedia.org/resource/The_Golden_Girls_(season_4), http://dbpedia.org/resource/A_Man_Called_Sloane, http://dbpedia.org/resource/Please_Don't_Eat_the_Daisies_(TV_series), http://dbpedia.org/resource/The_Parent_Trap_III, http://dbpedia.org/resource/The_Parent_Trap_IV:_Hawaiian_Honeymoon, http://dbpedia.org/resource/AFL_on_NBC, http://dbpedia.org/resource/The_Jensen_Project, http://dbpedia.org/resource/Three_Sisters_(TV_series), http://dbpedia.org/resource/50_Grand_Slam, http://dbpedia.org/resource/Against_the_Grain_(TV_series), http://dbpedia.org/resource/Eerie,_Indiana, http://dbpedia.org/resource/The_Dinah_Shore_Chevy_Show, http://dbpedia.org/resource/Halloween_with_the_New_Addams_Family, http://dbpedia.org/resource/It's_Worth_What%3F, http://dbpedia.org/resource/Lipshitz_Saves_the_World, http://dbpedia.org/resource/Little_Mo_(1978_film), http://dbpedia.org/resource/Saturday_Night_Live_(season_40), http://dbpedia.org/resource/The_Addams_Family_(1973_animated_series), http://dbpedia.org/resource/The_Bob_Cummings_Show, http://dbpedia.org/resource/The_Bold_Ones:_The_Protectors, http://dbpedia.org/resource/The_Cosby_Mysteries, http://dbpedia.org/resource/Whoopi, http://dbpedia.org/resource/Your_First_Impression, http://dbpedia.org/resource/Fire_on_the_Mountain_(1981_film), http://dbpedia.org/resource/Five_Fingers_(TV_series), http://dbpedia.org/resource/Liza_with_a_Z, http://dbpedia.org/resource/Rainbow_(1978_film), http://dbpedia.org/resource/Return_to_Mayberry, http://dbpedia.org/resource/The_Bold_Ones, http://dbpedia.org/resource/1_vs._100_(U.S._game_show), http://dbpedia.org/resource/Average_Joe_(TV_series), http://dbpedia.org/resource/Chain_Reaction_(game_show), http://dbpedia.org/resource/Jackpot_(game_show), http://dbpedia.org/resource/Time_Machine_(game_show), http://dbpedia.org/resource/Treasure_Hunt_(U.S._game_show), http://dbpedia.org/resource/Win,_Lose_or_Draw, http://dbpedia.org/resource/Banyon, http://dbpedia.org/resource/Café_Americain, http://dbpedia.org/resource/Camel_News_Caravan, http://dbpedia.org/resource/Celebrity_Family_Feud, http://dbpedia.org/resource/City_of_Angels_(1976_TV_series), http://dbpedia.org/resource/Dawn:_Portrait_of_a_Teenage_Runaway, http://dbpedia.org/resource/Fraggle_Rock:_The_Animated_Series, http://dbpedia.org/resource/I_Dream_of_Jeannie..._Fifteen_Years_Later, http://dbpedia.org/resource/Return_to_Peyton_Place_(TV_series), http://dbpedia.org/resource/Runaround_(game_show), http://dbpedia.org/resource/Surface_(TV_series), http://dbpedia.org/resource/Bonino_(TV_series), http://dbpedia.org/resource/Bret_Maverick, http://dbpedia.org/resource/Cursed_(TV_series), http://dbpedia.org/resource/LateLine, http://dbpedia.org/resource/Baggy_Pants_and_the_Nitwits, http://dbpedia.org/resource/Quantum_Leap_(season_2), http://dbpedia.org/resource/Quarterlife, http://dbpedia.org/resource/Something_for_a_Lonely_Man, http://dbpedia.org/resource/Then_Came_Bronson, http://dbpedia.org/resource/Wild_Kingdom, http://dbpedia.org/resource/Free_Agents_(U.S._TV_series), http://dbpedia.org/resource/Hitler's_SS:_Portrait_in_Evil, http://dbpedia.org/resource/Murder_in_Texas, http://dbpedia.org/resource/Revenge_of_the_Stepford_Wives, http://dbpedia.org/resource/The_Bob_Newhart_Show_(1961_TV_series), http://dbpedia.org/resource/Redigo_(TV_series), http://dbpedia.org/resource/Rockin'_Through_the_Decades, http://dbpedia.org/resource/Saturday_Night_Live_(season_39), http://dbpedia.org/resource/Ding_Dong_School, http://dbpedia.org/resource/M.Y.O.B._(TV_series), http://dbpedia.org/resource/Boone_(TV_series), http://dbpedia.org/resource/Kingpin_(TV_series), http://dbpedia.org/resource/Emergency_+4, http://dbpedia.org/resource/The_Snoop_Sisters, http://dbpedia.org/resource/30_Rock_(season_7), http://dbpedia.org/resource/33⅓_Revolutions_per_Monkee, http://dbpedia.org/resource/Battlestars_(game_show), http://dbpedia.org/resource/Joe_Forrester_(TV_series), http://dbpedia.org/resource/Law_&_Order_(season_9), http://dbpedia.org/resource/Little_Richard_(film), http://dbpedia.org/resource/Love,_Sidney, http://dbpedia.org/resource/Northwest_Passage_(TV_series), http://dbpedia.org/resource/The_More_You_Know, http://dbpedia.org/resource/Movin'_with_Nancy, http://dbpedia.org/resource/Scrubs_(season_4), http://dbpedia.org/resource/The_Spike_Jones_Show_(series), http://dbpedia.org/resource/Danger_Island_(film), http://dbpedia.org/resource/Dragnet_(1951_TV_series), http://dbpedia.org/resource/Fashion_Star_(season_1), http://dbpedia.org/resource/Living_Proof:_The_Hank_Williams,_Jr._Story, http://dbpedia.org/resource/Thank_God_You're_Here_(U.S._TV_series), http://dbpedia.org/resource/The_Real_Wedding_Crashers, http://dbpedia.org/resource/Fatal_Vision_(miniseries), http://dbpedia.org/resource/Call_My_Bluff_(U.S._game_show), http://dbpedia.org/resource/Trinity_(U.S._TV_series), http://dbpedia.org/resource/The_Biggest_Loser:_Families, http://dbpedia.org/resource/Saturday_Night_Live_(season_34), http://dbpedia.org/resource/The_Restaurant_(U.S._TV_series), http://dbpedia.org/resource/World's_Most_Amazing_Videos, http://dbpedia.org/resource/Your_Mother_Wears_Combat_Boots, http://dbpedia.org/resource/Parks_and_Recreation_(season_2), http://dbpedia.org/resource/Saturday_Night_Live_(season_10), http://dbpedia.org/resource/Saturday_Night_Live_(season_32), http://dbpedia.org/resource/Reach_for_the_Stars_(game_show), http://dbpedia.org/resource/The_Rockford_Files_(season_2), http://dbpedia.org/resource/Jigsaw_John_(TV_series), http://dbpedia.org/resource/Someone_Like_Me_(TV_series), http://dbpedia.org/resource/Winnetka_Road, http://dbpedia.org/resource/Combat_Academy, http://dbpedia.org/resource/Kate's_Secret, http://dbpedia.org/resource/Class_Cruise, http://dbpedia.org/resource/Science_of_Love, http://dbpedia.org/resource/Silent_Witness_(1985_film), http://dbpedia.org/resource/Moment_of_Truth:_Why_My_Daughter%3F, http://dbpedia.org/resource/Strong_(TV_series), http://dbpedia.org/resource/My_Name_Is_Earl_(season_4), http://dbpedia.org/resource/Chicago_Fire_(season_1), http://dbpedia.org/resource/Chicago_Fire_(season_2), http://dbpedia.org/resource/Chicago_Fire_(season_3), http://dbpedia.org/resource/Chicago_Fire_(season_4), http://dbpedia.org/resource/Chicago_P.D._(season_1), http://dbpedia.org/resource/Chicago_P.D._(season_2), http://dbpedia.org/resource/Chicago_P.D._(season_3), http://dbpedia.org/resource/Make_Your_Own_Kind_of_Music_(TV_series), http://dbpedia.org/resource/Knight_Rider_(season_4), http://dbpedia.org/resource/Fashion_Star_(season_2), http://dbpedia.org/resource/The_Six_of_Us, http://dbpedia.org/resource/The_Ted_Steele_Show, http://dbpedia.org/resource/Friday_Night_Lights_(season_2), http://dbpedia.org/resource/Take_a_Chance_(U.S._game_show), http://dbpedia.org/resource/Parenthood_(season_1), http://dbpedia.org/resource/Parenthood_(season_3), http://dbpedia.org/resource/Parenthood_(season_4), http://dbpedia.org/resource/Parenthood_(season_5), http://dbpedia.org/resource/Parenthood_(season_6), http://dbpedia.org/resource/Where_the_Girls_Are_(TV_program), http://dbpedia.org/resource/The_A-Team_(season_2), http://dbpedia.org/resource/The_A-Team_(season_3), http://dbpedia.org/resource/The_A-Team_(season_5), http://dbpedia.org/resource/Adam-12_(season_2), http://dbpedia.org/resource/Dragnet_(1967_series)_(season_1), http://dbpedia.org/resource/Dragnet_(1967_series)_(season_3), http://dbpedia.org/resource/Dragnet_(1967_series)_(season_4), http://dbpedia.org/resource/Ironside_(season_1), http://dbpedia.org/resource/Ironside_(season_2), http://dbpedia.org/resource/Ironside_(season_4), http://dbpedia.org/resource/Ironside_(season_5), http://dbpedia.org/resource/Ironside_(season_6), http://dbpedia.org/resource/Ironside_(season_8), http://dbpedia.org/resource/Quincy,_M.E._(season_1), http://dbpedia.org/resource/Quincy,_M.E._(season_2), http://dbpedia.org/resource/Quincy,_M.E._(season_3), http://dbpedia.org/resource/Quincy,_M.E._(season_4), http://dbpedia.org/resource/Quincy,_M.E._(season_5), http://dbpedia.org/resource/Quincy,_M.E._(season_6), http://dbpedia.org/resource/Quincy,_M.E._(season_7), http://dbpedia.org/resource/Quincy,_M.E._(season_8), http://dbpedia.org/resource/3rd_Rock_from_the_Sun, http://dbpedia.org/resource/Alfred_Hitchcock_Presents, http://dbpedia.org/resource/Alfred_Hitchcock_Presents_(1985_TV_series), http://dbpedia.org/resource/Jeopardy!, http://dbpedia.org/resource/Star_Trek:_The_Original_Series, http://dbpedia.org/resource/30_Rock_(season_3), http://dbpedia.org/resource/Crime_Story_(TV_series), http://dbpedia.org/resource/Frasier_(season_10), http://dbpedia.org/resource/Knight_Rider_(1982_TV_series), http://dbpedia.org/resource/Studio_60_on_the_Sunset_Strip, http://dbpedia.org/resource/The_Tonight_Show_Starring_Jimmy_Fallon, http://dbpedia.org/resource/Alice_in_Wonderland_(1999_film), http://dbpedia.org/resource/Caesar's_Hour, http://dbpedia.org/resource/College_Basketball_on_NBC, http://dbpedia.org/resource/Eischied, http://dbpedia.org/resource/Major_League_Baseball:_An_Inside_Look, http://dbpedia.org/resource/Thoroughbred_Racing_on_NBC, http://dbpedia.org/resource/WWE_Tribute_to_the_Troops, http://dbpedia.org/resource/American_Gladiators_(2008_TV_series), http://dbpedia.org/resource/Mann_&_Machine, http://dbpedia.org/resource/National_Heads-Up_Poker_Championship, http://dbpedia.org/resource/Ringo_(1978_film), http://dbpedia.org/resource/Space_Sentinels, http://dbpedia.org/resource/The_Golden_Girls_(season_1), http://dbpedia.org/resource/The_Rogues_(TV_series), http://dbpedia.org/resource/The_Yellow_Rose, http://dbpedia.org/resource/Suddenly_Susan, http://dbpedia.org/resource/A_Flintstone_Christmas, http://dbpedia.org/resource/Fred_and_Barney_Meet_The_Thing, http://dbpedia.org/resource/Fred_and_Barney_Meet_the_Shmoo, http://dbpedia.org/resource/Galaxy_Goof-Ups, http://dbpedia.org/resource/Serving_in_Silence:_The_Margarethe_Cammermeyer_Story, http://dbpedia.org/resource/The_Flintstone_Funnies, http://dbpedia.org/resource/The_New_Fred_and_Barney_Show, http://dbpedia.org/resource/You,_Me_and_the_Apocalypse, http://dbpedia.org/resource/Nearly_Departed, http://dbpedia.org/resource/Parks_and_Recreation_(season_7), http://dbpedia.org/resource/Sealab_2020, http://dbpedia.org/resource/The_Flintstones'_New_Neighbors, http://dbpedia.org/resource/The_Flintstones:_Fred's_Final_Fling, http://dbpedia.org/resource/The_Flintstones:_Jogging_Fever, http://dbpedia.org/resource/The_Flintstones:_Little_Big_League, http://dbpedia.org/resource/The_Flintstones:_Wind-Up_Wilma, http://dbpedia.org/resource/The_Flintstones_Meet_Rockula_and_Frankenstone, http://dbpedia.org/resource/Identity_(game_show), http://dbpedia.org/resource/On_Wings_of_Eagles_(miniseries), http://dbpedia.org/resource/Outlaws_(1960_TV_series), http://dbpedia.org/resource/Who_Said_That%3F, http://dbpedia.org/resource/The_Biggest_Loser:_No_Excuses, http://dbpedia.org/resource/The_Burning_Bed, http://dbpedia.org/resource/Awake_(TV_series), http://dbpedia.org/resource/The_Biggest_Loser_(season_3), http://dbpedia.org/resource/The_Book_of_Daniel_(TV_series), http://dbpedia.org/resource/Run,_Joe,_Run, http://dbpedia.org/resource/Sigmund_and_the_Sea_Monsters, http://dbpedia.org/resource/The_Bold_Ones:_The_Lawyers, http://dbpedia.org/resource/Alexander:_The_Other_Side_of_Dawn, http://dbpedia.org/resource/Law_&_Order:_Special_Victims_Unit_(season_10), http://dbpedia.org/resource/The_Gabby_Hayes_Show, http://dbpedia.org/resource/Deadline_(American_TV_series), http://dbpedia.org/resource/In_Love_and_War_(1987_film), http://dbpedia.org/resource/Saturday_Night_Live_(season_2), http://dbpedia.org/resource/She_Spies, http://dbpedia.org/resource/Adam_(1983_film), http://dbpedia.org/resource/The_Ruff_and_Reddy_Show, http://dbpedia.org/resource/McKeever_and_the_Colonel, http://dbpedia.org/resource/Sierra_(TV_series), http://dbpedia.org/resource/Columbo_(season_4), http://dbpedia.org/resource/Columbo_(season_5), http://dbpedia.org/resource/Jewels_(miniseries), http://dbpedia.org/resource/Kristin_(TV_series), http://dbpedia.org/resource/Quantum_Leap_(season_5), http://dbpedia.org/resource/The_Fresh_Prince_of_Bel-Air_(season_3), http://dbpedia.org/resource/The_Secret_Life_of_Kathy_McCormick, http://dbpedia.org/resource/Casper's_First_Christmas, http://dbpedia.org/resource/Doug_Henning's_World_of_Magic, http://dbpedia.org/resource/Most_Outrageous_Moments, http://dbpedia.org/resource/Betty_White's_Off_Their_Rockers, http://dbpedia.org/resource/Half_Nelson_(TV_series), http://dbpedia.org/resource/Jack_Frost_(TV_special), http://dbpedia.org/resource/Lookwell, http://dbpedia.org/resource/Saturday_Night_Live_(season_18), http://dbpedia.org/resource/Homeland_Security_(film), http://dbpedia.org/resource/Skunked_TV, http://dbpedia.org/resource/The_Man_and_the_Challenge, http://dbpedia.org/resource/Sweet_Justice, http://dbpedia.org/resource/A_Howling_in_the_Woods, http://dbpedia.org/resource/I_Still_Dream_of_Jeannie, http://dbpedia.org/resource/McDuff,_the_Talking_Dog, http://dbpedia.org/resource/In_the_Line_of_Duty:_The_F.B.I._Murders, http://dbpedia.org/resource/Legmen, http://dbpedia.org/resource/Revelations_(miniseries), http://dbpedia.org/resource/The_Innocent_(1994_film), http://dbpedia.org/resource/A_Chipmunk_Reunion, http://dbpedia.org/resource/Opposites_Attract_(1990_film), http://dbpedia.org/resource/Those_She_Left_Behind, http://dbpedia.org/resource/Popeye_Doyle_(film), http://dbpedia.org/resource/A_Child's_Cry_for_Help, http://dbpedia.org/resource/Frasier_(season_4), http://dbpedia.org/resource/Miracle_Child_(1993_film), http://dbpedia.org/resource/Snowbeast, http://dbpedia.org/resource/Grimm_(season_1), http://dbpedia.org/resource/Dragnet_(1967_series)_(season_2), http://dbpedia.org/resource/Come_to_Papa_(TV_series), http://dbpedia.org/resource/Lazer_Tag_Academy, http://dbpedia.org/resource/My_Son_Jeep, http://dbpedia.org/resource/Community_(season_2), http://dbpedia.org/resource/The_Jane_Whitney_Show, http://dbpedia.org/resource/The_Kids_From_C.A.P.E.R., http://dbpedia.org/resource/The_World's_Most_Dangerous_Magic, http://dbpedia.org/resource/My_Name_Is_Earl_(season_2), http://dbpedia.org/resource/The_Office_(U.S._season_2), http://dbpedia.org/resource/Amnesia_(game_show), http://dbpedia.org/resource/3rd_Rock_from_the_Sun_(season_2), http://dbpedia.org/resource/3rd_Rock_from_the_Sun_(season_6), http://dbpedia.org/resource/List_of_The_Tonight_Show_with_Jay_Leno_episodes_(2000–09), http://dbpedia.org/resource/The_Jack_Benny_Program_(season_15), http://dbpedia.org/resource/List_of_Late_Night_with_Jimmy_Fallon_episodes_(2010), http://dbpedia.org/resource/Harry's_Law_(season_1), http://dbpedia.org/resource/The_Fresh_Prince_of_Bel-Air_(season_5), http://dbpedia.org/resource/List_of_The_Tonight_Show_Starring_Johnny_Carson_episodes_(1974), http://dbpedia.org/resource/List_of_The_Tonight_Show_Starring_Johnny_Carson_episodes_(1975), http://dbpedia.org/resource/List_of_The_Tonight_Show_Starring_Johnny_Carson_episodes_(1976), http://dbpedia.org/resource/List_of_The_Tonight_Show_Starring_Johnny_Carson_episodes_(1978), http://dbpedia.org/resource/List_of_The_Tonight_Show_Starring_Johnny_Carson_episodes_(1979), http://dbpedia.org/resource/List_of_The_Tonight_Show_Starring_Johnny_Carson_episodes_(1980), http://dbpedia.org/resource/List_of_The_Tonight_Show_Starring_Johnny_Carson_episodes_(1981), http://dbpedia.org/resource/List_of_The_Tonight_Show_Starring_Johnny_Carson_episodes_(1982), http://dbpedia.org/resource/List_of_The_Tonight_Show_Starring_Johnny_Carson_episodes_(1983), http://dbpedia.org/resource/List_of_The_Tonight_Show_Starring_Johnny_Carson_episodes_(1984), http://dbpedia.org/resource/List_of_The_Tonight_Show_Starring_Johnny_Carson_episodes_(1985), http://dbpedia.org/resource/List_of_The_Tonight_Show_Starring_Johnny_Carson_episodes_(1986), http://dbpedia.org/resource/List_of_The_Tonight_Show_Starring_Johnny_Carson_episodes_(1987), http://dbpedia.org/resource/List_of_The_Tonight_Show_Starring_Johnny_Carson_episodes_(1988), http://dbpedia.org/resource/List_of_The_Tonight_Show_Starring_Johnny_Carson_episodes_(1989), http://dbpedia.org/resource/List_of_The_Tonight_Show_Starring_Johnny_Carson_episodes_(1990), http://dbpedia.org/resource/List_of_The_Tonight_Show_Starring_Johnny_Carson_episodes_(1991), http://dbpedia.org/resource/List_of_The_Tonight_Show_Starring_Johnny_Carson_episodes_(1992), http://dbpedia.org/resource/List_of_Late_Night_with_Jimmy_Fallon_episodes_(2009), http://dbpedia.org/resource/List_of_Late_Night_with_Jimmy_Fallon_episodes_(2011), http://dbpedia.org/resource/List_of_The_Tonight_Show_Starring_Jimmy_Fallon_episodes_(2014), http://dbpedia.org/resource/List_of_The_Tonight_Show_Starring_Jimmy_Fallon_episodes_(2016), http://dbpedia.org/resource/Dateline_NBC, http://dbpedia.org/resource/I_Dream_of_Jeannie, http://dbpedia.org/resource/Kings_(U.S._TV_series), http://dbpedia.org/resource/Medium_(TV_series), http://dbpedia.org/resource/Miami_Vice, http://dbpedia.org/resource/SeaQuest_DSV, http://dbpedia.org/resource/The_Celebrity_Apprentice, http://dbpedia.org/resource/The_Incredible_Hulk_(1978_TV_series), http://dbpedia.org/resource/The_Tonight_Show, http://dbpedia.org/resource/The_Tonight_Show_with_Jay_Leno, http://dbpedia.org/resource/Wheel_of_Fortune_(U.S._game_show), http://dbpedia.org/resource/Bionic_Woman_(2007_TV_series), http://dbpedia.org/resource/Homicide:_Life_on_the_Street_(season_6), http://dbpedia.org/resource/L.A._Law, http://dbpedia.org/resource/Law_&_Order:_Special_Victims_Unit, http://dbpedia.org/resource/Law_&_Order:_Special_Victims_Unit_(season_8), http://dbpedia.org/resource/Love,_Lies_and_Murder, http://dbpedia.org/resource/Misfits_of_Science, http://dbpedia.org/resource/Taxi_(TV_series), http://dbpedia.org/resource/The_Bionic_Woman, http://dbpedia.org/resource/The_Year_Without_a_Santa_Claus_(2006_film), http://dbpedia.org/resource/Earth_2_(TV_series), http://dbpedia.org/resource/Riverboat_(TV_series), http://dbpedia.org/resource/Underdog_(TV_series), http://dbpedia.org/resource/The_Apprentice_(TV_series), http://dbpedia.org/resource/Windfall_(TV_series), http://dbpedia.org/resource/Ellery_Queen_(TV_series), http://dbpedia.org/resource/Seinfeld_(season_1), http://dbpedia.org/resource/Seinfeld_(season_2), http://dbpedia.org/resource/Seinfeld_(season_3), http://dbpedia.org/resource/Seinfeld_(season_4), http://dbpedia.org/resource/Seinfeld_(season_5), http://dbpedia.org/resource/Seinfeld_(season_6), http://dbpedia.org/resource/Seinfeld_(season_7), http://dbpedia.org/resource/Seinfeld_(season_8), http://dbpedia.org/resource/Seinfeld_(season_9), http://dbpedia.org/resource/The_John_Larroquette_Show, http://dbpedia.org/resource/The_Michael_Richards_Show, http://dbpedia.org/resource/Watching_Ellie, http://dbpedia.org/resource/Backstairs_at_the_White_House, http://dbpedia.org/resource/Love_in_the_Wild, http://dbpedia.org/resource/The_Contender_(TV_series), http://dbpedia.org/resource/Choose_Up_Sides, http://dbpedia.org/resource/Dough_Re_Mi, http://dbpedia.org/resource/Make_the_Connection, http://dbpedia.org/resource/Sale_of_the_Century_(U.S._game_show), http://dbpedia.org/resource/The_Two_Mrs._Grenvilles, http://dbpedia.org/resource/3rd_Rock_from_the_Sun_(season_1), http://dbpedia.org/resource/3rd_Rock_from_the_Sun_(season_5), http://dbpedia.org/resource/America's_Most_Talented_Kid, http://dbpedia.org/resource/From_Here_to_Eternity_(miniseries), http://dbpedia.org/resource/I'll_Fly_Away_(TV_series), http://dbpedia.org/resource/I_Married_Joan, http://dbpedia.org/resource/Something_So_Right_(TV_series), http://dbpedia.org/resource/Bare_Essence, http://dbpedia.org/resource/C.P.O._Sharkey, http://dbpedia.org/resource/Freaks_and_Geeks, http://dbpedia.org/resource/Harper_Valley_PTA_(TV_series), http://dbpedia.org/resource/Heist_(TV_series), http://dbpedia.org/resource/Kath_&_Kim_(U.S._TV_series), http://dbpedia.org/resource/Lovers_and_Friends, http://dbpedia.org/resource/My_Hero_(U.S._TV_series), http://dbpedia.org/resource/Needles_and_Pins_(TV_series), http://dbpedia.org/resource/Rescue_from_Gilligan's_Island, http://dbpedia.org/resource/Saturday_Night_Live_(season_21), http://dbpedia.org/resource/Saturday_Night_Live_(season_36), http://dbpedia.org/resource/Saturday_Night_Live_(season_6), http://dbpedia.org/resource/Sirota's_Court, http://dbpedia.org/resource/Teachers_(2006_U.S._TV_series), http://dbpedia.org/resource/The_Bill_Cosby_Show, http://dbpedia.org/resource/The_Castaways_on_Gilligan's_Island, http://dbpedia.org/resource/The_Duck_Factory, http://dbpedia.org/resource/The_Others_(TV_series), http://dbpedia.org/resource/The_Pursuit_of_Happiness_(TV_series), http://dbpedia.org/resource/Twenty_Good_Years, http://dbpedia.org/resource/My_Name_Is_Earl_(season_1), http://dbpedia.org/resource/Raines, http://dbpedia.org/resource/Journeyman_(TV_series), http://dbpedia.org/resource/List_of_The_Tonight_Show_Starring_Johnny_Carson_episodes_(1977), http://dbpedia.org/resource/Born_Innocent_(film), http://dbpedia.org/resource/Polly_(1989_film), http://dbpedia.org/resource/How_to_Break_Up_a_Happy_Divorce, http://dbpedia.org/resource/Leave_It_to_the_Girls, http://dbpedia.org/resource/The_Greatest_Gift_(film), http://dbpedia.org/resource/America's_Next_Great_Restaurant, http://dbpedia.org/resource/A_Cry_for_Help:_The_Tracey_Thurman_Story, http://dbpedia.org/resource/Columbo_(season_1), http://dbpedia.org/resource/Columbo_(season_6), http://dbpedia.org/resource/Columbo_(season_7), http://dbpedia.org/resource/The_Hero_(1966_TV_series), http://dbpedia.org/resource/100_Questions, http://dbpedia.org/resource/Face_the_Ace, http://dbpedia.org/resource/Hank_(1965_TV_series), http://dbpedia.org/resource/Hawaii_(TV_series), http://dbpedia.org/resource/Inconceivable, http://dbpedia.org/resource/Karen_(1964_TV_series), http://dbpedia.org/resource/Meet_My_Folks, http://dbpedia.org/resource/Mr._Rhodes, http://dbpedia.org/resource/Pride_&_Joy_(TV_series), http://dbpedia.org/resource/The_Great_American_Road_Trip, http://dbpedia.org/resource/The_Night_They_Took_Miss_Beautiful, http://dbpedia.org/resource/Punch_and_Jody, http://dbpedia.org/resource/Somerset_Maugham_TV_Theatre, http://dbpedia.org/resource/Undercovers_(TV_series), http://dbpedia.org/resource/Growing_Up_Brady_(film), http://dbpedia.org/resource/How_to_Eat_Like_a_Child, http://dbpedia.org/resource/In_the_Beginning_(miniseries), http://dbpedia.org/resource/Homicide:_Life_on_the_Street_(season_1), http://dbpedia.org/resource/Homicide:_Life_on_the_Street_(season_2), http://dbpedia.org/resource/Homicide:_Life_on_the_Street_(season_3), http://dbpedia.org/resource/Homicide:_Life_on_the_Street_(season_4), http://dbpedia.org/resource/Mindreaders, http://dbpedia.org/resource/Crash:_The_Mystery_of_Flight_1501, http://dbpedia.org/resource/My_Dad_Is_Better_Than_Your_Dad, http://dbpedia.org/resource/Doomsday_Virus, http://dbpedia.org/resource/Law_&_Order:_Criminal_Intent_(season_5), http://dbpedia.org/resource/Sportsworld_(U.S._TV_series), http://dbpedia.org/resource/The_Home_Court, http://dbpedia.org/resource/Parks_and_Recreation_(season_4), http://dbpedia.org/resource/Parks_and_Recreation_(season_6), http://dbpedia.org/resource/In_the_Loop_with_iVillage, http://dbpedia.org/resource/Law_&_Order:_Criminal_Intent_(season_1), http://dbpedia.org/resource/Law_&_Order:_Criminal_Intent_(season_2), http://dbpedia.org/resource/Law_&_Order:_Criminal_Intent_(season_4), http://dbpedia.org/resource/Law_&_Order:_Criminal_Intent_(season_6), http://dbpedia.org/resource/Saturday_Night_Live_(season_14), http://dbpedia.org/resource/Saturday_Night_Live_(season_15), http://dbpedia.org/resource/Saturday_Night_Live_(season_17), http://dbpedia.org/resource/Saturday_Night_Live_(season_22), http://dbpedia.org/resource/Saturday_Night_Live_(season_25), http://dbpedia.org/resource/Saturday_Night_Live_(season_33), http://dbpedia.org/resource/Saturday_Night_Live_(season_41), http://dbpedia.org/resource/Saturday_Night_Live_(season_8), http://dbpedia.org/resource/Community_(season_1), http://dbpedia.org/resource/Eyes_of_Terror, http://dbpedia.org/resource/Stonestreet:_Who_Killed_the_Centerfold_Model%3F, http://dbpedia.org/resource/The_West_Wing_(season_3), http://dbpedia.org/resource/Money_Talks_News, http://dbpedia.org/resource/Lost_(game_show), http://dbpedia.org/resource/The_Biggest_Loser:_Battle_of_the_Ages, http://dbpedia.org/resource/The_Biggest_Loser:_Challenge_America, http://dbpedia.org/resource/The_Biggest_Loser:_Pay_It_Forward, http://dbpedia.org/resource/The_Biggest_Loser:_Second_Chances, http://dbpedia.org/resource/The_Biggest_Loser_(season_1), http://dbpedia.org/resource/The_Biggest_Loser_(season_2), http://dbpedia.org/resource/The_Biggest_Loser_(season_4), http://dbpedia.org/resource/Harry's_Law_(season_2), http://dbpedia.org/resource/Grimm_(season_3), http://dbpedia.org/resource/Community_(season_4), http://dbpedia.org/resource/Boss_Lady, http://dbpedia.org/resource/Frasier_(season_7), http://dbpedia.org/resource/Casper's_Halloween_Special, http://dbpedia.org/resource/Son-Rise:_A_Miracle_of_Love, http://dbpedia.org/resource/The_Golden_Girls_(season_7), http://dbpedia.org/resource/Quantum_Leap_(season_1), http://dbpedia.org/resource/Quantum_Leap_(season_4), http://dbpedia.org/resource/Three_Wishes_(U.S._TV_series), http://dbpedia.org/resource/List_of_Late_Night_with_Jimmy_Fallon_episodes_(2013–14), http://dbpedia.org/resource/List_of_The_Tonight_Show_Starring_Jimmy_Fallon_episodes_(2015), http://dbpedia.org/resource/List_of_Late_Night_with_Seth_Meyers_episodes_(2014), http://dbpedia.org/resource/List_of_Late_Night_with_Seth_Meyers_episodes_(2016), http://dbpedia.org/resource/30_Rock, http://dbpedia.org/resource/A.D._The_Bible_Continues, http://dbpedia.org/resource/ALF:_The_Animated_Series, http://dbpedia.org/resource/Another_World_(TV_series), http://dbpedia.org/resource/Candid_Camera, http://dbpedia.org/resource/Days_of_Our_Lives, http://dbpedia.org/resource/Hallmark_Hall_of_Fame, http://dbpedia.org/resource/Heidi_(1968_film), http://dbpedia.org/resource/Law_&_Order, http://dbpedia.org/resource/Law_&_Order_(season_5), http://dbpedia.org/resource/Meet_the_Press, http://dbpedia.org/resource/Seinfeld, http://dbpedia.org/resource/Treasure_Hunters_(TV_series), http://dbpedia.org/resource/Wild_Texas_Wind, http://dbpedia.org/resource/Amazing_Stories_(TV_series), http://dbpedia.org/resource/Dr._Kildare_(TV_series), http://dbpedia.org/resource/J.J._Starbuck, http://dbpedia.org/resource/Land_of_the_Lost_(1974_TV_series), http://dbpedia.org/resource/Late_Night_with_Jimmy_Fallon, http://dbpedia.org/resource/Man_of_the_People_(TV_series), http://dbpedia.org/resource/Nashville_Star, http://dbpedia.org/resource/The_Danny_Thomas_Hour, http://dbpedia.org/resource/The_Death_of_the_Incredible_Hulk, http://dbpedia.org/resource/The_Incredible_Hulk_Returns, http://dbpedia.org/resource/The_Lieutenant, http://dbpedia.org/resource/The_Magician_(U.S._TV_series), http://dbpedia.org/resource/The_Oregon_Trail_(TV_series), http://dbpedia.org/resource/The_Trial_of_the_Incredible_Hulk, http://dbpedia.org/resource/You_Bet_Your_Life, http://dbpedia.org/resource/A_Killer_Among_Us, http://dbpedia.org/resource/Bitter_Harvest_(1981_film), http://dbpedia.org/resource/CHiPs, http://dbpedia.org/resource/Centennial_(miniseries), http://dbpedia.org/resource/Ed_(TV_series), http://dbpedia.org/resource/Friday_Night_Lights_(TV_series), http://dbpedia.org/resource/Hollywood_Squares, http://dbpedia.org/resource/Kidnapped_(TV_series), http://dbpedia.org/resource/Late_Night_(NBC), http://dbpedia.org/resource/NASCAR_on_NBC, http://dbpedia.org/resource/Police_Story_(TV_series), http://dbpedia.org/resource/The_A-Team, http://dbpedia.org/resource/The_David_Letterman_Show, http://dbpedia.org/resource/The_Dick_Powell_Show, http://dbpedia.org/resource/The_Gong_Show, http://dbpedia.org/resource/The_Tonight_Show_with_Conan_O'Brien, http://dbpedia.org/resource/Babes_in_Toyland_(1986_film), http://dbpedia.org/resource/Caroline_in_the_City, http://dbpedia.org/resource/Cocaine:_One_Man's_Seduction, http://dbpedia.org/resource/David_Copperfield_(1993_film), http://dbpedia.org/resource/Elf:_Buddy's_Musical_Christmas, http://dbpedia.org/resource/Friday_Night_Videos, http://dbpedia.org/resource/Hollywood_Game_Night, http://dbpedia.org/resource/Homicide:_Life_on_the_Street_(season_5), http://dbpedia.org/resource/How_Murray_Saved_Christmas, http://dbpedia.org/resource/Light's_Diamond_Jubilee, http://dbpedia.org/resource/McMillan_&_Wife, http://dbpedia.org/resource/Mr._Krueger's_Christmas, http://dbpedia.org/resource/My_World_and_Welcome_to_It, http://dbpedia.org/resource/Passions, http://dbpedia.org/resource/Phenomenon_(TV_series), http://dbpedia.org/resource/Poker_After_Dark, http://dbpedia.org/resource/Providence_(TV_series), http://dbpedia.org/resource/Saturday_Night_Live_(season_7), http://dbpedia.org/resource/Saved_by_the_Bell, http://dbpedia.org/resource/Shannon's_Deal, http://dbpedia.org/resource/The_Family_Tree, http://dbpedia.org/resource/The_Jimmy_Stewart_Show, http://dbpedia.org/resource/The_Magic_Clown, http://dbpedia.org/resource/The_New_Fantastic_Four, http://dbpedia.org/resource/The_Office_(U.S._TV_series), http://dbpedia.org/resource/The_Office_(U.S._season_7), http://dbpedia.org/resource/Thundarr_the_Barbarian, http://dbpedia.org/resource/Tonight_Starring_Jack_Paar, http://dbpedia.org/resource/Veronica's_Closet, http://dbpedia.org/resource/Death_of_a_Centerfold:_The_Dorothy_Stratten_Story, http://dbpedia.org/resource/Ensign_O'Toole, http://dbpedia.org/resource/Grandpa_Goes_to_Washington, http://dbpedia.org/resource/Poison_Ivy_(1985_film), http://dbpedia.org/resource/The_Harlem_Globetrotters_on_Gilligan's_Island, http://dbpedia.org/resource/The_Restless_Gun, http://dbpedia.org/resource/The_Black_Donnellys, http://dbpedia.org/resource/Law_&_Order:_Special_Victims_Unit_(season_14), http://dbpedia.org/resource/Undateable, http://dbpedia.org/resource/Fame_Is_the_Name_of_the_Game, http://dbpedia.org/resource/Saturday_Night_Live_(season_28), http://dbpedia.org/resource/Saturday_Night_Live_(season_29), http://dbpedia.org/resource/The_Incredible_Hulk_(1982_TV_series), http://dbpedia.org/resource/The_Odyssey_(miniseries), http://dbpedia.org/resource/Brand_New_Life_(TV_series), http://dbpedia.org/resource/Everything's_Relative_(1999_TV_series), http://dbpedia.org/resource/30_Rock_(season_4), http://dbpedia.org/resource/Snap_Judgment_(game_show), http://dbpedia.org/resource/Will_&_Grace_(season_1), http://dbpedia.org/resource/Will_&_Grace_(season_2), http://dbpedia.org/resource/Will_&_Grace_(season_3), http://dbpedia.org/resource/Will_&_Grace_(season_4), http://dbpedia.org/resource/Will_&_Grace_(season_5), http://dbpedia.org/resource/Will_&_Grace_(season_6), http://dbpedia.org/resource/Will_&_Grace_(season_7), http://dbpedia.org/resource/Will_&_Grace_(season_8), http://dbpedia.org/resource/Child_Bride_of_Short_Creek, http://dbpedia.org/resource/Sarah_T._–_Portrait_of_a_Teenage_Alcoholic, http://dbpedia.org/resource/The_Rockford_Files_(season_3), http://dbpedia.org/resource/Star_Trek:_The_Original_Series_(season_1), http://dbpedia.org/resource/Star_Trek:_The_Original_Series_(season_2), http://dbpedia.org/resource/The_Kid_with_the_200_I.Q., http://dbpedia.org/resource/The_Possessed_(1977_film), http://dbpedia.org/resource/One_Happy_Family, http://dbpedia.org/resource/Say_When!!, http://dbpedia.org/resource/Space_Cats, http://dbpedia.org/resource/The_Red_Pony_(1973_film), http://dbpedia.org/resource/Mama's_Boy_(TV_series), http://dbpedia.org/resource/Crusoe_(TV_series), http://dbpedia.org/resource/She_Cried_No, http://dbpedia.org/resource/30_Rock_(season_6), http://dbpedia.org/resource/The_Little_Rascals_Christmas_Special, http://dbpedia.org/resource/The_Rockford_Files_(season_1), http://dbpedia.org/resource/Meteor_(miniseries), http://dbpedia.org/resource/Rosetti_and_Ryan, http://dbpedia.org/resource/Who's_Watching_the_Kids%3F, http://dbpedia.org/resource/Diana:_Her_True_Story, http://dbpedia.org/resource/Great_Ghost_Tales, http://dbpedia.org/resource/The_Rockford_Files_(season_4), http://dbpedia.org/resource/The_A-Team_(season_4), http://dbpedia.org/resource/Norby_(TV_series), http://dbpedia.org/resource/The_Armed_Forces_Hour, http://dbpedia.org/resource/Friday_Night_Lights_(season_4), http://dbpedia.org/resource/My_Name_Is_Earl_(season_3), http://dbpedia.org/resource/List_of_The_Tonight_Show_with_Jay_Leno_episodes_(1992–99), http://dbpedia.org/resource/List_of_The_Tonight_Show_with_Jay_Leno_episodes_(2010), http://dbpedia.org/resource/The_Winds_of_Whoopie, http://dbpedia.org/resource/Cheers_(season_10), http://dbpedia.org/resource/The_Biggest_Loser_(Dutch_TV_series), http://dbpedia.org/resource/Baywatch, http://dbpedia.org/resource/Cheers, http://dbpedia.org/resource/Fear_Factor, http://dbpedia.org/resource/Gambit_(game_show), http://dbpedia.org/resource/Major_League_Baseball_on_NBC, http://dbpedia.org/resource/NBC_Nightly_News, http://dbpedia.org/resource/The_Baseball_Network, http://dbpedia.org/resource/The_Fresh_Prince_of_Bel-Air, http://dbpedia.org/resource/The_Revenge_of_Al_Capone, http://dbpedia.org/resource/The_Tonight_Show_Starring_Johnny_Carson, http://dbpedia.org/resource/The_West_Wing, http://dbpedia.org/resource/This_Is_Your_Life, http://dbpedia.org/resource/Today_(U.S._TV_program), http://dbpedia.org/resource/Will_&_Grace, http://dbpedia.org/resource/Your_Show_of_Shows, http://dbpedia.org/resource/A_Different_World, http://dbpedia.org/resource/Hannibal_(TV_series), http://dbpedia.org/resource/Law_&_Order:_Special_Victims_Unit_(season_13), http://dbpedia.org/resource/Let's_Make_a_Deal, http://dbpedia.org/resource/My_Name_Is_Earl, http://dbpedia.org/resource/Night_Court, http://dbpedia.org/resource/Star_Trek:_The_Animated_Series, http://dbpedia.org/resource/The_Price_Is_Right_(1956_U.S._game_show), http://dbpedia.org/resource/This_Is_Show_Business, http://dbpedia.org/resource/Truth_or_Consequences, http://dbpedia.org/resource/Twenty_One_(game_show), http://dbpedia.org/resource/Unsolved_Mysteries, http://dbpedia.org/resource/The_Hobbit_(1977_film), http://dbpedia.org/resource/The_Murder_of_Mary_Phagan, http://dbpedia.org/resource/Bates_Motel_(film), http://dbpedia.org/resource/Dick_Clark's_New_Year's_Rockin'_Eve, http://dbpedia.org/resource/Law_of_the_Plainsman, http://dbpedia.org/resource/Lipstick_Jungle_(TV_series), http://dbpedia.org/resource/Scared_Shrekless, http://dbpedia.org/resource/Gavilan_(TV_series), http://dbpedia.org/resource/Name_That_Tune, http://dbpedia.org/resource/Profiler_(TV_series), http://dbpedia.org/resource/American_Dreams, http://dbpedia.org/resource/Crusader_Rabbit, http://dbpedia.org/resource/Dante_(TV_series), http://dbpedia.org/resource/King_Leonardo_and_His_Short_Subjects, http://dbpedia.org/resource/Meet_McGraw, http://dbpedia.org/resource/The_Loneliest_Runner, http://dbpedia.org/resource/Laura_Lansing_Slept_Here, http://dbpedia.org/resource/One_Christmas_(film), http://dbpedia.org/resource/Parenthood_(1990_TV_series), http://dbpedia.org/resource/The_Rebels_(TV_series), http://dbpedia.org/resource/The_Troubleshooters_(1959_TV_series), http://dbpedia.org/resource/Law_&_Order:_Special_Victims_Unit_(season_1), http://dbpedia.org/resource/Law_&_Order:_Special_Victims_Unit_(season_2), http://dbpedia.org/resource/Sunday_Night_(American_TV_program), http://dbpedia.org/resource/CB_Bears, http://dbpedia.org/resource/Casper_and_the_Angels, http://dbpedia.org/resource/Foofur, http://dbpedia.org/resource/Going_Bananas_(U.S._series), http://dbpedia.org/resource/Hey,_Landlord, http://dbpedia.org/resource/Jokebook, http://dbpedia.org/resource/Match_Game-Hollywood_Squares_Hour, http://dbpedia.org/resource/Shirley_(TV_series), http://dbpedia.org/resource/Shirt_Tales, http://dbpedia.org/resource/Snorks, http://dbpedia.org/resource/Space_Stars, http://dbpedia.org/resource/The_Space_Kidettes, http://dbpedia.org/resource/Wheelie_and_the_Chopper_Bunch, http://dbpedia.org/resource/Yo_Yogi!, http://dbpedia.org/resource/Yogi's_Space_Race, http://dbpedia.org/resource/Danielle_Steel's_Star, http://dbpedia.org/resource/Family_Album_(miniseries), http://dbpedia.org/resource/Mister_Sterling, http://dbpedia.org/resource/Perry_Mason_(TV_movies), http://dbpedia.org/resource/Saturday_Night's_Main_Event, http://dbpedia.org/resource/Saturday_Night_Live_(season_9), http://dbpedia.org/resource/Tate_(TV_series), http://dbpedia.org/resource/The_Firm_(2012_TV_series), http://dbpedia.org/resource/Michael_Shayne, http://dbpedia.org/resource/My_Sweet_Charlie, http://dbpedia.org/resource/The_Californians_(TV_series), http://dbpedia.org/resource/For_Your_Love_(TV_series), http://dbpedia.org/resource/Tennis_on_NBC, http://dbpedia.org/resource/The_Gangster_Chronicles, http://dbpedia.org/resource/The_Great_Escape_II:_The_Untold_Story, http://dbpedia.org/resource/The_Temptations_(miniseries), http://dbpedia.org/resource/A_Year_in_the_Life, http://dbpedia.org/resource/Blind_Date_(1950s_game_show), http://dbpedia.org/resource/Brotherly_Love_(1995_TV_series), http://dbpedia.org/resource/Fall_into_Darkness, http://dbpedia.org/resource/Hidden_Faces, http://dbpedia.org/resource/How_to_Survive_a_Marriage, http://dbpedia.org/resource/My_Friend_Tony, http://dbpedia.org/resource/Our_Five_Daughters, http://dbpedia.org/resource/Saber_of_London, http://dbpedia.org/resource/Sister,_Sister_(1982_film), http://dbpedia.org/resource/The_Bold_Ones:_The_Senator, http://dbpedia.org/resource/The_Brighter_Day, http://dbpedia.org/resource/The_John_Forsythe_Show, http://dbpedia.org/resource/The_Round_Table_(TV_series), http://dbpedia.org/resource/The_Sophisticated_Gents, http://dbpedia.org/resource/A_Reason_to_Live_(1985_film), http://dbpedia.org/resource/Little_House_on_the_Prairie_(film), http://dbpedia.org/resource/Law_&_Order:_Special_Victims_Unit_(season_11), http://dbpedia.org/resource/Law_&_Order:_Special_Victims_Unit_(season_12), http://dbpedia.org/resource/Law_&_Order:_Special_Victims_Unit_(season_15), http://dbpedia.org/resource/Law_&_Order:_Special_Victims_Unit_(season_16), http://dbpedia.org/resource/Law_&_Order:_Special_Victims_Unit_(season_17), http://dbpedia.org/resource/Law_&_Order:_Special_Victims_Unit_(season_3), http://dbpedia.org/resource/Law_&_Order:_Special_Victims_Unit_(season_4), http://dbpedia.org/resource/Law_&_Order:_Special_Victims_Unit_(season_5), http://dbpedia.org/resource/Law_&_Order:_Special_Victims_Unit_(season_6), http://dbpedia.org/resource/Andy_Barker,_P.I., http://dbpedia.org/resource/Saturday_Night_Live_(season_26), http://dbpedia.org/resource/Saturday_Night_Live_(season_27), http://dbpedia.org/resource/Saturday_Night_Live_(season_30), http://dbpedia.org/resource/Scrubs_(season_1), http://dbpedia.org/resource/Scrubs_(season_2), http://dbpedia.org/resource/Scrubs_(season_3), http://dbpedia.org/resource/Scrubs_(season_5), http://dbpedia.org/resource/Scrubs_(season_6), http://dbpedia.org/resource/Scrubs_(season_7), http://dbpedia.org/resource/The_Golden_Girls_(season_2), http://dbpedia.org/resource/Saturday_Night_Live_(season_38), http://dbpedia.org/resource/The_Berenstain_Bears_Meet_Bigpaw, http://dbpedia.org/resource/ALF_Tales, http://dbpedia.org/resource/Private_Eye_(TV_series), http://dbpedia.org/resource/Age_of_Love_(TV_series), http://dbpedia.org/resource/Clash_of_the_Choirs, http://dbpedia.org/resource/A.U.S.A., http://dbpedia.org/resource/Frasier_(season_5), http://dbpedia.org/resource/The_Gambler_(film_series), http://dbpedia.org/resource/The_Jake_Effect, http://dbpedia.org/resource/The_Second_Half, http://dbpedia.org/resource/Crisis_Center, http://dbpedia.org/resource/PDQ_(game_show), http://dbpedia.org/resource/What's_This_Song%3F, http://dbpedia.org/resource/The_Stepford_Children, http://dbpedia.org/resource/Built_to_Last_(TV_series), http://dbpedia.org/resource/House_Rules_(1998_TV_series), http://dbpedia.org/resource/Mr._and_Mrs._North, http://dbpedia.org/resource/Black_Tie_Affair_(TV_series), http://dbpedia.org/resource/Chicago_Sons, http://dbpedia.org/resource/The_World_of_Mr._Sweeney, http://dbpedia.org/resource/Friday_Night_Lights_(season_1), http://dbpedia.org/resource/Snip_(TV_series), http://dbpedia.org/resource/Her_Hidden_Truth, http://dbpedia.org/resource/If_Someone_Had_Known, http://dbpedia.org/resource/Police_Story:_Confessions_of_a_Lady_Cop, http://dbpedia.org/resource/Parks_and_Recreation_(season_1), http://dbpedia.org/resource/Parks_and_Recreation_(season_3), http://dbpedia.org/resource/Parks_and_Recreation_(season_5), http://dbpedia.org/resource/The_West_Wing_(season_6), http://dbpedia.org/resource/The_Biggest_Loser:_Glory_Days, http://dbpedia.org/resource/Valley_Forge_(film), http://dbpedia.org/resource/The_Fresh_Prince_of_Bel-Air_(season_4), http://dbpedia.org/resource/Rewind_(2013_film), http://dbpedia.org/resource/Community_(season_5), http://dbpedia.org/resource/A_Burning_Passion:_The_Margaret_Mitchell_Story, http://dbpedia.org/resource/The_Crisis_(TV_series), http://dbpedia.org/resource/In_Wine_Country, http://dbpedia.org/resource/Friday_Night_Lights_(season_3), http://dbpedia.org/resource/People_Will_Talk_(game_show), http://dbpedia.org/resource/The_Office:_The_Accountants, http://dbpedia.org/resource/The_Swift_Show, http://dbpedia.org/resource/Frasier_(season_11), http://dbpedia.org/resource/The_Blacklist_(season_3), http://dbpedia.org/resource/Chain_Letter_(U.S._game_show), http://dbpedia.org/resource/3rd_Rock_from_the_Sun_(season_3), http://dbpedia.org/resource/Just_Shoot_Me!_(season_1), http://dbpedia.org/resource/Just_Shoot_Me!_(season_2), http://dbpedia.org/resource/Just_Shoot_Me!_(season_3), http://dbpedia.org/resource/Just_Shoot_Me!_(season_4), http://dbpedia.org/resource/Just_Shoot_Me!_(season_5), http://dbpedia.org/resource/Just_Shoot_Me!_(season_6), http://dbpedia.org/resource/Just_Shoot_Me!_(season_7), http://dbpedia.org/resource/The_Rockford_Files_(season_5), http://dbpedia.org/resource/The_Rockford_Files_(season_6), http://dbpedia.org/resource/All_Star_Secrets, http://dbpedia.org/resource/Columbo, http://dbpedia.org/resource/Crossing_Jordan, http://dbpedia.org/resource/Cruel_Doubt, http://dbpedia.org/resource/ER_(TV_series), http://dbpedia.org/resource/Empire_(1962_TV_series), http://dbpedia.org/resource/Football_Night_in_America, http://dbpedia.org/resource/Frasier, http://dbpedia.org/resource/Generations_(U.S._TV_series), http://dbpedia.org/resource/Golf_Channel_on_NBC, http://dbpedia.org/resource/Hill_Street_Blues, http://dbpedia.org/resource/Homicide:_Life_on_the_Street, http://dbpedia.org/resource/Las_Vegas_(TV_series), http://dbpedia.org/resource/Late_Night_with_Conan_O'Brien, http://dbpedia.org/resource/Late_Night_with_David_Letterman, http://dbpedia.org/resource/Later_(talk_show), http://dbpedia.org/resource/Matlock_(TV_series), http://dbpedia.org/resource/NBA_on_NBC, http://dbpedia.org/resource/NBC_Sunday_Night_Football, http://dbpedia.org/resource/NFL_on_NBC, http://dbpedia.org/resource/NHL_on_NBC, http://dbpedia.org/resource/NewsRadio, http://dbpedia.org/resource/Olympics_on_NBC, http://dbpedia.org/resource/Quantum_Leap, http://dbpedia.org/resource/Rock_Center_with_Brian_Williams, http://dbpedia.org/resource/Second_City_Television, http://dbpedia.org/resource/Sunshine_(U.S._TV_series), http://dbpedia.org/resource/Tattingers, http://dbpedia.org/resource/The_Rockford_Files, http://dbpedia.org/resource/Thursday_Night_Football, http://dbpedia.org/resource/Up_All_Night_(TV_series), http://dbpedia.org/resource/A_Man_and_His_Music_+_Ella_+_Jobim, http://dbpedia.org/resource/Baa_Baa_Black_Sheep_(TV_series), http://dbpedia.org/resource/Frank_Sinatra:_A_Man_and_His_Music, http://dbpedia.org/resource/Friends_(season_1), http://dbpedia.org/resource/Friends_(season_10), http://dbpedia.org/resource/Friends_(season_2), http://dbpedia.org/resource/Friends_(season_3), http://dbpedia.org/resource/Friends_(season_4), http://dbpedia.org/resource/Friends_(season_5), http://dbpedia.org/resource/Friends_(season_6), http://dbpedia.org/resource/Friends_(season_7), http://dbpedia.org/resource/Friends_(season_8), http://dbpedia.org/resource/Friends_(season_9), http://dbpedia.org/resource/Get_Smart, http://dbpedia.org/resource/Grimm_(season_4), http://dbpedia.org/resource/Joey_(TV_series), http://dbpedia.org/resource/Mad_About_You, http://dbpedia.org/resource/Wichita_Town, http://dbpedia.org/resource/Wings_(1990_TV_series), http://dbpedia.org/resource/Cartoon_All-Stars_to_the_Rescue, http://dbpedia.org/resource/Just_Shoot_Me!, http://dbpedia.org/resource/Law_&_Order:_Criminal_Intent, http://dbpedia.org/resource/The_New_Adventures_of_Huckleberry_Finn, http://dbpedia.org/resource/A_Muppets_Christmas:_Letters_to_Santa, http://dbpedia.org/resource/America's_Got_Talent, http://dbpedia.org/resource/Captain_N:_The_Game_Master, http://dbpedia.org/resource/Chuck_(TV_series), http://dbpedia.org/resource/ER_(season_3), http://dbpedia.org/resource/Emergency!, http://dbpedia.org/resource/Fired_Up_(TV_series), http://dbpedia.org/resource/Hunter_(U.S._TV_series), http://dbpedia.org/resource/It's_a_Very_Merry_Muppet_Christmas_Movie, http://dbpedia.org/resource/Lucas_Tanner, http://dbpedia.org/resource/Macy's_Thanksgiving_Day_Parade, http://dbpedia.org/resource/Man_from_Atlantis, http://dbpedia.org/resource/Mickey's_60th_Birthday, http://dbpedia.org/resource/Police_Woman_(TV_series), http://dbpedia.org/resource/Sisters_(TV_series), http://dbpedia.org/resource/Stark_Raving_Mad_(TV_series), http://dbpedia.org/resource/The_Bold_Ones:_The_New_Doctors, http://dbpedia.org/resource/The_Highwayman_(TV_series), http://dbpedia.org/resource/Third_Watch, http://dbpedia.org/resource/Westwind_(TV_series), http://dbpedia.org/resource/1st_Look, http://dbpedia.org/resource/3-2-1_Penguins!, http://dbpedia.org/resource/A_Man_and_His_Music_–_Part_II, http://dbpedia.org/resource/Adam-12, http://dbpedia.org/resource/Buffalo_Bill_(TV_series), http://dbpedia.org/resource/California_Dreams, http://dbpedia.org/resource/Card_Sharks, http://dbpedia.org/resource/City_Guys, http://dbpedia.org/resource/Concentration_(game_show), http://dbpedia.org/resource/Deal_or_No_Deal_(U.S._game_show), http://dbpedia.org/resource/Early_Today, http://dbpedia.org/resource/Endurance_(TV_series), http://dbpedia.org/resource/Got_Talent, http://dbpedia.org/resource/H.R._Pufnstuf, http://dbpedia.org/resource/Hang_Time_(TV_series), http://dbpedia.org/resource/High_Rollers, http://dbpedia.org/resource/Highway_to_Heaven, http://dbpedia.org/resource/Huntley-Brinkley_Report, http://dbpedia.org/resource/In_the_Heat_of_the_Night_(TV_series), http://dbpedia.org/resource/Knight_Rider_(2008_TV_series), http://dbpedia.org/resource/Leeza, http://dbpedia.org/resource/Minute_to_Win_It, http://dbpedia.org/resource/NBC_News_at_Sunrise, http://dbpedia.org/resource/NBC_Nightside, http://dbpedia.org/resource/Nero_Wolfe_(1981_TV_series), http://dbpedia.org/resource/Notre_Dame_Football_on_NBC, http://dbpedia.org/resource/One_World_(TV_series), http://dbpedia.org/resource/One_to_Grow_On, http://dbpedia.org/resource/Password_Plus_and_Super_Password, http://dbpedia.org/resource/Santa_Barbara_(TV_series), http://dbpedia.org/resource/Saved_by_the_Bell:_The_New_Class, http://dbpedia.org/resource/Scout's_Safari, http://dbpedia.org/resource/Somerset_(TV_series), http://dbpedia.org/resource/Sunset_Beach_(TV_series), http://dbpedia.org/resource/The_Apprentice_(U.S._TV_series), http://dbpedia.org/resource/The_Banana_Splits, http://dbpedia.org/resource/The_Biggest_Loser_(U.S._TV_series), http://dbpedia.org/resource/The_Bugaloos, http://dbpedia.org/resource/The_Flip_Wilson_Show, http://dbpedia.org/resource/The_Hogan_Family, http://dbpedia.org/resource/The_Krofft_Superstar_Hour, http://dbpedia.org/resource/The_Marriage_Ref_(U.S._TV_series), http://dbpedia.org/resource/The_Smurfs_(TV_series), http://dbpedia.org/resource/The_Waverly_Wonders, http://dbpedia.org/resource/Time_Warp_Trio, http://dbpedia.org/resource/Tomorrow_(TV_series), http://dbpedia.org/resource/Tonight_Starring_Steve_Allen, http://dbpedia.org/resource/Weekend_Today, http://dbpedia.org/resource/Fabulous_Funnies, http://dbpedia.org/resource/God,_the_Devil_and_Bob, http://dbpedia.org/resource/Star_Trek:_The_Original_Series_(season_3), http://dbpedia.org/resource/The_Kid_Super_Power_Hour_with_Shazam!, http://dbpedia.org/resource/The_New_Adventures_of_Flash_Gordon, http://dbpedia.org/resource/The_Secret_Lives_of_Waldo_Kitty, http://dbpedia.org/resource/A_Tour_of_the_White_House_with_Mrs._John_F._Kennedy, http://dbpedia.org/resource/ProStars, http://dbpedia.org/resource/Boomtown_(2002_TV_series), http://dbpedia.org/resource/Frasier_(season_6), http://dbpedia.org/resource/Knight_Rider_2000, http://dbpedia.org/resource/Team_Knight_Rider, http://dbpedia.org/resource/Evil_Roy_Slade, http://dbpedia.org/resource/Jackpot_Bowling, http://dbpedia.org/resource/San_Francisco_International_Airport_(TV_series), http://dbpedia.org/resource/The_Paul_Winchell_Show, http://dbpedia.org/resource/The_Pink_Panther_Show, http://dbpedia.org/resource/An_Evening_with_Fred_Astaire, http://dbpedia.org/resource/Choices_of_the_Heart, http://dbpedia.org/resource/Down_You_Go, http://dbpedia.org/resource/Father_Dowling_Mysteries, http://dbpedia.org/resource/Fear_Itself_(TV_series), http://dbpedia.org/resource/Godzilla_(animated_series), http://dbpedia.org/resource/Hec_Ramsey, http://dbpedia.org/resource/Monster_Squad, http://dbpedia.org/resource/Poor_Little_Rich_Girl:_The_Barbara_Hutton_Story, http://dbpedia.org/resource/Prime_Suspect_(U.S._TV_series), http://dbpedia.org/resource/The_Psychiatrist_(TV_series), http://dbpedia.org/resource/Law_&_Order:_LA, http://dbpedia.org/resource/Law_&_Order:_Trial_by_Jury, http://dbpedia.org/resource/Witness_to_the_Mob, http://dbpedia.org/resource/Camp_Candy, http://dbpedia.org/resource/Saturday_Night_Live_(season_13), http://dbpedia.org/resource/Witch's_Night_Out, http://dbpedia.org/resource/Columbo_(season_3), http://dbpedia.org/resource/Committed_(2005_TV_series), http://dbpedia.org/resource/Legends_of_the_Superheroes, http://dbpedia.org/resource/List_of_The_Alfred_Hitchcock_Hour_episodes, http://dbpedia.org/resource/Saturday_Night_Live_(season_20), http://dbpedia.org/resource/Stars_Earn_Stripes, http://dbpedia.org/resource/The_Court_of_Last_Resort, http://dbpedia.org/resource/$25_Million_Dollar_Hoax, http://dbpedia.org/resource/Cold_Feet_(U.S._TV_series), http://dbpedia.org/resource/Coupling_(U.S._TV_series), http://dbpedia.org/resource/I'm_a_Celebrity...Get_Me_Out_of_Here!_(U.S._TV_series), http://dbpedia.org/resource/Men_Behaving_Badly_(U.S._TV_series), http://dbpedia.org/resource/Return_to_the_Planet_of_the_Apes, http://dbpedia.org/resource/America's_Toughest_Jobs, http://dbpedia.org/resource/Amy_Prentiss, http://dbpedia.org/resource/Are_You_There,_Chelsea%3F, http://dbpedia.org/resource/Baby_Boom_(U.S._TV_series), http://dbpedia.org/resource/Battery_Park_(TV_series), http://dbpedia.org/resource/Blank_Check_(game_show), http://dbpedia.org/resource/Buford_and_the_Galloping_Ghost, http://dbpedia.org/resource/C._Everett_Koop,_M.D., http://dbpedia.org/resource/Campus_Hoopla, http://dbpedia.org/resource/Chicago_Story]... [INFO] P: 0.9200792602377807, R: 1.0, F: 0.9583763329893361 -[INFO] Cumulative metrics, sample: 14, P: 0.811, R: 0.929, macro F1: 0.834, macro F1*: 0.866 -[INFO] Current time: 2735 ms, average: 3773.29 ms/ques, entity detection: 775.64 ms/ques (linking tool: 667.50 ms/ques), relation detection: 998.07 ms/ques (similarity: 725.50 ms/ques), EDG generation: 307.93 ms/ques +[INFO] Cumulative metrics, sample: 14, P: 0.811, R: 0.929, macro F1: 0.866 +[INFO] Current question consumed time: 2714 ms, average: 3961.29 ms/ques, entity detection: 777.64 ms/ques (linking tool: 678.50 ms/ques), relation detection: 1012.29 ms/ques (similarity: 727.00 ms/ques), EDG generation: 511.14 ms/ques [INFO] Entity Linking P: 1.0, R: 0.9642857142857143, micro F1: 0.9761904761904762, macro F1: 0.9818181818181818 [INFO] Relation Linking P: 0.8571428571428571, R: 0.8571428571428571, micro F1: 0.8571428571428571, macro F1: 0.8571428571428571 [INFO] Type Linking P: 0.5714285714285714, R: 0.5714285714285714, micro F1: 0.5714285714285714, macro F1: 0.5714285714285714 @@ -203,8 +203,8 @@ Question 14: Is William H Blanchard buried in Colorado? [INFO] Golden Answer: [true] [INFO] Predicted Answer: [true] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 15, P: 0.824, R: 0.933, macro F1: 0.845, macro F1*: 0.875 -[INFO] Current time: 6519 ms, average: 3956.33 ms/ques, entity detection: 785.00 ms/ques (linking tool: 674.60 ms/ques), relation detection: 1240.13 ms/ques (similarity: 847.60 ms/ques), EDG generation: 288.00 ms/ques +[INFO] Cumulative metrics, sample: 15, P: 0.824, R: 0.933, macro F1: 0.875 +[INFO] Current question consumed time: 6130 ms, average: 4105.87 ms/ques, entity detection: 783.53 ms/ques (linking tool: 681.67 ms/ques), relation detection: 1237.93 ms/ques (similarity: 849.80 ms/ques), EDG generation: 477.67 ms/ques [INFO] Entity Linking P: 1.0, R: 0.9666666666666667, micro F1: 0.9777777777777777, macro F1: 0.983050847457627 [INFO] Relation Linking P: 0.8666666666666667, R: 0.8666666666666667, micro F1: 0.8666666666666667, macro F1: 0.8666666666666667 [INFO] Type Linking P: 0.6, R: 0.6, micro F1: 0.6, macro F1: 0.6 @@ -217,8 +217,8 @@ Question 15: Which cport of Vishwajyot High School is also the genere of Battle [INFO] Golden Answer: [http://dbpedia.org/resource/Chess] [INFO] Predicted Answer: [http://dbpedia.org/resource/Chess] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 16, P: 0.835, R: 0.938, macro F1: 0.855, macro F1*: 0.883 -[INFO] Current time: 5709 ms, average: 4065.88 ms/ques, entity detection: 786.94 ms/ques (linking tool: 665.38 ms/ques), relation detection: 1192.56 ms/ques (similarity: 812.94 ms/ques), EDG generation: 472.38 ms/ques +[INFO] Cumulative metrics, sample: 16, P: 0.835, R: 0.938, macro F1: 0.883 +[INFO] Current question consumed time: 2333 ms, average: 3995.06 ms/ques, entity detection: 780.56 ms/ques (linking tool: 668.44 ms/ques), relation detection: 1189.94 ms/ques (similarity: 815.12 ms/ques), EDG generation: 456.62 ms/ques [INFO] Entity Linking P: 1.0, R: 0.96875, micro F1: 0.9791666666666666, macro F1: 0.9841269841269841 [INFO] Relation Linking P: 0.875, R: 0.875, micro F1: 0.875, macro F1: 0.875 [INFO] Type Linking P: 0.625, R: 0.625, micro F1: 0.625, macro F1: 0.625 @@ -231,8 +231,8 @@ Question 16: In which state is the Channel district? [INFO] Golden Answer: [http://dbpedia.org/resource/Florida] [INFO] Predicted Answer: [http://dbpedia.org/resource/Florida, Florida@en] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 17, P: 0.845, R: 0.941, macro F1: 0.863, macro F1*: 0.890 -[INFO] Current time: 673 ms, average: 3866.29 ms/ques, entity detection: 749.29 ms/ques (linking tool: 631.12 ms/ques), relation detection: 1129.24 ms/ques (similarity: 773.00 ms/ques), EDG generation: 445.12 ms/ques +[INFO] Cumulative metrics, sample: 17, P: 0.845, R: 0.941, macro F1: 0.890 +[INFO] Current question consumed time: 622 ms, average: 3796.65 ms/ques, entity detection: 742.12 ms/ques (linking tool: 632.94 ms/ques), relation detection: 1126.53 ms/ques (similarity: 773.76 ms/ques), EDG generation: 430.29 ms/ques [INFO] Entity Linking P: 1.0, R: 0.9705882352941176, micro F1: 0.980392156862745, macro F1: 0.9850746268656716 [INFO] Relation Linking P: 0.8823529411764706, R: 0.8823529411764706, micro F1: 0.8823529411764706, macro F1: 0.8823529411764706 [INFO] Type Linking P: 0.6470588235294118, R: 0.6470588235294118, micro F1: 0.6470588235294118, macro F1: 0.6470588235294118 @@ -245,8 +245,8 @@ Question 17: Who was in the missions of Apollo 11 and Gemini 8? [INFO] Golden Answer: [http://dbpedia.org/resource/Neil_Armstrong] [INFO] Predicted Answer: [http://dbpedia.org/resource/Neil_Armstrong] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 18, P: 0.853, R: 0.944, macro F1: 0.871, macro F1*: 0.897 -[INFO] Current time: 2437 ms, average: 3786.89 ms/ques, entity detection: 731.33 ms/ques (linking tool: 611.83 ms/ques), relation detection: 1117.78 ms/ques (similarity: 762.56 ms/ques), EDG generation: 423.50 ms/ques +[INFO] Cumulative metrics, sample: 18, P: 0.853, R: 0.944, macro F1: 0.897 +[INFO] Current question consumed time: 2249 ms, average: 3710.67 ms/ques, entity detection: 722.17 ms/ques (linking tool: 611.17 ms/ques), relation detection: 1113.89 ms/ques (similarity: 764.17 ms/ques), EDG generation: 409.22 ms/ques [INFO] Entity Linking P: 1.0, R: 0.9722222222222222, micro F1: 0.9814814814814814, macro F1: 0.9859154929577464 [INFO] Relation Linking P: 0.8888888888888888, R: 0.8888888888888888, micro F1: 0.8888888888888888, macro F1: 0.8888888888888888 [INFO] Type Linking P: 0.6666666666666666, R: 0.6666666666666666, micro F1: 0.6666666666666666, macro F1: 0.6666666666666666 @@ -259,8 +259,8 @@ Question 18: What is the total number of other tenant of the stadia whose one of [INFO] Golden Answer: [16] [INFO] Predicted Answer: [16] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 19, P: 0.861, R: 0.947, macro F1: 0.878, macro F1*: 0.902 -[INFO] Current time: 1718 ms, average: 3678.00 ms/ques, entity detection: 701.16 ms/ques (linking tool: 584.00 ms/ques), relation detection: 1093.47 ms/ques (similarity: 750.68 ms/ques), EDG generation: 407.42 ms/ques +[INFO] Cumulative metrics, sample: 19, P: 0.861, R: 0.947, macro F1: 0.902 +[INFO] Current question consumed time: 1625 ms, average: 3600.89 ms/ques, entity detection: 690.79 ms/ques (linking tool: 582.32 ms/ques), relation detection: 1093.58 ms/ques (similarity: 758.16 ms/ques), EDG generation: 393.32 ms/ques [INFO] Entity Linking P: 1.0, R: 0.9736842105263158, micro F1: 0.982456140350877, macro F1: 0.9866666666666666 [INFO] Relation Linking P: 0.8947368421052632, R: 0.8947368421052632, micro F1: 0.8947368421052632, macro F1: 0.8947368421052632 [INFO] Type Linking P: 0.631578947368421, R: 0.631578947368421, micro F1: 0.631578947368421, macro F1: 0.631578947368421 @@ -273,8 +273,8 @@ Question 19: Where can one find the Dzogchen Ponolop Rinpoche? [INFO] Golden Answer: [http://dbpedia.org/resource/Seattle, http://dbpedia.org/resource/Washington_(state)] [INFO] Predicted Answer: [http://dbpedia.org/resource/Sichuan] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 20, P: 0.818, R: 0.900, macro F1: 0.834, macro F1*: 0.857 -[INFO] Current time: 924 ms, average: 3540.30 ms/ques, entity detection: 672.10 ms/ques (linking tool: 559.80 ms/ques), relation detection: 1054.90 ms/ques (similarity: 725.15 ms/ques), EDG generation: 388.10 ms/ques +[INFO] Cumulative metrics, sample: 20, P: 0.818, R: 0.900, macro F1: 0.857 +[INFO] Current question consumed time: 712 ms, average: 3456.45 ms/ques, entity detection: 661.45 ms/ques (linking tool: 557.55 ms/ques), relation detection: 1048.10 ms/ques (similarity: 726.00 ms/ques), EDG generation: 374.65 ms/ques [INFO] Entity Linking P: 0.95, R: 0.925, micro F1: 0.9333333333333332, macro F1: 0.9373333333333334 [INFO] Relation Linking P: 0.9, R: 0.9, micro F1: 0.9, macro F1: 0.9 [INFO] Type Linking P: 0.65, R: 0.65, micro F1: 0.65, macro F1: 0.65 @@ -287,8 +287,8 @@ Question 20: Who is the stockholder of the road tunnels operated by the Massachu [INFO] Golden Answer: [http://dbpedia.org/resource/Massachusetts] [INFO] Predicted Answer: [http://dbpedia.org/resource/Massachusetts] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 21, P: 0.827, R: 0.905, macro F1: 0.842, macro F1*: 0.864 -[INFO] Current time: 2630 ms, average: 3496.95 ms/ques, entity detection: 668.24 ms/ques (linking tool: 559.10 ms/ques), relation detection: 1035.52 ms/ques (similarity: 716.29 ms/ques), EDG generation: 375.57 ms/ques +[INFO] Cumulative metrics, sample: 21, P: 0.827, R: 0.905, macro F1: 0.864 +[INFO] Current question consumed time: 2536 ms, average: 3412.62 ms/ques, entity detection: 656.52 ms/ques (linking tool: 554.81 ms/ques), relation detection: 1029.33 ms/ques (similarity: 717.90 ms/ques), EDG generation: 362.81 ms/ques [INFO] Entity Linking P: 0.9523809523809523, R: 0.9285714285714286, micro F1: 0.9365079365079364, macro F1: 0.9403254972875226 [INFO] Relation Linking P: 0.8809523809523809, R: 0.8809523809523809, micro F1: 0.8809523809523809, macro F1: 0.8809523809523809 [INFO] Type Linking P: 0.6190476190476191, R: 0.6190476190476191, micro F1: 0.6190476190476191, macro F1: 0.6190476190476191 @@ -301,8 +301,8 @@ Question 21: What are the awards won by the film director of Saraband ? [INFO] Golden Answer: [http://dbpedia.org/resource/Academy_Awards, http://dbpedia.org/resource/Goethe_Prize, http://dbpedia.org/resource/Praemium_Imperiale] [INFO] Predicted Answer: [http://dbpedia.org/resource/Academy_Awards, http://dbpedia.org/resource/Goethe_Prize, http://dbpedia.org/resource/Praemium_Imperiale] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 22, P: 0.835, R: 0.909, macro F1: 0.849, macro F1*: 0.870 -[INFO] Current time: 1294 ms, average: 3396.82 ms/ques, entity detection: 645.23 ms/ques (linking tool: 537.18 ms/ques), relation detection: 1015.00 ms/ques (similarity: 698.91 ms/ques), EDG generation: 363.05 ms/ques +[INFO] Cumulative metrics, sample: 22, P: 0.835, R: 0.909, macro F1: 0.870 +[INFO] Current question consumed time: 1384 ms, average: 3320.41 ms/ques, entity detection: 635.23 ms/ques (linking tool: 533.73 ms/ques), relation detection: 1008.64 ms/ques (similarity: 700.32 ms/ques), EDG generation: 352.00 ms/ques [INFO] Entity Linking P: 0.9545454545454546, R: 0.9318181818181818, micro F1: 0.9393939393939393, macro F1: 0.9430449069003286 [INFO] Relation Linking P: 0.8636363636363636, R: 0.8636363636363636, micro F1: 0.8636363636363636, macro F1: 0.8636363636363636 [INFO] Type Linking P: 0.6363636363636364, R: 0.6363636363636364, micro F1: 0.6363636363636364, macro F1: 0.6363636363636364 @@ -315,8 +315,8 @@ Question 22: What is the almamater of the president whose vice president was Enr [INFO] Golden Answer: [http://dbpedia.org/resource/Cornell_University] [INFO] Predicted Answer: [http://dbpedia.org/resource/Cornell_University] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 23, P: 0.842, R: 0.913, macro F1: 0.855, macro F1*: 0.876 -[INFO] Current time: 1316 ms, average: 3306.35 ms/ques, entity detection: 622.00 ms/ques (linking tool: 516.39 ms/ques), relation detection: 993.61 ms/ques (similarity: 688.87 ms/ques), EDG generation: 348.48 ms/ques +[INFO] Cumulative metrics, sample: 23, P: 0.842, R: 0.913, macro F1: 0.876 +[INFO] Current question consumed time: 1410 ms, average: 3237.35 ms/ques, entity detection: 612.91 ms/ques (linking tool: 513.52 ms/ques), relation detection: 989.74 ms/ques (similarity: 690.83 ms/ques), EDG generation: 337.96 ms/ques [INFO] Entity Linking P: 0.9565217391304348, R: 0.9347826086956522, micro F1: 0.9420289855072462, macro F1: 0.9455272363818092 [INFO] Relation Linking P: 0.8695652173913043, R: 0.8695652173913043, micro F1: 0.8695652173913043, macro F1: 0.8695652173913043 [INFO] Type Linking P: 0.6086956521739131, R: 0.6086956521739131, micro F1: 0.6086956521739131, macro F1: 0.6086956521739131 @@ -329,8 +329,8 @@ Question 23: What is the route end of Birmingham and Oxford Junction Railway ? [INFO] Golden Answer: [http://dbpedia.org/resource/Birmingham] [INFO] Predicted Answer: [http://dbpedia.org/resource/Birmingham] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 24, P: 0.848, R: 0.917, macro F1: 0.861, macro F1*: 0.881 -[INFO] Current time: 945 ms, average: 3207.96 ms/ques, entity detection: 614.79 ms/ques (linking tool: 513.29 ms/ques), relation detection: 959.25 ms/ques (similarity: 664.79 ms/ques), EDG generation: 334.50 ms/ques +[INFO] Cumulative metrics, sample: 24, P: 0.848, R: 0.917, macro F1: 0.881 +[INFO] Current question consumed time: 971 ms, average: 3142.92 ms/ques, entity detection: 606.12 ms/ques (linking tool: 510.21 ms/ques), relation detection: 955.71 ms/ques (similarity: 666.04 ms/ques), EDG generation: 324.42 ms/ques [INFO] Entity Linking P: 0.9583333333333334, R: 0.9375, micro F1: 0.9444444444444443, macro F1: 0.9478021978021978 [INFO] Relation Linking P: 0.875, R: 0.875, micro F1: 0.875, macro F1: 0.875 [INFO] Type Linking P: 0.625, R: 0.625, micro F1: 0.625, macro F1: 0.625 @@ -343,8 +343,8 @@ Question 24: Where did the Vitis Vinifera grapes originate? [INFO] Golden Answer: [http://dbpedia.org/resource/California, http://dbpedia.org/resource/Italy, http://dbpedia.org/resource/Spain, http://dbpedia.org/resource/Portugal, http://dbpedia.org/resource/Germany, http://dbpedia.org/resource/Weinsberg, http://dbpedia.org/resource/Central_Europe, http://dbpedia.org/resource/Greece, http://dbpedia.org/resource/Sicily, http://dbpedia.org/resource/France, http://dbpedia.org/resource/Turkey, http://dbpedia.org/resource/Freiburg_im_Breisgau, http://dbpedia.org/resource/Province_of_Verona, http://dbpedia.org/resource/Province_of_Vicenza, http://dbpedia.org/resource/South_West_France_(wine_region), http://dbpedia.org/resource/Georgia_(country), http://dbpedia.org/resource/Croatia, http://dbpedia.org/resource/Alsace_wine, http://dbpedia.org/resource/Cyprus, http://dbpedia.org/resource/Loire, http://dbpedia.org/resource/Palatinate_(wine_region), http://dbpedia.org/resource/Lauffen_am_Neckar, http://dbpedia.org/resource/Switzerland, http://dbpedia.org/resource/Thurgau, http://dbpedia.org/resource/Austria, http://dbpedia.org/resource/Geisenheim, http://dbpedia.org/resource/Austria-Hungary, http://dbpedia.org/resource/Transylvania, http://dbpedia.org/resource/Santorini, http://dbpedia.org/resource/Czechoslovakia, http://dbpedia.org/resource/Moravia, http://dbpedia.org/resource/Egypt, http://dbpedia.org/resource/Rhine, http://dbpedia.org/resource/Argentina, http://dbpedia.org/resource/Israel, http://dbpedia.org/resource/United_States, http://dbpedia.org/resource/Rheingau, http://dbpedia.org/resource/German_wine, http://dbpedia.org/resource/Bordeaux_wine, http://dbpedia.org/resource/French_wine, http://dbpedia.org/resource/Hungary, http://dbpedia.org/resource/Slovakia, http://dbpedia.org/resource/Burgundy, http://dbpedia.org/resource/Klosterneuburg, http://dbpedia.org/resource/Alzey, http://dbpedia.org/resource/Lorraine, http://dbpedia.org/resource/Randersacker, http://dbpedia.org/resource/Moldavia, http://dbpedia.org/resource/Palatinate_(region), http://dbpedia.org/resource/Bulgaria, http://dbpedia.org/resource/Siebeldingen, http://dbpedia.org/resource/Romania, http://dbpedia.org/resource/Ukraine, http://dbpedia.org/resource/Italian_wine, http://dbpedia.org/resource/Moldova, http://dbpedia.org/resource/Serbia, http://dbpedia.org/resource/Slovenia, http://dbpedia.org/resource/Friuli-Venezia_Giulia_wine, http://dbpedia.org/resource/Bosnia_and_Herzegovina, http://dbpedia.org/resource/Refosco, http://dbpedia.org/resource/Montferrat, http://dbpedia.org/resource/Jura_(department), http://dbpedia.org/resource/Maissau, http://dbpedia.org/resource/Ardèche, http://dbpedia.org/resource/Montpellier, http://dbpedia.org/resource/South_Africa, http://dbpedia.org/resource/Republic_of_Macedonia, http://dbpedia.org/resource/Franconia_(wine_region)] [INFO] Predicted Answer: [http://dbpedia.org/resource/Plastid] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 25, P: 0.814, R: 0.880, macro F1: 0.827, macro F1*: 0.846 -[INFO] Current time: 1043 ms, average: 3121.36 ms/ques, entity detection: 598.56 ms/ques (linking tool: 499.16 ms/ques), relation detection: 933.24 ms/ques (similarity: 645.32 ms/ques), EDG generation: 321.40 ms/ques +[INFO] Cumulative metrics, sample: 25, P: 0.814, R: 0.880, macro F1: 0.846 +[INFO] Current question consumed time: 1061 ms, average: 3059.64 ms/ques, entity detection: 590.04 ms/ques (linking tool: 496.08 ms/ques), relation detection: 930.36 ms/ques (similarity: 646.40 ms/ques), EDG generation: 312.08 ms/ques [INFO] Entity Linking P: 0.96, R: 0.94, micro F1: 0.9466666666666665, macro F1: 0.9498947368421051 [INFO] Relation Linking P: 0.84, R: 0.84, micro F1: 0.84, macro F1: 0.8399999999999999 [INFO] Type Linking P: 0.6, R: 0.6, micro F1: 0.6, macro F1: 0.6 @@ -357,8 +357,8 @@ Question 25: How many people are there whose children died in Indiana? [INFO] Golden Answer: [10] [INFO] Predicted Answer: [26, 1] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 26, P: 0.783, R: 0.846, macro F1: 0.795, macro F1*: 0.813 -[INFO] Current time: 5672 ms, average: 3219.46 ms/ques, entity detection: 604.92 ms/ques (linking tool: 505.23 ms/ques), relation detection: 979.31 ms/ques (similarity: 703.00 ms/ques), EDG generation: 309.69 ms/ques +[INFO] Cumulative metrics, sample: 26, P: 0.783, R: 0.846, macro F1: 0.813 +[INFO] Current question consumed time: 5540 ms, average: 3155.04 ms/ques, entity detection: 594.58 ms/ques (linking tool: 500.04 ms/ques), relation detection: 972.65 ms/ques (similarity: 704.31 ms/ques), EDG generation: 300.69 ms/ques [INFO] Entity Linking P: 0.9615384615384616, R: 0.9423076923076923, micro F1: 0.9487179487179487, macro F1: 0.9518259518259519 [INFO] Relation Linking P: 0.8076923076923077, R: 0.8076923076923077, micro F1: 0.8076923076923077, macro F1: 0.8076923076923077 [INFO] Type Linking P: 0.5769230769230769, R: 0.5769230769230769, micro F1: 0.5769230769230769, macro F1: 0.5769230769230769 @@ -371,8 +371,8 @@ Question 26: What is the military branch of the William French Smith which is al [INFO] Golden Answer: [http://dbpedia.org/resource/United_States_Navy] [INFO] Predicted Answer: [http://dbpedia.org/resource/United_States_Navy] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 27, P: 0.791, R: 0.852, macro F1: 0.803, macro F1*: 0.820 -[INFO] Current time: 1693 ms, average: 3162.96 ms/ques, entity detection: 595.37 ms/ques (linking tool: 493.59 ms/ques), relation detection: 963.07 ms/ques (similarity: 690.07 ms/ques), EDG generation: 300.81 ms/ques +[INFO] Cumulative metrics, sample: 27, P: 0.791, R: 0.852, macro F1: 0.820 +[INFO] Current question consumed time: 1736 ms, average: 3102.48 ms/ques, entity detection: 584.78 ms/ques (linking tool: 487.93 ms/ques), relation detection: 957.30 ms/ques (similarity: 691.26 ms/ques), EDG generation: 294.19 ms/ques [INFO] Entity Linking P: 0.9629629629629629, R: 0.9444444444444444, micro F1: 0.9506172839506172, macro F1: 0.95361380798274 [INFO] Relation Linking P: 0.8148148148148148, R: 0.8148148148148148, micro F1: 0.8148148148148148, macro F1: 0.8148148148148148 [INFO] Type Linking P: 0.5925925925925926, R: 0.5925925925925926, micro F1: 0.5925925925925926, macro F1: 0.5925925925925926 @@ -385,8 +385,8 @@ Question 27: What is the river whose source is Lake Ontario? [INFO] Golden Answer: [http://dbpedia.org/resource/Saint_Lawrence_River] [INFO] Predicted Answer: [http://dbpedia.org/resource/Saint_Lawrence_River] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 28, P: 0.799, R: 0.857, macro F1: 0.810, macro F1*: 0.827 -[INFO] Current time: 1834 ms, average: 3115.50 ms/ques, entity detection: 582.50 ms/ques (linking tool: 481.21 ms/ques), relation detection: 953.32 ms/ques (similarity: 692.89 ms/ques), EDG generation: 290.68 ms/ques +[INFO] Cumulative metrics, sample: 28, P: 0.799, R: 0.857, macro F1: 0.827 +[INFO] Current question consumed time: 1764 ms, average: 3054.68 ms/ques, entity detection: 571.86 ms/ques (linking tool: 475.68 ms/ques), relation detection: 947.39 ms/ques (similarity: 693.50 ms/ques), EDG generation: 284.25 ms/ques [INFO] Entity Linking P: 0.9642857142857143, R: 0.9464285714285714, micro F1: 0.9523809523809523, macro F1: 0.9552736982643525 [INFO] Relation Linking P: 0.8214285714285714, R: 0.8214285714285714, micro F1: 0.8214285714285714, macro F1: 0.8214285714285714 [INFO] Type Linking P: 0.5714285714285714, R: 0.5714285714285714, micro F1: 0.5714285714285714, macro F1: 0.5714285714285714 @@ -399,8 +399,8 @@ Question 28: Whichlocation city of the Denver Broncos is the palce of birth of S [INFO] Golden Answer: [http://dbpedia.org/resource/Colorado, http://dbpedia.org/resource/Denver] [INFO] Predicted Answer: [http://dbpedia.org/resource/Colorado, http://dbpedia.org/resource/Denver] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 29, P: 0.805, R: 0.862, macro F1: 0.816, macro F1*: 0.833 -[INFO] Current time: 1149 ms, average: 3047.69 ms/ques, entity detection: 568.21 ms/ques (linking tool: 468.17 ms/ques), relation detection: 924.21 ms/ques (similarity: 671.86 ms/ques), EDG generation: 282.62 ms/ques +[INFO] Cumulative metrics, sample: 29, P: 0.805, R: 0.862, macro F1: 0.833 +[INFO] Current question consumed time: 1047 ms, average: 2985.45 ms/ques, entity detection: 557.45 ms/ques (linking tool: 462.45 ms/ques), relation detection: 918.14 ms/ques (similarity: 672.00 ms/ques), EDG generation: 275.97 ms/ques [INFO] Entity Linking P: 0.9655172413793104, R: 0.9310344827586207, micro F1: 0.9425287356321839, macro F1: 0.9479623824451411 [INFO] Relation Linking P: 0.8275862068965517, R: 0.8103448275862069, micro F1: 0.8160919540229885, macro F1: 0.818874773139746 [INFO] Type Linking P: 0.5862068965517241, R: 0.5862068965517241, micro F1: 0.5862068965517241, macro F1: 0.5862068965517241 @@ -413,8 +413,8 @@ Question 29: What is the religion of the person who founded the Emel magazine? [INFO] Golden Answer: [http://dbpedia.org/resource/Islam] [INFO] Predicted Answer: [http://dbpedia.org/resource/Islam] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 30, P: 0.812, R: 0.867, macro F1: 0.822, macro F1*: 0.838 -[INFO] Current time: 1583 ms, average: 2998.87 ms/ques, entity detection: 553.70 ms/ques (linking tool: 455.17 ms/ques), relation detection: 907.77 ms/ques (similarity: 661.20 ms/ques), EDG generation: 274.47 ms/ques +[INFO] Cumulative metrics, sample: 30, P: 0.812, R: 0.867, macro F1: 0.838 +[INFO] Current question consumed time: 1283 ms, average: 2928.70 ms/ques, entity detection: 542.60 ms/ques (linking tool: 449.27 ms/ques), relation detection: 900.67 ms/ques (similarity: 660.83 ms/ques), EDG generation: 267.67 ms/ques [INFO] Entity Linking P: 0.9666666666666667, R: 0.9333333333333333, micro F1: 0.9444444444444444, macro F1: 0.9497076023391814 [INFO] Relation Linking P: 0.8333333333333334, R: 0.8166666666666667, micro F1: 0.8222222222222223, macro F1: 0.8249158249158249 [INFO] Type Linking P: 0.5666666666666667, R: 0.5666666666666667, micro F1: 0.5666666666666667, macro F1: 0.5666666666666667 @@ -427,8 +427,8 @@ Question 30: Give the name of the managers of England national under-20 footbal [INFO] Golden Answer: [http://dbpedia.org/resource/Peter_Taylor_(footballer,_born_1953), http://dbpedia.org/resource/Gareth_Southgate] [INFO] Predicted Answer: [http://dbpedia.org/resource/England_national_football_team, http://dbpedia.org/resource/England_national_under-18_football_team] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 31, P: 0.786, R: 0.839, macro F1: 0.796, macro F1*: 0.811 -[INFO] Current time: 2496 ms, average: 2982.65 ms/ques, entity detection: 544.71 ms/ques (linking tool: 446.74 ms/ques), relation detection: 899.39 ms/ques (similarity: 652.16 ms/ques), EDG generation: 266.52 ms/ques +[INFO] Cumulative metrics, sample: 31, P: 0.786, R: 0.839, macro F1: 0.811 +[INFO] Current question consumed time: 2395 ms, average: 2911.48 ms/ques, entity detection: 534.29 ms/ques (linking tool: 441.26 ms/ques), relation detection: 892.26 ms/ques (similarity: 651.87 ms/ques), EDG generation: 259.90 ms/ques [INFO] Entity Linking P: 0.967741935483871, R: 0.9193548387096774, micro F1: 0.9354838709677419, macro F1: 0.9429280397022332 [INFO] Relation Linking P: 0.8225806451612904, R: 0.8225806451612904, micro F1: 0.8172043010752689, macro F1: 0.8225806451612904 [INFO] Type Linking P: 0.5806451612903226, R: 0.5806451612903226, micro F1: 0.5806451612903226, macro F1: 0.5806451612903226 @@ -441,8 +441,8 @@ Question 31: How many cities are around the sea in which the the fig tree bay me [INFO] Golden Answer: [13] [INFO] Predicted Answer: [2] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 32, P: 0.761, R: 0.812, macro F1: 0.771, macro F1*: 0.786 -[INFO] Current time: 4975 ms, average: 3044.91 ms/ques, entity detection: 531.62 ms/ques (linking tool: 435.03 ms/ques), relation detection: 939.47 ms/ques (similarity: 677.25 ms/ques), EDG generation: 259.09 ms/ques +[INFO] Cumulative metrics, sample: 32, P: 0.761, R: 0.812, macro F1: 0.786 +[INFO] Current question consumed time: 4450 ms, average: 2959.56 ms/ques, entity detection: 521.06 ms/ques (linking tool: 429.44 ms/ques), relation detection: 930.19 ms/ques (similarity: 677.41 ms/ques), EDG generation: 252.69 ms/ques [INFO] Entity Linking P: 0.96875, R: 0.921875, micro F1: 0.9375, macro F1: 0.9447314049586777 [INFO] Relation Linking P: 0.796875, R: 0.796875, micro F1: 0.7916666666666667, macro F1: 0.796875 [INFO] Type Linking P: 0.5625, R: 0.5625, micro F1: 0.5625, macro F1: 0.5625 @@ -455,8 +455,8 @@ Question 32: is blue lagoon island a part of the bahamas achipelago? [INFO] Golden Answer: [true] [INFO] Predicted Answer: [true] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 33, P: 0.768, R: 0.818, macro F1: 0.778, macro F1*: 0.793 -[INFO] Current time: 1043 ms, average: 2984.24 ms/ques, entity detection: 525.48 ms/ques (linking tool: 427.61 ms/ques), relation detection: 916.91 ms/ques (similarity: 660.42 ms/ques), EDG generation: 251.82 ms/ques +[INFO] Cumulative metrics, sample: 33, P: 0.768, R: 0.818, macro F1: 0.793 +[INFO] Current question consumed time: 976 ms, average: 2899.45 ms/ques, entity detection: 514.30 ms/ques (linking tool: 421.61 ms/ques), relation detection: 907.70 ms/ques (similarity: 660.55 ms/ques), EDG generation: 245.58 ms/ques [INFO] Entity Linking P: 0.9696969696969697, R: 0.9090909090909091, micro F1: 0.9292929292929293, macro F1: 0.93841642228739 [INFO] Relation Linking P: 0.803030303030303, R: 0.803030303030303, micro F1: 0.797979797979798, macro F1: 0.803030303030303 [INFO] Type Linking P: 0.5757575757575758, R: 0.5757575757575758, micro F1: 0.5757575757575758, macro F1: 0.5757575757575758 @@ -469,8 +469,8 @@ Question 33: Name the movies distributed by RKO Pictures and has music composer [INFO] Golden Answer: [http://dbpedia.org/resource/Citizen_Kane, http://dbpedia.org/resource/On_Dangerous_Ground, http://dbpedia.org/resource/The_Devil_and_Daniel_Webster_(film)] [INFO] Predicted Answer: [http://dbpedia.org/resource/Citizen_Kane, http://dbpedia.org/resource/On_Dangerous_Ground, http://dbpedia.org/resource/The_Devil_and_Daniel_Webster_(film)] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 34, P: 0.775, R: 0.824, macro F1: 0.784, macro F1*: 0.799 -[INFO] Current time: 2256 ms, average: 2962.82 ms/ques, entity detection: 534.91 ms/ques (linking tool: 435.32 ms/ques), relation detection: 906.00 ms/ques (similarity: 649.56 ms/ques), EDG generation: 245.53 ms/ques +[INFO] Cumulative metrics, sample: 34, P: 0.775, R: 0.824, macro F1: 0.799 +[INFO] Current question consumed time: 2070 ms, average: 2875.06 ms/ques, entity detection: 521.38 ms/ques (linking tool: 427.38 ms/ques), relation detection: 895.97 ms/ques (similarity: 649.50 ms/ques), EDG generation: 239.35 ms/ques [INFO] Entity Linking P: 0.9705882352941176, R: 0.9117647058823529, micro F1: 0.9313725490196079, macro F1: 0.9402573529411764 [INFO] Relation Linking P: 0.8088235294117647, R: 0.8088235294117647, micro F1: 0.803921568627451, macro F1: 0.8088235294117648 [INFO] Type Linking P: 0.5588235294117647, R: 0.5588235294117647, micro F1: 0.5588235294117647, macro F1: 0.5588235294117647 @@ -483,8 +483,8 @@ Question 34: Did the New Way party merge in the One Israel party? [INFO] Golden Answer: [true] [INFO] Predicted Answer: [true] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 35, P: 0.782, R: 0.829, macro F1: 0.791, macro F1*: 0.804 -[INFO] Current time: 2174 ms, average: 2940.29 ms/ques, entity detection: 537.91 ms/ques (linking tool: 438.09 ms/ques), relation detection: 897.03 ms/ques (similarity: 641.51 ms/ques), EDG generation: 239.11 ms/ques +[INFO] Cumulative metrics, sample: 35, P: 0.782, R: 0.829, macro F1: 0.804 +[INFO] Current question consumed time: 2138 ms, average: 2854.00 ms/ques, entity detection: 525.57 ms/ques (linking tool: 431.31 ms/ques), relation detection: 885.51 ms/ques (similarity: 640.60 ms/ques), EDG generation: 235.14 ms/ques [INFO] Entity Linking P: 0.9714285714285714, R: 0.9142857142857143, micro F1: 0.9333333333333335, macro F1: 0.941991341991342 [INFO] Relation Linking P: 0.7857142857142857, R: 0.7857142857142857, micro F1: 0.7809523809523811, macro F1: 0.7857142857142857 [INFO] Type Linking P: 0.5714285714285714, R: 0.5714285714285714, micro F1: 0.5714285714285714, macro F1: 0.5714285714285714 @@ -497,8 +497,8 @@ Question 35: How many pole drivers have been there in grand prixes where first d [INFO] Golden Answer: [9] [INFO] Predicted Answer: [9, 10] [INFO] P: 0.5, R: 1.0, F: 0.6666666666666666 -[INFO] Cumulative metrics, sample: 36, P: 0.774, R: 0.833, macro F1: 0.787, macro F1*: 0.803 -[INFO] Current time: 3909 ms, average: 2967.22 ms/ques, entity detection: 530.25 ms/ques (linking tool: 430.00 ms/ques), relation detection: 904.06 ms/ques (similarity: 650.75 ms/ques), EDG generation: 233.78 ms/ques +[INFO] Cumulative metrics, sample: 36, P: 0.774, R: 0.833, macro F1: 0.803 +[INFO] Current question consumed time: 3833 ms, average: 2881.19 ms/ques, entity detection: 518.00 ms/ques (linking tool: 423.25 ms/ques), relation detection: 891.50 ms/ques (similarity: 649.81 ms/ques), EDG generation: 229.92 ms/ques [INFO] Entity Linking P: 0.9722222222222222, R: 0.9166666666666666, micro F1: 0.9351851851851853, macro F1: 0.9436274509803921 [INFO] Relation Linking P: 0.7916666666666666, R: 0.7916666666666666, micro F1: 0.7870370370370371, macro F1: 0.7916666666666666 [INFO] Type Linking P: 0.5555555555555556, R: 0.5555555555555556, micro F1: 0.5555555555555556, macro F1: 0.5555555555555556 @@ -511,8 +511,8 @@ Question 36: Where are jockeys in Dubai World Cup from? [INFO] Golden Answer: [http://dbpedia.org/resource/Panama, http://dbpedia.org/resource/Panama_City, http://dbpedia.org/resource/Avignon, http://dbpedia.org/resource/France, http://dbpedia.org/resource/United_States, http://dbpedia.org/resource/Dallas, http://dbpedia.org/resource/Puerto_Rico, http://dbpedia.org/resource/Carolina,_Puerto_Rico, Altadena, California@en, http://dbpedia.org/resource/Herrera_Province, http://dbpedia.org/resource/Santo_Domingo, http://dbpedia.org/resource/Lafayette,_Louisiana, http://dbpedia.org/resource/Mobridge,_South_Dakota, http://dbpedia.org/resource/Arizona, http://dbpedia.org/resource/Nogales,_Arizona, http://dbpedia.org/resource/West_Covina,_California, http://dbpedia.org/resource/Dublin, http://dbpedia.org/resource/Republic_of_Ireland, http://dbpedia.org/resource/Brazil] [INFO] Predicted Answer: [http://dbpedia.org/resource/Alex_Solis, http://dbpedia.org/resource/Invasor_(horse), http://dbpedia.org/resource/Pleasantly_Perfect, http://dbpedia.org/resource/Mickael_Barzalona, http://dbpedia.org/resource/Roses_In_May, http://dbpedia.org/resource/Cigar_(horse), http://dbpedia.org/resource/Jerry_D._Bailey, http://dbpedia.org/resource/John_R._Velazquez, http://dbpedia.org/resource/Richard_E._Mandella, http://dbpedia.org/resource/Captain_Steve, http://dbpedia.org/resource/Fernando_Jara, http://dbpedia.org/resource/Joel_Rosario, http://dbpedia.org/resource/Almutawakel, http://dbpedia.org/resource/Dubai_Millennium, http://dbpedia.org/resource/Singspiel_(horse), http://dbpedia.org/resource/Curlin, http://dbpedia.org/resource/Dale_Romans, http://dbpedia.org/resource/Kiaran_McLaughlin, http://dbpedia.org/resource/Robby_Albarado, http://dbpedia.org/resource/William_I._Mott, http://dbpedia.org/resource/Animal_Kingdom_(horse), http://dbpedia.org/resource/Bob_Baffert, http://dbpedia.org/resource/Silver_Charm, http://dbpedia.org/resource/Street_Cry, http://dbpedia.org/resource/Moon_Ballad, http://dbpedia.org/resource/Aaron_Gryder, http://dbpedia.org/resource/African_Story, http://dbpedia.org/resource/Electrocutionist, http://dbpedia.org/resource/Eoin_G._Harty, http://dbpedia.org/resource/Monterosso_(horse), http://dbpedia.org/resource/Prince_Bishop_(horse), http://dbpedia.org/resource/T._J._Pereira, http://dbpedia.org/resource/Victoire_Pisa, http://dbpedia.org/resource/Well_Armed] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 37, P: 0.753, R: 0.811, macro F1: 0.766, macro F1*: 0.781 -[INFO] Current time: 846 ms, average: 2909.92 ms/ques, entity detection: 522.84 ms/ques (linking tool: 423.46 ms/ques), relation detection: 883.32 ms/ques (similarity: 635.65 ms/ques), EDG generation: 227.76 ms/ques +[INFO] Cumulative metrics, sample: 37, P: 0.753, R: 0.811, macro F1: 0.781 +[INFO] Current question consumed time: 735 ms, average: 2823.19 ms/ques, entity detection: 509.73 ms/ques (linking tool: 416.27 ms/ques), relation detection: 871.08 ms/ques (similarity: 634.68 ms/ques), EDG generation: 223.97 ms/ques [INFO] Entity Linking P: 0.972972972972973, R: 0.918918918918919, micro F1: 0.936936936936937, macro F1: 0.9451737451737452 [INFO] Relation Linking P: 0.7972972972972973, R: 0.7837837837837838, micro F1: 0.7837837837837839, macro F1: 0.7904827904827905 [INFO] Type Linking P: 0.5405405405405406, R: 0.5405405405405406, micro F1: 0.5405405405405406, macro F1: 0.5405405405405406 @@ -525,8 +525,8 @@ Question 37: Who owns the newspaper which was founded by Nehru? [INFO] Golden Answer: [http://dbpedia.org/resource/Indian_National_Congress] [INFO] Predicted Answer: [http://dbpedia.org/resource/India] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 38, P: 0.733, R: 0.789, macro F1: 0.746, macro F1*: 0.760 -[INFO] Current time: 1423 ms, average: 2870.79 ms/ques, entity detection: 514.39 ms/ques (linking tool: 414.58 ms/ques), relation detection: 875.55 ms/ques (similarity: 629.97 ms/ques), EDG generation: 222.21 ms/ques +[INFO] Cumulative metrics, sample: 38, P: 0.733, R: 0.789, macro F1: 0.760 +[INFO] Current question consumed time: 1418 ms, average: 2786.21 ms/ques, entity detection: 502.11 ms/ques (linking tool: 407.82 ms/ques), relation detection: 861.84 ms/ques (similarity: 629.50 ms/ques), EDG generation: 218.82 ms/ques [INFO] Entity Linking P: 0.9473684210526315, R: 0.8947368421052632, micro F1: 0.9122807017543861, macro F1: 0.9203007518796993 [INFO] Relation Linking P: 0.7763157894736842, R: 0.7631578947368421, micro F1: 0.7631578947368421, macro F1: 0.769680611785875 [INFO] Type Linking P: 0.5263157894736842, R: 0.5263157894736842, micro F1: 0.5263157894736842, macro F1: 0.5263157894736842 @@ -539,8 +539,8 @@ Question 38: To which places do the flights go by airlines headquartered in the [INFO] Golden Answer: [worldwide@en, 16, Worldwide@en, Charters@en, See below@en, 18, 5, 3, 24, http://dbpedia.org/resource/Europe, 0, Mwanza, Kilimanjaro, Mtwara, Nairobi, Johannesburg,Dubai and London Gatwick@en, 10, 26, 4, Unknown@en, 42, http://dbpedia.org/resource/British_Isles, http://dbpedia.org/resource/Continental_Europe, http://dbpedia.org/resource/North_America, 17, Varies@en, 2] [INFO] Predicted Answer: [http://dbpedia.org/resource/1001_Gaussia, http://dbpedia.org/resource/10_Canis_Majoris, http://dbpedia.org/resource/10th_parallel_south, http://dbpedia.org/resource/11_Camelopardalis, http://dbpedia.org/resource/14827_Hypnos, http://dbpedia.org/resource/1604_Tombaugh, http://dbpedia.org/resource/16_Cygni, http://dbpedia.org/resource/16_East_Broad_Street, http://dbpedia.org/resource/1931_Nicaragua_earthquake, http://dbpedia.org/resource/1972_Nicaragua_earthquake, http://dbpedia.org/resource/19P/Borrelly, http://dbpedia.org/resource/200_Public_Square, http://dbpedia.org/resource/204_Kallisto, http://dbpedia.org/resource/216_Kleopatra, http://dbpedia.org/resource/23rd_Street_Grounds, http://dbpedia.org/resource/2839_Annette, http://dbpedia.org/resource/31_Cygni, http://dbpedia.org/resource/34_Circe, http://dbpedia.org/resource/3552_Don_Quixote, http://dbpedia.org/resource/3754_Kathleen, http://dbpedia.org/resource/3_Centauri, http://dbpedia.org/resource/40th_parallel_south, http://dbpedia.org/resource/4492_Debussy, http://dbpedia.org/resource/60558_Echeclus, http://dbpedia.org/resource/67P/Churyumov–Gerasimenko, http://dbpedia.org/resource/6th_of_October_City, http://dbpedia.org/resource/70th_parallel_north, http://dbpedia.org/resource/75th_parallel_south, http://dbpedia.org/resource/7_Camelopardalis, http://dbpedia.org/resource/9th_arrondissement_of_Paris, http://dbpedia.org/resource/A._E._Staley, http://dbpedia.org/resource/A371_road, http://dbpedia.org/resource/A4161_road, http://dbpedia.org/resource/A4232_road, http://dbpedia.org/resource/A470_road, http://dbpedia.org/resource/A48(M)_motorway, http://dbpedia.org/resource/A4_autostrada_(Poland), http://dbpedia.org/resource/ADS_9731, http://dbpedia.org/resource/AM_Canum_Venaticorum, http://dbpedia.org/resource/A_Christmas_Story, http://dbpedia.org/resource/A_Christmas_Story_House, http://dbpedia.org/resource/Abdeen_Palace, http://dbpedia.org/resource/Abilene,_Texas, http://dbpedia.org/resource/Achill_Island, http://dbpedia.org/resource/Aci_Catena, http://dbpedia.org/resource/Acra_(fortress), http://dbpedia.org/resource/Adal_Sultanate, http://dbpedia.org/resource/Adamsdown, http://dbpedia.org/resource/Adamstown,_Dublin, http://dbpedia.org/resource/Aeaea, http://dbpedia.org/resource/Aelia_Capitolina, http://dbpedia.org/resource/Afton,_Wyoming, http://dbpedia.org/resource/Agioi_Pantes, http://dbpedia.org/resource/Agios_Nikolaos,_Crete, http://dbpedia.org/resource/Ailsa_Craig, http://dbpedia.org/resource/Aisne, http://dbpedia.org/resource/Aix-en-Provence, http://dbpedia.org/resource/Ajijic, http://dbpedia.org/resource/Ajuran_Sultanate, http://dbpedia.org/resource/Al-Askar, http://dbpedia.org/resource/Al-Azhar_Mosque, http://dbpedia.org/resource/Al-Hakim_Mosque, http://dbpedia.org/resource/Al-Hussein_Mosque, http://dbpedia.org/resource/Al-Nasir_Muhammad_Mosque, http://dbpedia.org/resource/Alamogordo_Daily_News, http://dbpedia.org/resource/Alania, http://dbpedia.org/resource/Albanifriedhof, http://dbpedia.org/resource/Albertina, http://dbpedia.org/resource/Albireo, http://dbpedia.org/resource/Alconbury, http://dbpedia.org/resource/Alcorn_State_University, http://dbpedia.org/resource/Alexander_Island, http://dbpedia.org/resource/Algol, http://dbpedia.org/resource/Algonquin_Provincial_Park, http://dbpedia.org/resource/Allan_H._Treman_State_Marine_Park, http://dbpedia.org/resource/Allen_Theatre, http://dbpedia.org/resource/Alon_Shvut, http://dbpedia.org/resource/Alpha_Caeli, http://dbpedia.org/resource/Alpha_Camelopardalis, http://dbpedia.org/resource/Alpha_Ceti, http://dbpedia.org/resource/Alpha_Coronae_Australis, http://dbpedia.org/resource/Alpha_Coronae_Borealis, http://dbpedia.org/resource/Alpha_Telescopii, http://dbpedia.org/resource/Alum_Creek_(Ohio), http://dbpedia.org/resource/Amari_Valley, http://dbpedia.org/resource/American_Antiquarian_Society, http://dbpedia.org/resource/American_Woolen_Company, http://dbpedia.org/resource/Amherst,_Nova_Scotia, http://dbpedia.org/resource/Anafi, http://dbpedia.org/resource/Anagni, http://dbpedia.org/resource/Ancient_Canaanite_religion, http://dbpedia.org/resource/Ancient_Carthage, http://dbpedia.org/resource/Andros, http://dbpedia.org/resource/Andros_(town), http://dbpedia.org/resource/Anglican_Diocese_of_Southwark, http://dbpedia.org/resource/Animas,_New_Mexico, http://dbpedia.org/resource/Annals_of_the_Four_Masters, http://dbpedia.org/resource/Ano_Syros, http://dbpedia.org/resource/Anqing, http://dbpedia.org/resource/Anqing_Yangtze_River_Bridge, http://dbpedia.org/resource/Antlia, http://dbpedia.org/resource/Aon_Center_(Chicago), http://dbpedia.org/resource/Aptera,_Greece, http://dbpedia.org/resource/Arab_Baths_(Ceuta), http://dbpedia.org/resource/Arab_Monetary_Fund, http://dbpedia.org/resource/Arauca_River, http://dbpedia.org/resource/Arcadia_(utopia), http://dbpedia.org/resource/Archbasilica_of_St._John_Lateran, http://dbpedia.org/resource/Archbishopric_of_Riga, http://dbpedia.org/resource/Archie_Comics, http://dbpedia.org/resource/Ardea,_Lazio, http://dbpedia.org/resource/Arena_District, http://dbpedia.org/resource/Areopagus_of_Eastern_Continental_Greece, http://dbpedia.org/resource/Argun_River_(Caucasus), http://dbpedia.org/resource/Argyre_Planitia, http://dbpedia.org/resource/Arkadi_Monastery, http://dbpedia.org/resource/Armour_Square,_Chicago, http://dbpedia.org/resource/Arpino, http://dbpedia.org/resource/Artane,_Dublin, http://dbpedia.org/resource/Artscape_Theatre_Centre, http://dbpedia.org/resource/Asgard_(crater), http://dbpedia.org/resource/Ashburton,_Devon, http://dbpedia.org/resource/Ashfield,_Massachusetts, http://dbpedia.org/resource/Ashford_Castle, http://dbpedia.org/resource/Ashtabula,_Ohio, http://dbpedia.org/resource/Ashville,_Ohio, http://dbpedia.org/resource/Asia_Society, http://dbpedia.org/resource/Asiago, http://dbpedia.org/resource/Aston_Hall, http://dbpedia.org/resource/Atabapo_River, http://dbpedia.org/resource/Athlone,_Cape_Town, http://dbpedia.org/resource/Athlone_Stadium, http://dbpedia.org/resource/Atlantis,_Western_Cape, http://dbpedia.org/resource/Attymass, http://dbpedia.org/resource/Auburn,_California, http://dbpedia.org/resource/Aughagower, http://dbpedia.org/resource/Aurora,_Cayuga_County,_New_York, http://dbpedia.org/resource/Aurora,_Ontario, http://dbpedia.org/resource/Autoroutes_of_Quebec, http://dbpedia.org/resource/Auxerre, http://dbpedia.org/resource/Avenue_Foch, http://dbpedia.org/resource/Axbridge, http://dbpedia.org/resource/Axis_mundi, http://dbpedia.org/resource/Azay-le-Rideau, http://dbpedia.org/resource/Aš, http://dbpedia.org/resource/BE_Camelopardalis, http://dbpedia.org/resource/BPM_37093, http://dbpedia.org/resource/Baalbek, http://dbpedia.org/resource/Bab_Ksiba, http://dbpedia.org/resource/Bab_Zuweila, http://dbpedia.org/resource/Bab_al-Futuh, http://dbpedia.org/resource/Bab_al-Nasr_(Cairo), http://dbpedia.org/resource/Babylon_Fortress, http://dbpedia.org/resource/Bad_Saarow, http://dbpedia.org/resource/Bahir_Dar, http://dbpedia.org/resource/Baiae, http://dbpedia.org/resource/Baiheliang, http://dbpedia.org/resource/Bajo_Nuevo_Bank, http://dbpedia.org/resource/Baker_College, http://dbpedia.org/resource/Balance_spring, http://dbpedia.org/resource/Balbridie, http://dbpedia.org/resource/Baldoyle, http://dbpedia.org/resource/Balgriffin, http://dbpedia.org/resource/Balla,_County_Mayo, http://dbpedia.org/resource/Ballina,_County_Mayo, http://dbpedia.org/resource/Ballina_railway_station, http://dbpedia.org/resource/Ballinamallard, http://dbpedia.org/resource/Ballindine, http://dbpedia.org/resource/Ballinrobe, http://dbpedia.org/resource/Ballintubber, http://dbpedia.org/resource/Ballintubber_Abbey, http://dbpedia.org/resource/Ballyboden, http://dbpedia.org/resource/Ballyboughal, http://dbpedia.org/resource/Ballycastle,_County_Mayo, http://dbpedia.org/resource/Ballycroy,_County_Mayo, http://dbpedia.org/resource/Ballyfermot, http://dbpedia.org/resource/Ballygall, http://dbpedia.org/resource/Ballyhaunis, http://dbpedia.org/resource/Ballyhaunis_railway_station, http://dbpedia.org/resource/Ballymun, http://dbpedia.org/resource/Ballyroan,_County_Dublin, http://dbpedia.org/resource/Balsas_River, http://dbpedia.org/resource/Baltimore,_Ohio, http://dbpedia.org/resource/Banff_Springs_Hotel, http://dbpedia.org/resource/Bangor_Abbey, http://dbpedia.org/resource/Bank_Street_Grounds, http://dbpedia.org/resource/Bankoff_circle, http://dbpedia.org/resource/Bankstown_Airport, http://dbpedia.org/resource/Bantry_Bay,_Cape_Town, http://dbpedia.org/resource/Baptism_of_the_Lord, http://dbpedia.org/resource/Barangay, http://dbpedia.org/resource/Barbeau_Peak, http://dbpedia.org/resource/Barnes-Jewish_Hospital, http://dbpedia.org/resource/Barry,_Vale_of_Glamorgan, http://dbpedia.org/resource/Barstow,_California, http://dbpedia.org/resource/Barzowice, http://dbpedia.org/resource/Basilica_of_St_Denis, http://dbpedia.org/resource/Battelle_Hall, http://dbpedia.org/resource/Batting_cage, http://dbpedia.org/resource/Battle_of_Aquae_Sextiae, http://dbpedia.org/resource/Batts_Combe_quarry, http://dbpedia.org/resource/Bayside,_Dublin, http://dbpedia.org/resource/Bear_Creek_Regional_Park_and_Nature_Center, http://dbpedia.org/resource/Bear_Mountain_(Connecticut), http://dbpedia.org/resource/Beaujolais, http://dbpedia.org/resource/Bedford,_Ohio, http://dbpedia.org/resource/Bedford_Heights,_Ohio, http://dbpedia.org/resource/Bedfordshire, http://dbpedia.org/resource/Beethoven_(crater), http://dbpedia.org/resource/Belcoo, http://dbpedia.org/resource/Belfield,_Dublin, http://dbpedia.org/resource/Belhar, http://dbpedia.org/resource/Bellanaleck, http://dbpedia.org/resource/Belleek,_County_Fermanagh, http://dbpedia.org/resource/Bellefontaine,_Ohio, http://dbpedia.org/resource/Bellville,_Western_Cape, http://dbpedia.org/resource/Bellville_Velodrome, http://dbpedia.org/resource/Bellville_railway_station, http://dbpedia.org/resource/Belmullet, http://dbpedia.org/resource/Belron, http://dbpedia.org/resource/Bemidji,_Minnesota, http://dbpedia.org/resource/Ben_Ezra_Synagogue, http://dbpedia.org/resource/Benešov, http://dbpedia.org/resource/Benicia_Arsenal, http://dbpedia.org/resource/Benwee_Head, http://dbpedia.org/resource/Benzú, http://dbpedia.org/resource/Bergisch_Gladbach, http://dbpedia.org/resource/Berlin,_Connecticut, http://dbpedia.org/resource/Berlin,_New_Hampshire, http://dbpedia.org/resource/Beslan, http://dbpedia.org/resource/Beta_Caeli, http://dbpedia.org/resource/Beta_Camelopardalis, http://dbpedia.org/resource/Beta_Canis_Majoris, http://dbpedia.org/resource/Beta_Canum_Venaticorum, http://dbpedia.org/resource/Beta_Carinae, http://dbpedia.org/resource/Beta_Cephei, http://dbpedia.org/resource/Beta_Ceti, http://dbpedia.org/resource/Beta_Chamaeleontis, http://dbpedia.org/resource/Beta_Coronae_Australis, http://dbpedia.org/resource/Beta_Coronae_Borealis, http://dbpedia.org/resource/Beta_Crucis, http://dbpedia.org/resource/Beta_Pictoris, http://dbpedia.org/resource/Bexley_Hall, http://dbpedia.org/resource/Bexleyheath, http://dbpedia.org/resource/Big_Darby_Creek, http://dbpedia.org/resource/Big_Muddy_National_Fish_and_Wildlife_Refuge, http://dbpedia.org/resource/Big_Rapids,_Michigan, http://dbpedia.org/resource/Big_Walnut_Creek, http://dbpedia.org/resource/Billy_Bishop_Toronto_City_Airport, http://dbpedia.org/resource/Birchgrove,_Cardiff, http://dbpedia.org/resource/Bishop's_storehouse, http://dbpedia.org/resource/Bishop_Lavis, http://dbpedia.org/resource/Bishopscourt,_Cape_Town, http://dbpedia.org/resource/Black_Hawk_County,_Iowa, http://dbpedia.org/resource/Black_Hawk_Hotel, http://dbpedia.org/resource/Black_River_Academy, http://dbpedia.org/resource/Blackrock,_Dublin, http://dbpedia.org/resource/Blacksod_Lighthouse, http://dbpedia.org/resource/Blaine,_Washington, http://dbpedia.org/resource/Blair,_Nebraska, http://dbpedia.org/resource/Blair_Field, http://dbpedia.org/resource/Blanchard,_North_Dakota, http://dbpedia.org/resource/Blanchardstown, http://dbpedia.org/resource/Blanco,_Texas, http://dbpedia.org/resource/Blanket_bog, http://dbpedia.org/resource/Blansko, http://dbpedia.org/resource/Block_Island, http://dbpedia.org/resource/Block_Island_Sound, http://dbpedia.org/resource/Bloomfield,_Connecticut, http://dbpedia.org/resource/Blossom_Music_Center, http://dbpedia.org/resource/Bloubergstrand, http://dbpedia.org/resource/Blue_Downs, http://dbpedia.org/resource/Blue_Train_(South_Africa), http://dbpedia.org/resource/Blue_Wahoos_Stadium, http://dbpedia.org/resource/Blue_and_Green_Lines_(Cleveland), http://dbpedia.org/resource/Bluebell,_Dublin, http://dbpedia.org/resource/Bo-Kaap, http://dbpedia.org/resource/Bobbio, http://dbpedia.org/resource/Boho,_County_Fermanagh, http://dbpedia.org/resource/Bohola, http://dbpedia.org/resource/Bolesławiec, http://dbpedia.org/resource/Bolognese_Republic, http://dbpedia.org/resource/Bolton_Field, http://dbpedia.org/resource/Bonfield,_Ontario, http://dbpedia.org/resource/Boone_County,_Missouri, http://dbpedia.org/resource/Boone_County_Courthouse_(Columbia,_Missouri), http://dbpedia.org/resource/Boone_Hospital_Center, http://dbpedia.org/resource/Boonville,_Missouri, http://dbpedia.org/resource/Booterstown, http://dbpedia.org/resource/Boscawen,_New_Hampshire, http://dbpedia.org/resource/Boston_Public_Library, http://dbpedia.org/resource/Boswednack, http://dbpedia.org/resource/Boulders_Beach, http://dbpedia.org/resource/Bourbonnais,_Illinois, http://dbpedia.org/resource/Bourdeaux, http://dbpedia.org/resource/Bourn, http://dbpedia.org/resource/Bow,_New_Hampshire, http://dbpedia.org/resource/Bradley_International_Airport, http://dbpedia.org/resource/Breaffy, http://dbpedia.org/resource/Brecon, http://dbpedia.org/resource/Breney_Common, http://dbpedia.org/resource/Bridge_River,_British_Columbia, http://dbpedia.org/resource/Bridgend, http://dbpedia.org/resource/Bridgeport_&_Port_Jefferson_Ferry, http://dbpedia.org/resource/Bridgetown,_Devon, http://dbpedia.org/resource/Bright_giant, http://dbpedia.org/resource/Bristol_Channel, http://dbpedia.org/resource/British_Somaliland, http://dbpedia.org/resource/Broadhaven_Bay, http://dbpedia.org/resource/Broadmoor,_Colorado, http://dbpedia.org/resource/Broadstairs, http://dbpedia.org/resource/Brocard_circle, http://dbpedia.org/resource/Brook_Park,_Ohio, http://dbpedia.org/resource/Brookeborough, http://dbpedia.org/resource/Brooklyn,_Ohio, http://dbpedia.org/resource/Brooklyn_Centre, http://dbpedia.org/resource/Brooklyn_Heights,_Ohio, http://dbpedia.org/resource/Broumov, http://dbpedia.org/resource/Brunswick_School, http://dbpedia.org/resource/Bruntál, http://dbpedia.org/resource/Buckden_Towers, http://dbpedia.org/resource/Buckeye-Shaker, http://dbpedia.org/resource/Buckfast_Abbey, http://dbpedia.org/resource/Buenaventura,_Valle_del_Cauca, http://dbpedia.org/resource/Buenos_Aires_and_Pacific_Railway, http://dbpedia.org/resource/Bundenbach, http://dbpedia.org/resource/Bundesautobahn_3, http://dbpedia.org/resource/Bundesautobahn_555, http://dbpedia.org/resource/Burdett,_Kansas, http://dbpedia.org/resource/Burg_bei_Magdeburg, http://dbpedia.org/resource/Burning_Mountain, http://dbpedia.org/resource/Burwell,_Cambridgeshire, http://dbpedia.org/resource/Bury_St_Edmunds, http://dbpedia.org/resource/Burzenland, http://dbpedia.org/resource/Bute_Park, http://dbpedia.org/resource/Butler_County,_Nebraska, http://dbpedia.org/resource/Byrsa, http://dbpedia.org/resource/Břeclav, http://dbpedia.org/resource/CF_postcode_area, http://dbpedia.org/resource/CS_Camelopardalis, http://dbpedia.org/resource/Caberfae_Peaks_Ski_&_Golf_Resort, http://dbpedia.org/resource/Cabinteely, http://dbpedia.org/resource/Cabo_de_la_Vela, http://dbpedia.org/resource/Cabra,_Dublin, http://dbpedia.org/resource/Cadillac,_Gironde, http://dbpedia.org/resource/Cadillac,_Michigan, http://dbpedia.org/resource/Cadillac,_Saskatchewan, http://dbpedia.org/resource/Cadillac-en-Fronsadais, http://dbpedia.org/resource/Cadillac_(Montreal_Metro), http://dbpedia.org/resource/Cadillac_AOC, http://dbpedia.org/resource/Cadillac_Mountain, http://dbpedia.org/resource/Caerau,_Cardiff, http://dbpedia.org/resource/Caerau_Hillfort, http://dbpedia.org/resource/Caguán_River, http://dbpedia.org/resource/Cai_(state), http://dbpedia.org/resource/Cairo_International_Airport, http://dbpedia.org/resource/Cairo_International_Stadium, http://dbpedia.org/resource/Cairo_Transportation_Authority, http://dbpedia.org/resource/Caiyuanba_Bridge, http://dbpedia.org/resource/Callisto_(band), http://dbpedia.org/resource/Callisto_(moon), http://dbpedia.org/resource/Callow_Rock_quarry, http://dbpedia.org/resource/Caloris_Basin, http://dbpedia.org/resource/Camel_cavalry, http://dbpedia.org/resource/Camelback_Ranch, http://dbpedia.org/resource/Camp_Chase, http://dbpedia.org/resource/Camp_Greentop_Historic_District, http://dbpedia.org/resource/Camp_Misty_Mount_Historic_District, http://dbpedia.org/resource/Campeche, http://dbpedia.org/resource/Camps_Bay, http://dbpedia.org/resource/Canada_House, http://dbpedia.org/resource/Canadian_Pacific_Building_(Toronto), http://dbpedia.org/resource/Canadian_Railway_Museum, http://dbpedia.org/resource/Cancer_(constellation), http://dbpedia.org/resource/Canis_Major_Overdensity, http://dbpedia.org/resource/Canon,_Georgia, http://dbpedia.org/resource/Cantagalo_Station, http://dbpedia.org/resource/Canterbury,_New_Hampshire, http://dbpedia.org/resource/Canton,_Baltimore, http://dbpedia.org/resource/Canton,_Barron_County,_Wisconsin, http://dbpedia.org/resource/Canton,_California, http://dbpedia.org/resource/Canton,_Cardiff, http://dbpedia.org/resource/Canton,_Connecticut, http://dbpedia.org/resource/Canton,_Georgia, http://dbpedia.org/resource/Canton,_Illinois, http://dbpedia.org/resource/Canton,_Iowa, http://dbpedia.org/resource/Canton,_Kansas, http://dbpedia.org/resource/Canton,_Maine, http://dbpedia.org/resource/Canton,_Massachusetts, http://dbpedia.org/resource/Canton,_Minnesota, http://dbpedia.org/resource/Canton,_Mississippi, http://dbpedia.org/resource/Canton,_Missouri, http://dbpedia.org/resource/Canton,_Montana, http://dbpedia.org/resource/Canton,_New_Jersey, http://dbpedia.org/resource/Canton,_North_Carolina, http://dbpedia.org/resource/Canton,_Oklahoma, http://dbpedia.org/resource/Canton,_Ontario, http://dbpedia.org/resource/Canton,_Pennsylvania, http://dbpedia.org/resource/Canton,_South_Dakota, http://dbpedia.org/resource/Canton,_Texas, http://dbpedia.org/resource/Canton,_West_Virginia, http://dbpedia.org/resource/Canton,_Wisconsin, http://dbpedia.org/resource/Canton_(village),_New_York, http://dbpedia.org/resource/Canton_City,_North_Dakota, http://dbpedia.org/resource/Canton_Tower, http://dbpedia.org/resource/Canóvanas,_Puerto_Rico, http://dbpedia.org/resource/Cape_Baba, http://dbpedia.org/resource/Cape_Breton_(electoral_district), http://dbpedia.org/resource/Cape_Floristic_Region, http://dbpedia.org/resource/Cape_Girardeau_Regional_Airport, http://dbpedia.org/resource/Cape_Point, http://dbpedia.org/resource/Cape_Town_City_Hall, http://dbpedia.org/resource/Cape_Town_International_Airport, http://dbpedia.org/resource/Cape_Town_Stadium, http://dbpedia.org/resource/Cape_Town_railway_station, http://dbpedia.org/resource/Caphtor, http://dbpedia.org/resource/Capitol_Center_for_the_Arts, http://dbpedia.org/resource/Capitol_Centre, http://dbpedia.org/resource/Capitol_Reef_National_Park, http://dbpedia.org/resource/Capitol_Square, http://dbpedia.org/resource/Capiz, http://dbpedia.org/resource/Cardeal_Arcoverde_Station, http://dbpedia.org/resource/Cardiff_Airport, http://dbpedia.org/resource/Cardiff_Arena, http://dbpedia.org/resource/Cardiff_Athletics_Stadium, http://dbpedia.org/resource/Cardiff_Bay, http://dbpedia.org/resource/Cardiff_Bay_Barrage, http://dbpedia.org/resource/Cardiff_Castle, http://dbpedia.org/resource/Cardiff_City_Stadium, http://dbpedia.org/resource/Cardiff_International_Pool, http://dbpedia.org/resource/Cardiff_International_Sports_Stadium, http://dbpedia.org/resource/Cardiff_Market, http://dbpedia.org/resource/Cardiff_Metropolitan_Cathedral, http://dbpedia.org/resource/Cardiff_Queen_Street_railway_station, http://dbpedia.org/resource/Cardiff_Roman_Fort, http://dbpedia.org/resource/Cardiff_Story, http://dbpedia.org/resource/Cardiff_city_centre, http://dbpedia.org/resource/Cardinal,_Virginia, http://dbpedia.org/resource/Carlyle_circle, http://dbpedia.org/resource/Carnacon, http://dbpedia.org/resource/Carra,_County_Mayo, http://dbpedia.org/resource/Carriacou, http://dbpedia.org/resource/Carrowmore_Lake, http://dbpedia.org/resource/Carrowteige, http://dbpedia.org/resource/Cartersville,_Georgia, http://dbpedia.org/resource/Carthage,_Illinois, http://dbpedia.org/resource/Carthage_amphitheatre, http://dbpedia.org/resource/Case_Township,_Michigan, http://dbpedia.org/resource/Case_del_Conte, http://dbpedia.org/resource/Casper–Natrona_County_International_Airport, http://dbpedia.org/resource/Cassiobury_House, http://dbpedia.org/resource/Cassiopeia_A, http://dbpedia.org/resource/Cassiterides, http://dbpedia.org/resource/Castell_Coch, http://dbpedia.org/resource/Castile_and_León, http://dbpedia.org/resource/Castillo_San_Felipe_del_Morro, http://dbpedia.org/resource/Castle_Quarter, http://dbpedia.org/resource/Castle_of_Good_Hope, http://dbpedia.org/resource/Castlebar, http://dbpedia.org/resource/Castlebar_railway_station, http://dbpedia.org/resource/Castlebar_transmitter, http://dbpedia.org/resource/Castleknock, http://dbpedia.org/resource/Cathays, http://dbpedia.org/resource/Cathays_(electoral_ward), http://dbpedia.org/resource/Cathays_Park, http://dbpedia.org/resource/Cathedral_Church_of_St._James_(Toronto), http://dbpedia.org/resource/Cathedral_floorplan, http://dbpedia.org/resource/Catoctin_Mountain_Park, http://dbpedia.org/resource/Caucasian_Knot, http://dbpedia.org/resource/Cave_of_the_Crystals, http://dbpedia.org/resource/Cayuga,_New_York, http://dbpedia.org/resource/Cayuga_County,_New_York, http://dbpedia.org/resource/Cedar_Falls_Ice_House, http://dbpedia.org/resource/Cell_site, http://dbpedia.org/resource/Cemetery_of_Confucius, http://dbpedia.org/resource/Centaurus_A, http://dbpedia.org/resource/Centennial_Park,_New_South_Wales, http://dbpedia.org/resource/Central,_Cleveland, http://dbpedia.org/resource/Central_American_montane_forests, http://dbpedia.org/resource/Central_Corridor_Rail_Line, http://dbpedia.org/resource/Central_Eastern_Alps, http://dbpedia.org/resource/Central_Federal_District, http://dbpedia.org/resource/Central_High_School_(Columbus,_Ohio), http://dbpedia.org/resource/Central_Ohio_Transit_Authority, http://dbpedia.org/resource/Central_Ontario, http://dbpedia.org/resource/Central_Plain_Metropolitan_Intercity_Rail, http://dbpedia.org/resource/Centralia,_Missouri, http://dbpedia.org/resource/Centralia,_Pennsylvania, http://dbpedia.org/resource/Century_City,_Cape_Town, http://dbpedia.org/resource/Ceres,_California, http://dbpedia.org/resource/Ceres,_Fife, http://dbpedia.org/resource/Ceres,_Georgia, http://dbpedia.org/resource/Ceres,_Piedmont, http://dbpedia.org/resource/Ceres,_Santa_Fe, http://dbpedia.org/resource/Ceres,_Victoria, http://dbpedia.org/resource/Ceres,_Virginia, http://dbpedia.org/resource/Ceres,_West_Virginia, http://dbpedia.org/resource/Ceres,_Western_Cape, http://dbpedia.org/resource/Ceres_Flat,_California, http://dbpedia.org/resource/Ceres_Koekedouw_Dam, http://dbpedia.org/resource/Ceres_Microregion, http://dbpedia.org/resource/Ceres_School, http://dbpedia.org/resource/Ceres_Township,_McKean_County,_Pennsylvania, http://dbpedia.org/resource/Cerro_El_Pital, http://dbpedia.org/resource/Cerro_Las_Minas, http://dbpedia.org/resource/Certified_Credit_Professional, http://dbpedia.org/resource/Certified_Insurance_Counselor, http://dbpedia.org/resource/Certified_International_Investment_Analyst, http://dbpedia.org/resource/Certified_Property_Manager, http://dbpedia.org/resource/Certified_Risk_Manager, http://dbpedia.org/resource/Ceuta_Heliport, http://dbpedia.org/resource/Chadwick_Arboretum, http://dbpedia.org/resource/Chagrin_Falls,_Ohio, http://dbpedia.org/resource/Chaldia, http://dbpedia.org/resource/Chalk,_Kent, http://dbpedia.org/resource/Champaign,_Illinois, http://dbpedia.org/resource/Changsha, http://dbpedia.org/resource/Chania, http://dbpedia.org/resource/Chania_(regional_unit), http://dbpedia.org/resource/Chania_International_Airport, http://dbpedia.org/resource/Chao_Lake, http://dbpedia.org/resource/Chaohu, http://dbpedia.org/resource/Chaoshan, http://dbpedia.org/resource/Chaozhou, http://dbpedia.org/resource/Chapel_Hill,_North_Carolina, http://dbpedia.org/resource/Chapelizod, http://dbpedia.org/resource/Charing_Cross_railway_station, http://dbpedia.org/resource/Chariot_burial, http://dbpedia.org/resource/Charlemagne_building, http://dbpedia.org/resource/Charles_W._Morgan_(ship), http://dbpedia.org/resource/Charleston,_Angus, http://dbpedia.org/resource/Charleston,_Arizona, http://dbpedia.org/resource/Charleston,_Arkansas, http://dbpedia.org/resource/Charleston,_Dundee, http://dbpedia.org/resource/Charleston,_Halifax_County,_North_Carolina, http://dbpedia.org/resource/Charleston,_Illinois, http://dbpedia.org/resource/Charleston,_Iowa, http://dbpedia.org/resource/Charleston,_Kentucky, http://dbpedia.org/resource/Charleston,_Maine, http://dbpedia.org/resource/Charleston,_Merced_County,_California, http://dbpedia.org/resource/Charleston,_Mississippi, http://dbpedia.org/resource/Charleston,_Missouri, http://dbpedia.org/resource/Charleston,_Nevada, http://dbpedia.org/resource/Charleston,_New_Jersey, http://dbpedia.org/resource/Charleston,_New_York, http://dbpedia.org/resource/Charleston,_New_Zealand, http://dbpedia.org/resource/Charleston,_Newfoundland_and_Labrador, http://dbpedia.org/resource/Charleston,_Nova_Scotia, http://dbpedia.org/resource/Charleston,_Oklahoma, http://dbpedia.org/resource/Charleston,_Oregon, http://dbpedia.org/resource/Charleston,_South_Australia, http://dbpedia.org/resource/Charleston,_Tennessee, http://dbpedia.org/resource/Charleston,_Utah, http://dbpedia.org/resource/Charleston,_Vermont, http://dbpedia.org/resource/Charleston,_West_Virginia, http://dbpedia.org/resource/Charleston,_Yolo_County,_California, http://dbpedia.org/resource/Charleston_County,_South_Carolina, http://dbpedia.org/resource/Charleston_Township,_Coles_County,_Illinois, http://dbpedia.org/resource/Charleston_Township,_Kalamazoo_County,_Michigan, http://dbpedia.org/resource/Charleston_Township,_Tioga_County,_Pennsylvania, http://dbpedia.org/resource/Charlestown,_County_Mayo, http://dbpedia.org/resource/Charlotte_center_city, http://dbpedia.org/resource/Charon,_Louisiana, http://dbpedia.org/resource/Chartered_Alternative_Investment_Analyst, http://dbpedia.org/resource/Chartered_Property_Casualty_Underwriter, http://dbpedia.org/resource/Chateau_Lake_Louise, http://dbpedia.org/resource/Chatham,_Kent, http://dbpedia.org/resource/Chatham_Dockyard, http://dbpedia.org/resource/Chatteris, http://dbpedia.org/resource/Chechen-Ingush_Autonomous_Soviet_Socialist_Republic, http://dbpedia.org/resource/Chechen_Republic_of_Ichkeria, http://dbpedia.org/resource/Cheddar_Complex, http://dbpedia.org/resource/Cheddar_Gorge, http://dbpedia.org/resource/Cheddar_Palace, http://dbpedia.org/resource/Cheddar_Reservoir, http://dbpedia.org/resource/Cheddar_Wood, http://dbpedia.org/resource/Cheddar_Yeo, http://dbpedia.org/resource/Chellah, http://dbpedia.org/resource/Chen_(state), http://dbpedia.org/resource/Chepo_District, http://dbpedia.org/resource/Chepstow, http://dbpedia.org/resource/Chernobyl_Raion, http://dbpedia.org/resource/Cherry_Hinton, http://dbpedia.org/resource/Cherrywood,_Dublin, http://dbpedia.org/resource/Cheyenne_County,_Colorado, http://dbpedia.org/resource/Chi_Carinae, http://dbpedia.org/resource/Chi_Cygni, http://dbpedia.org/resource/Chiang_Mai, http://dbpedia.org/resource/Chicago_History_Museum, http://dbpedia.org/resource/Chicago_Rapid_Transit_Company, http://dbpedia.org/resource/Chicago_Spire, http://dbpedia.org/resource/Chicago_Stadium, http://dbpedia.org/resource/Chicago_metropolitan_area, http://dbpedia.org/resource/Chillicothe,_Ohio, http://dbpedia.org/resource/Chimor, http://dbpedia.org/resource/Chinese_Cuisine_Training_Institute, http://dbpedia.org/resource/Chiquinquirá, http://dbpedia.org/resource/Chishui_River_(Southwest_China), http://dbpedia.org/resource/Chizhou, http://dbpedia.org/resource/Chociwel, http://dbpedia.org/resource/Chomutov, http://dbpedia.org/resource/Chorweiler, http://dbpedia.org/resource/Christ_Church_Cathedral,_Oxford, http://dbpedia.org/resource/Christian,_West_Virginia, http://dbpedia.org/resource/Christian_County,_Illinois, http://dbpedia.org/resource/Christian_County,_Kentucky, http://dbpedia.org/resource/Christian_County,_Missouri, http://dbpedia.org/resource/Christian_Island_(Ontario), http://dbpedia.org/resource/Chrysi_(island), http://dbpedia.org/resource/Church_History_Library, http://dbpedia.org/resource/Church_History_Museum, http://dbpedia.org/resource/Church_of_St._George_(Cairo), http://dbpedia.org/resource/Churchill_Hospital, http://dbpedia.org/resource/Churchtown_Farm, http://dbpedia.org/resource/Chuzhou, http://dbpedia.org/resource/Château_Frontenac, http://dbpedia.org/resource/Château_de_Chambord, http://dbpedia.org/resource/Chûn_Castle, http://dbpedia.org/resource/Cilicia_(Roman_province), http://dbpedia.org/resource/Cincinnati_metropolitan_area, http://dbpedia.org/resource/Circle_of_antisimilitude, http://dbpedia.org/resource/Circleville,_Ohio, http://dbpedia.org/resource/Citadel_of_Salah_Ed-Din, http://dbpedia.org/resource/Citadelle_Laferrière, http://dbpedia.org/resource/CityPlace,_Toronto, http://dbpedia.org/resource/City_Hall,_Cardiff, http://dbpedia.org/resource/City_of_the_Dead_(Cairo), http://dbpedia.org/resource/Ciumești, http://dbpedia.org/resource/Clairton,_Pennsylvania, http://dbpedia.org/resource/Clanawley, http://dbpedia.org/resource/Clankelly, http://dbpedia.org/resource/Clare,_Michigan, http://dbpedia.org/resource/Claremont,_Cape_Town, http://dbpedia.org/resource/Claremorris, http://dbpedia.org/resource/Claremorris_railway_station, http://dbpedia.org/resource/Clark-Fulton, http://dbpedia.org/resource/Clark_Park, http://dbpedia.org/resource/Clayton_County,_Iowa, http://dbpedia.org/resource/Clearwater_Lakes, http://dbpedia.org/resource/Cleopatra,_Kentucky, http://dbpedia.org/resource/Cleopatra_(crater), http://dbpedia.org/resource/Cleveland,_England, http://dbpedia.org/resource/Cleveland_Arcade, http://dbpedia.org/resource/Cleveland_Arena, http://dbpedia.org/resource/Cleveland_Clinic, http://dbpedia.org/resource/Cleveland_Cultural_Gardens, http://dbpedia.org/resource/Cleveland_Hopkins_International_Airport, http://dbpedia.org/resource/Cleveland_Lakefront_Station, http://dbpedia.org/resource/Cleveland_Memorial_Shoreway, http://dbpedia.org/resource/Cleveland_Metroparks, http://dbpedia.org/resource/Cleveland_Metroparks_Zoo, http://dbpedia.org/resource/Cleveland_Stadium, http://dbpedia.org/resource/Cleveland_Theater_District, http://dbpedia.org/resource/Clifton,_Cape_Town, http://dbpedia.org/resource/Clinton,_Indiana, http://dbpedia.org/resource/Cliodynamics, http://dbpedia.org/resource/Clondalkin, http://dbpedia.org/resource/Clonsilla, http://dbpedia.org/resource/Clonskeagh, http://dbpedia.org/resource/Coat_of_arms_of_the_Chechen_Republic, http://dbpedia.org/resource/Coedkernew, http://dbpedia.org/resource/Coleman,_Texas, http://dbpedia.org/resource/Colgate_Clock_(Indiana), http://dbpedia.org/resource/Coligny,_Ain, http://dbpedia.org/resource/College_of_West_Anglia, http://dbpedia.org/resource/Collinwood, http://dbpedia.org/resource/Colloidal_crystal, http://dbpedia.org/resource/Cologne-Weidenpesch_Racecourse, http://dbpedia.org/resource/Cologne/Bonn_Airport_station, http://dbpedia.org/resource/Cologne_Butzweilerhof_Airport, http://dbpedia.org/resource/Cologne_Zoological_Garden, http://dbpedia.org/resource/Color_printing, http://dbpedia.org/resource/Colorado_Springs_Fine_Arts_Center, http://dbpedia.org/resource/Colorado_State_Capitol, http://dbpedia.org/resource/Colorado_State_Highway_115, http://dbpedia.org/resource/Colorado_State_Highway_21, http://dbpedia.org/resource/Colorado_State_Highway_83, http://dbpedia.org/resource/Colorado_State_Highway_94, http://dbpedia.org/resource/Columba_(constellation), http://dbpedia.org/resource/Columbia_Glacier_(Alaska), http://dbpedia.org/resource/Columbia_Regional_Airport, http://dbpedia.org/resource/Columbia_Soccer_Stadium, http://dbpedia.org/resource/Columbia_University_Medical_Center, http://dbpedia.org/resource/Columbus,_Ohio_metropolitan_area, http://dbpedia.org/resource/Columbus_City_Center, http://dbpedia.org/resource/Columbus_Metropolitan_Library, http://dbpedia.org/resource/Columbus_Motor_Speedway, http://dbpedia.org/resource/Columbus_Museum_of_Art, http://dbpedia.org/resource/Columbus_Ohio_Temple, http://dbpedia.org/resource/Columbus_Zoo_and_Aquarium, http://dbpedia.org/resource/Comet_Hyakutake, http://dbpedia.org/resource/Commuter_(Iarnród_Éireann), http://dbpedia.org/resource/Comoro_Islands, http://dbpedia.org/resource/Company's_Garden, http://dbpedia.org/resource/Complex_of_Sultan_al-Ashraf_Qaytbay, http://dbpedia.org/resource/Compton/Woodley_Airport, http://dbpedia.org/resource/Conakry, http://dbpedia.org/resource/Concord,_North_Carolina, http://dbpedia.org/resource/Concord_Historic_District, http://dbpedia.org/resource/Concord_Hospital_(New_Hampshire), http://dbpedia.org/resource/Concord_Monitor, http://dbpedia.org/resource/Concord_Municipal_Airport, http://dbpedia.org/resource/Concord_School_District_(New_Hampshire), http://dbpedia.org/resource/Cong,_County_Mayo, http://dbpedia.org/resource/Congo,_Alabama, http://dbpedia.org/resource/Congo,_New_South_Wales, http://dbpedia.org/resource/Congo,_Paraíba, http://dbpedia.org/resource/Congo,_West_Virginia, http://dbpedia.org/resource/Congo_River_(Ecuador), http://dbpedia.org/resource/Congo_River_(Panama), http://dbpedia.org/resource/Connecticut_Route_15, http://dbpedia.org/resource/Connecticut_Route_8, http://dbpedia.org/resource/Connecticut_Route_9, http://dbpedia.org/resource/Connecticut_Tigers, http://dbpedia.org/resource/Connecticut_Turnpike, http://dbpedia.org/resource/Connecticut_Wildcats, http://dbpedia.org/resource/Connecticut_panhandle, http://dbpedia.org/resource/Connemara, http://dbpedia.org/resource/Constantia,_Cape_Town, http://dbpedia.org/resource/Constantine,_Cornwall, http://dbpedia.org/resource/Constantine,_Michigan, http://dbpedia.org/resource/Constantine,_Switzerland, http://dbpedia.org/resource/Constantine_Bay, http://dbpedia.org/resource/Constantine_Province, http://dbpedia.org/resource/Constitutional_Court_of_South_Africa, http://dbpedia.org/resource/Coole_(barony), http://dbpedia.org/resource/Coolidge,_Arizona, http://dbpedia.org/resource/Coolidge_Dam, http://dbpedia.org/resource/Coolidge_Homestead, http://dbpedia.org/resource/Coolmine, http://dbpedia.org/resource/Coolock, http://dbpedia.org/resource/Cooper_Stadium, http://dbpedia.org/resource/Coptic_Cairo, http://dbpedia.org/resource/Coptic_Museum, http://dbpedia.org/resource/Coquihalla_Pass, http://dbpedia.org/resource/Corcovado_Rack_Railway, http://dbpedia.org/resource/Cornelius_Pass_Road, http://dbpedia.org/resource/Cornelscourt, http://dbpedia.org/resource/Corner_Gas, http://dbpedia.org/resource/Coronet_cluster, http://dbpedia.org/resource/Corroy,_County_Mayo, http://dbpedia.org/resource/Corvus_(constellation), http://dbpedia.org/resource/Cottage_Grove,_Minnesota, http://dbpedia.org/resource/Cottenham, http://dbpedia.org/resource/County_Cavan, http://dbpedia.org/resource/County_Fermanagh, http://dbpedia.org/resource/County_Kildare, http://dbpedia.org/resource/County_Leitrim, http://dbpedia.org/resource/County_Meath, http://dbpedia.org/resource/County_Roscommon, http://dbpedia.org/resource/County_of_Burgundy, http://dbpedia.org/resource/County_of_Holland, http://dbpedia.org/resource/County_of_La_Marche, http://dbpedia.org/resource/County_of_Mark, http://dbpedia.org/resource/County_of_Zeeland, http://dbpedia.org/resource/Coventry,_Connecticut, http://dbpedia.org/resource/Cowboy_church, http://dbpedia.org/resource/Cowley,_Oxfordshire, http://dbpedia.org/resource/Cox's_Cave, http://dbpedia.org/resource/Craigavon, http://dbpedia.org/resource/Craigellachie,_British_Columbia, http://dbpedia.org/resource/Craigmont,_Ontario, http://dbpedia.org/resource/Craney_Island_(Virginia), http://dbpedia.org/resource/Crank,_Merseyside, http://dbpedia.org/resource/Crank_Halt_railway_station, http://dbpedia.org/resource/Cranks,_Kentucky, http://dbpedia.org/resource/Cranks_(restaurant), http://dbpedia.org/resource/Crater_(constellation), http://dbpedia.org/resource/Crawfordsburn, http://dbpedia.org/resource/Creigiau, http://dbpedia.org/resource/Cresson,_Texas, http://dbpedia.org/resource/Creta_et_Cyrenaica, http://dbpedia.org/resource/Croagh_Patrick, http://dbpedia.org/resource/Crosley_Field, http://dbpedia.org/resource/Crossmolina, http://dbpedia.org/resource/Cuckooland_Museum, http://dbpedia.org/resource/Cuero,_Texas, http://dbpedia.org/resource/Culverhouse_Cross, http://dbpedia.org/resource/Cumberland_County,_Nova_Scotia, http://dbpedia.org/resource/Custer_State_Park, http://dbpedia.org/resource/Cuyahoga_County,_Ohio, http://dbpedia.org/resource/Cuyahoga_County_Courthouse, http://dbpedia.org/resource/Cuyahoga_County_Public_Library, http://dbpedia.org/resource/Cuyahoga_Falls,_Ohio, http://dbpedia.org/resource/Cuyahoga_Heights,_Ohio, http://dbpedia.org/resource/Cuyahoga_River, http://dbpedia.org/resource/Cygnus_A, http://dbpedia.org/resource/Cygnus_Loop, http://dbpedia.org/resource/Cygnus_OB2-12, http://dbpedia.org/resource/Cyncoed, http://dbpedia.org/resource/Cyril,_Oklahoma, http://dbpedia.org/resource/Cyrus,_Minnesota, http://dbpedia.org/resource/Cyrus,_West_Virginia, http://dbpedia.org/resource/Céide_Fields, http://dbpedia.org/resource/Daitō_Islands, http://dbpedia.org/resource/Dali_City, http://dbpedia.org/resource/Dalkey, http://dbpedia.org/resource/Dan_yr_Ogof, http://dbpedia.org/resource/Danbury_Branch, http://dbpedia.org/resource/Danbury_Municipal_Airport, http://dbpedia.org/resource/Darien,_Connecticut, http://dbpedia.org/resource/Darndale, http://dbpedia.org/resource/Dartmoor_crosses, http://dbpedia.org/resource/DeWitt_County,_Texas, http://dbpedia.org/resource/Deansgrange, http://dbpedia.org/resource/Debussy_(crater), http://dbpedia.org/resource/Deep_Impact_(spacecraft), http://dbpedia.org/resource/Deep_River,_Connecticut, http://dbpedia.org/resource/Dehler_Park, http://dbpedia.org/resource/Delaware,_Ohio, http://dbpedia.org/resource/Delaware_County,_Ohio, http://dbpedia.org/resource/Delft,_Cape_Town, http://dbpedia.org/resource/Delta2_Canis_Minoris, http://dbpedia.org/resource/Delta_Caeli, http://dbpedia.org/resource/Delta_Canis_Majoris, http://dbpedia.org/resource/Delta_Ceti, http://dbpedia.org/resource/Delta_Coronae_Borealis, http://dbpedia.org/resource/Delta_Crucis, http://dbpedia.org/resource/Delta_Cygni, http://dbpedia.org/resource/Delta_Velorum, http://dbpedia.org/resource/Democratic_Republic_of_Georgia, http://dbpedia.org/resource/Demre, http://dbpedia.org/resource/Dermoepidermal_junction, http://dbpedia.org/resource/Derrygonnelly, http://dbpedia.org/resource/Derrylin, http://dbpedia.org/resource/Detroit-Shoreway, http://dbpedia.org/resource/Detroit_River, http://dbpedia.org/resource/Detroit–Superior_Bridge, http://dbpedia.org/resource/Deux_arabesques, http://dbpedia.org/resource/Devenish_Island, http://dbpedia.org/resource/Devil's_Peak_(Cape_Town), http://dbpedia.org/resource/Dia_(island), http://dbpedia.org/resource/Dian_Lake, http://dbpedia.org/resource/Dick_LeBeau, http://dbpedia.org/resource/Dikti, http://dbpedia.org/resource/Dinas_Powys, http://dbpedia.org/resource/Diocese_of_Chichester, http://dbpedia.org/resource/Diocese_of_Durham, http://dbpedia.org/resource/Diocese_of_Guildford, http://dbpedia.org/resource/Diocese_of_Lichfield, http://dbpedia.org/resource/Diocese_of_St_Albans, http://dbpedia.org/resource/Dionysades, http://dbpedia.org/resource/Director_circle, http://dbpedia.org/resource/District_Six, http://dbpedia.org/resource/Diyu, http://dbpedia.org/resource/Dodger_Stadium, http://dbpedia.org/resource/Dollymount, http://dbpedia.org/resource/Domažlice, http://dbpedia.org/resource/Dominion_Observatory, http://dbpedia.org/resource/Domus_Aurea, http://dbpedia.org/resource/Don_Republic, http://dbpedia.org/resource/Donabate, http://dbpedia.org/resource/Donaghmede, http://dbpedia.org/resource/Dongting_Lake, http://dbpedia.org/resource/Donnybrook,_Dublin, http://dbpedia.org/resource/Donnycarney, http://dbpedia.org/resource/Dorado, http://dbpedia.org/resource/Dostoevskij_(crater), http://dbpedia.org/resource/Doudleby, http://dbpedia.org/resource/Doughty_Street, http://dbpedia.org/resource/Downpatrick, http://dbpedia.org/resource/Downtown_Cleveland, http://dbpedia.org/resource/Downtown_Columbia,_Missouri, http://dbpedia.org/resource/Dransfeld, http://dbpedia.org/resource/Dream_Team_(TV_series), http://dbpedia.org/resource/Driving_Park, http://dbpedia.org/resource/Drumcondra,_Dublin, http://dbpedia.org/resource/Drummond_Parish,_New_Brunswick, http://dbpedia.org/resource/Drydock_Number_One,_Norfolk_Naval_Shipyard, http://dbpedia.org/resource/Drymalia, http://dbpedia.org/resource/Dublin,_Ohio, http://dbpedia.org/resource/Dublin_Area_Rapid_Transit, http://dbpedia.org/resource/Dublin_South–Central_(Dáil_Éireann_constituency), http://dbpedia.org/resource/Dublin_South–West_(Dáil_Éireann_constituency), http://dbpedia.org/resource/Dubuque,_Iowa, http://dbpedia.org/resource/Duchy_of_Courland_and_Semigallia_(1918), http://dbpedia.org/resource/Duchy_of_Greater_Poland, http://dbpedia.org/resource/Duchy_of_Legnica, http://dbpedia.org/resource/Duchy_of_Limburg, http://dbpedia.org/resource/Duchy_of_Masovia, http://dbpedia.org/resource/Duchy_of_Pomerania, http://dbpedia.org/resource/Dunoon,_Cape_Town, http://dbpedia.org/resource/Dunquin, http://dbpedia.org/resource/Dunstable_Priory, http://dbpedia.org/resource/Dura-Europos_church, http://dbpedia.org/resource/Durrow,_County_Offaly, http://dbpedia.org/resource/Durrow_Abbey, http://dbpedia.org/resource/Dutch_Language_Union, http://dbpedia.org/resource/Duvillaun, http://dbpedia.org/resource/Duxford, http://dbpedia.org/resource/Dʿmt, http://dbpedia.org/resource/EL-DE_Haus, http://dbpedia.org/resource/EPOXI, http://dbpedia.org/resource/EZ_Canis_Majoris, http://dbpedia.org/resource/Eagle_Hotel_(Concord,_New_Hampshire), http://dbpedia.org/resource/Earth_(Wu_Xing), http://dbpedia.org/resource/East_Campus_Neighborhood, http://dbpedia.org/resource/East_Canton,_Ohio, http://dbpedia.org/resource/East_Cleveland,_Ohio, http://dbpedia.org/resource/East_Haddam,_Connecticut, http://dbpedia.org/resource/East_London,_Eastern_Cape, http://dbpedia.org/resource/East_Rail_Line, http://dbpedia.org/resource/East_Wall, http://dbpedia.org/resource/Eastern_Canadian_Shield_taiga, http://dbpedia.org/resource/Eastern_Connecticut_State_University, http://dbpedia.org/resource/Eastern_Ontario, http://dbpedia.org/resource/Eastlake,_Ohio, http://dbpedia.org/resource/Eastland_Mall_(Columbus,_Ohio), http://dbpedia.org/resource/Easton_Town_Center, http://dbpedia.org/resource/Echternach, http://dbpedia.org/resource/EcoCute, http://dbpedia.org/resource/Edmondstown, http://dbpedia.org/resource/Edmonton_Composting_Facility, http://dbpedia.org/resource/Edmund_the_Martyr, http://dbpedia.org/resource/Edwards_County,_Illinois, http://dbpedia.org/resource/Egelsbach, http://dbpedia.org/resource/Eglinton_Castle, http://dbpedia.org/resource/Egyptian_Exchange, http://dbpedia.org/resource/Egyptian_National_Military_Museum, http://dbpedia.org/resource/Egypt–Israel_Peace_Treaty, http://dbpedia.org/resource/Eilean_Chaluim_Chille, http://dbpedia.org/resource/El_Capitolio, http://dbpedia.org/resource/El_Centro,_California, http://dbpedia.org/resource/El_Paso,_Cesar, http://dbpedia.org/resource/Elafonisi, http://dbpedia.org/resource/Ellis_Fischel_Cancer_Center, http://dbpedia.org/resource/Ellis_Library, http://dbpedia.org/resource/Elounda, http://dbpedia.org/resource/Elsie's_River, http://dbpedia.org/resource/Ely,_Cambridgeshire, http://dbpedia.org/resource/Ely,_Cardiff, http://dbpedia.org/resource/Emeline_Fairbanks_Memorial_Library, http://dbpedia.org/resource/Emirate_of_Jabal_Shammar, http://dbpedia.org/resource/Endodermis, http://dbpedia.org/resource/Endostatin, http://dbpedia.org/resource/Englewood,_Chicago, http://dbpedia.org/resource/Enniskillen, http://dbpedia.org/resource/Enniskillen_Castle, http://dbpedia.org/resource/Ent_Air_Force_Base, http://dbpedia.org/resource/Ephrata_Cloister, http://dbpedia.org/resource/Epidamnos, http://dbpedia.org/resource/Epsilon_Carinae, http://dbpedia.org/resource/Epsilon_Coronae_Borealis, http://dbpedia.org/resource/Epsilon_Crucis, http://dbpedia.org/resource/Epsilon_Cygni, http://dbpedia.org/resource/Ermoupoli, http://dbpedia.org/resource/Errew, http://dbpedia.org/resource/Erris, http://dbpedia.org/resource/Eskilstuna, http://dbpedia.org/resource/Eta_Canis_Majoris, http://dbpedia.org/resource/Eta_Centauri, http://dbpedia.org/resource/Eta_Coronae_Borealis, http://dbpedia.org/resource/Eta_Sagittarii, http://dbpedia.org/resource/Etemenanki, http://dbpedia.org/resource/Euclid,_Ohio, http://dbpedia.org/resource/Euclid_Avenue_(Cleveland), http://dbpedia.org/resource/Euclid_Beach_Park, http://dbpedia.org/resource/European_Audiovisual_Observatory, http://dbpedia.org/resource/European_Directorate_for_the_Quality_of_Medicines, http://dbpedia.org/resource/Evansdale,_Iowa, http://dbpedia.org/resource/Everyman_Theatre, http://dbpedia.org/resource/Exarchate_of_Africa, http://dbpedia.org/resource/Excelsior_Springs,_Missouri, http://dbpedia.org/resource/Exomvourgo, http://dbpedia.org/resource/Ezhou, http://dbpedia.org/resource/FS_Canis_Majoris, http://dbpedia.org/resource/FU_Orionis_star, http://dbpedia.org/resource/Fairfax,_Cleveland, http://dbpedia.org/resource/Fairfield,_Connecticut, http://dbpedia.org/resource/Fairfield_County,_Connecticut, http://dbpedia.org/resource/Fairfield_County,_Ohio, http://dbpedia.org/resource/Fairmont_Royal_York, http://dbpedia.org/resource/Fairview,_Dublin, http://dbpedia.org/resource/Fairview_Park,_Ohio, http://dbpedia.org/resource/Falls_Village,_Connecticut, http://dbpedia.org/resource/Family_History_Library, http://dbpedia.org/resource/Faneuil_Hall, http://dbpedia.org/resource/Far_West,_Missouri, http://dbpedia.org/resource/Farmington,_Connecticut, http://dbpedia.org/resource/Faulagh, http://dbpedia.org/resource/Faurot_Field, http://dbpedia.org/resource/Fenland, http://dbpedia.org/resource/Fermanagh_District_Council, http://dbpedia.org/resource/Ferraiolo, http://dbpedia.org/resource/Fier, http://dbpedia.org/resource/Fingal, http://dbpedia.org/resource/Fingerpori, http://dbpedia.org/resource/Finglas, http://dbpedia.org/resource/First_Canadian_Place, http://dbpedia.org/resource/Fish_Hoek, http://dbpedia.org/resource/Fishers_Island,_New_York, http://dbpedia.org/resource/Fitzrovia, http://dbpedia.org/resource/Flag_Fen, http://dbpedia.org/resource/Flat_Holm, http://dbpedia.org/resource/Flin_Flon, http://dbpedia.org/resource/Flintshire, http://dbpedia.org/resource/Flixborough_disaster, http://dbpedia.org/resource/Flora_und_Botanischer_Garten_Köln, http://dbpedia.org/resource/Flyway, http://dbpedia.org/resource/Folegandros]... [INFO] P: 4.322959671109228E-6, R: 0.17391304347826086, F: 8.645704435786732E-6 -[INFO] Cumulative metrics, sample: 39, P: 0.714, R: 0.774, macro F1: 0.727, macro F1*: 0.743 -[INFO] Current time: 42847 ms, average: 3895.82 ms/ques, entity detection: 501.21 ms/ques (linking tool: 403.95 ms/ques), relation detection: 862.82 ms/ques (similarity: 620.49 ms/ques), EDG generation: 218.92 ms/ques +[INFO] Cumulative metrics, sample: 39, P: 0.714, R: 0.774, macro F1: 0.743 +[INFO] Current question consumed time: 37911 ms, average: 3686.85 ms/ques, entity detection: 489.23 ms/ques (linking tool: 397.36 ms/ques), relation detection: 849.23 ms/ques (similarity: 619.77 ms/ques), EDG generation: 215.62 ms/ques [INFO] Entity Linking P: 0.9230769230769231, R: 0.8717948717948718, micro F1: 0.8888888888888891, macro F1: 0.8967032967032966 [INFO] Relation Linking P: 0.7564102564102564, R: 0.7435897435897436, micro F1: 0.7435897435897437, macro F1: 0.749945211483673 [INFO] Type Linking P: 0.5128205128205128, R: 0.5128205128205128, micro F1: 0.5128205128205128, macro F1: 0.5128205128205128 @@ -553,8 +553,8 @@ Question 39: What is the magazine which have been founded by Ralph J. Gleason? [INFO] Golden Answer: [http://dbpedia.org/resource/Rolling_Stone] [INFO] Predicted Answer: [http://dbpedia.org/resource/Rolling_Stone] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 40, P: 0.721, R: 0.779, macro F1: 0.733, macro F1*: 0.749 -[INFO] Current time: 1009 ms, average: 3823.65 ms/ques, entity detection: 492.98 ms/ques (linking tool: 396.32 ms/ques), relation detection: 848.25 ms/ques (similarity: 611.92 ms/ques), EDG generation: 213.95 ms/ques +[INFO] Cumulative metrics, sample: 40, P: 0.721, R: 0.779, macro F1: 0.749 +[INFO] Current question consumed time: 942 ms, average: 3618.22 ms/ques, entity detection: 480.52 ms/ques (linking tool: 389.00 ms/ques), relation detection: 835.23 ms/ques (similarity: 611.27 ms/ques), EDG generation: 210.68 ms/ques [INFO] Entity Linking P: 0.925, R: 0.875, micro F1: 0.8916666666666668, macro F1: 0.8993055555555556 [INFO] Relation Linking P: 0.7625, R: 0.75, micro F1: 0.7500000000000001, macro F1: 0.7561983471074379 [INFO] Type Linking P: 0.5, R: 0.5, micro F1: 0.5, macro F1: 0.5 @@ -567,8 +567,8 @@ Question 40: Is Seattle the Ship builder of USS Marmora (IX-189)? [INFO] Golden Answer: [true] [INFO] Predicted Answer: [true] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 41, P: 0.728, R: 0.785, macro F1: 0.740, macro F1*: 0.755 -[INFO] Current time: 1718 ms, average: 3772.29 ms/ques, entity detection: 494.15 ms/ques (linking tool: 396.56 ms/ques), relation detection: 832.88 ms/ques (similarity: 599.90 ms/ques), EDG generation: 209.54 ms/ques +[INFO] Cumulative metrics, sample: 41, P: 0.728, R: 0.785, macro F1: 0.755 +[INFO] Current question consumed time: 1761 ms, average: 3572.93 ms/ques, entity detection: 483.39 ms/ques (linking tool: 390.56 ms/ques), relation detection: 819.34 ms/ques (similarity: 599.27 ms/ques), EDG generation: 205.90 ms/ques [INFO] Entity Linking P: 0.926829268292683, R: 0.8780487804878049, micro F1: 0.894308943089431, macro F1: 0.901779828609097 [INFO] Relation Linking P: 0.7682926829268293, R: 0.7560975609756098, micro F1: 0.7560975609756099, macro F1: 0.7621463414634148 [INFO] Type Linking P: 0.5121951219512195, R: 0.5121951219512195, micro F1: 0.5121951219512195, macro F1: 0.5121951219512195 @@ -581,8 +581,8 @@ Question 41: Is Tirana the largest city of Albania? [INFO] Golden Answer: [true] [INFO] Predicted Answer: [true] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 42, P: 0.735, R: 0.790, macro F1: 0.746, macro F1*: 0.761 -[INFO] Current time: 4267 ms, average: 3784.07 ms/ques, entity detection: 506.38 ms/ques (linking tool: 406.76 ms/ques), relation detection: 870.52 ms/ques (similarity: 617.79 ms/ques), EDG generation: 204.81 ms/ques +[INFO] Cumulative metrics, sample: 42, P: 0.735, R: 0.790, macro F1: 0.761 +[INFO] Current question consumed time: 4105 ms, average: 3585.60 ms/ques, entity detection: 496.38 ms/ques (linking tool: 401.38 ms/ques), relation detection: 854.76 ms/ques (similarity: 617.10 ms/ques), EDG generation: 201.24 ms/ques [INFO] Entity Linking P: 0.9285714285714286, R: 0.8809523809523809, micro F1: 0.896825396825397, macro F1: 0.9041353383458646 [INFO] Relation Linking P: 0.7738095238095238, R: 0.7619047619047619, micro F1: 0.7619047619047619, macro F1: 0.7678110003691399 [INFO] Type Linking P: 0.5238095238095238, R: 0.5238095238095238, micro F1: 0.5238095238095238, macro F1: 0.5238095238095238 @@ -595,8 +595,8 @@ Question 42: In which cities can the beverage related to Barq's be found? [INFO] Golden Answer: [http://dbpedia.org/resource/Indiana, http://dbpedia.org/resource/Jasper,_Indiana] [INFO] Predicted Answer: [http://dbpedia.org/resource/Jasper,_Indiana] [INFO] P: 1.0, R: 0.5, F: 0.6666666666666666 -[INFO] Cumulative metrics, sample: 43, P: 0.741, R: 0.783, macro F1: 0.744, macro F1*: 0.761 -[INFO] Current time: 2075 ms, average: 3744.33 ms/ques, entity detection: 497.98 ms/ques (linking tool: 399.12 ms/ques), relation detection: 867.58 ms/ques (similarity: 622.05 ms/ques), EDG generation: 200.53 ms/ques +[INFO] Cumulative metrics, sample: 43, P: 0.741, R: 0.783, macro F1: 0.761 +[INFO] Current question consumed time: 1926 ms, average: 3547.00 ms/ques, entity detection: 487.70 ms/ques (linking tool: 393.58 ms/ques), relation detection: 851.44 ms/ques (similarity: 621.40 ms/ques), EDG generation: 197.05 ms/ques [INFO] Entity Linking P: 0.9302325581395349, R: 0.8837209302325582, micro F1: 0.8992248062015505, macro F1: 0.9063804412641622 [INFO] Relation Linking P: 0.7790697674418605, R: 0.7674418604651163, micro F1: 0.7674418604651163, macro F1: 0.7732121000174855 [INFO] Type Linking P: 0.5116279069767442, R: 0.5116279069767442, micro F1: 0.5116279069767442, macro F1: 0.5116279069767442 @@ -609,8 +609,8 @@ Question 43: What is the birth place of the astronaut whose mission was the Vost [INFO] Golden Answer: [http://dbpedia.org/resource/Moscow] [INFO] Predicted Answer: [http://dbpedia.org/resource/Moscow] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 44, P: 0.747, R: 0.788, macro F1: 0.750, macro F1*: 0.767 -[INFO] Current time: 1118 ms, average: 3684.64 ms/ques, entity detection: 490.84 ms/ques (linking tool: 392.55 ms/ques), relation detection: 854.98 ms/ques (similarity: 614.48 ms/ques), EDG generation: 196.84 ms/ques +[INFO] Cumulative metrics, sample: 44, P: 0.747, R: 0.788, macro F1: 0.767 +[INFO] Current question consumed time: 1103 ms, average: 3491.45 ms/ques, entity detection: 480.30 ms/ques (linking tool: 386.89 ms/ques), relation detection: 838.75 ms/ques (similarity: 613.55 ms/ques), EDG generation: 194.84 ms/ques [INFO] Entity Linking P: 0.9318181818181818, R: 0.8863636363636364, micro F1: 0.9015151515151516, macro F1: 0.9085227272727272 [INFO] Relation Linking P: 0.7840909090909091, R: 0.7727272727272727, micro F1: 0.7727272727272727, macro F1: 0.7783676177836761 [INFO] Type Linking P: 0.5, R: 0.5, micro F1: 0.5, macro F1: 0.5 @@ -623,8 +623,8 @@ Question 44: Count everyone who studied at an institute which are in Suburbs? [INFO] Golden Answer: [1074] [INFO] Predicted Answer: [1] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 45, P: 0.730, R: 0.771, macro F1: 0.733, macro F1*: 0.750 -[INFO] Current time: 2125 ms, average: 3649.98 ms/ques, entity detection: 486.69 ms/ques (linking tool: 388.91 ms/ques), relation detection: 844.80 ms/ques (similarity: 611.07 ms/ques), EDG generation: 193.00 ms/ques +[INFO] Cumulative metrics, sample: 45, P: 0.730, R: 0.771, macro F1: 0.750 +[INFO] Current question consumed time: 1989 ms, average: 3458.07 ms/ques, entity detection: 476.18 ms/ques (linking tool: 383.31 ms/ques), relation detection: 828.60 ms/ques (similarity: 610.02 ms/ques), EDG generation: 191.07 ms/ques [INFO] Entity Linking P: 0.9111111111111111, R: 0.8666666666666667, micro F1: 0.8814814814814815, macro F1: 0.8883333333333334 [INFO] Relation Linking P: 0.7666666666666667, R: 0.7555555555555555, micro F1: 0.7555555555555555, macro F1: 0.7610705596107056 [INFO] Type Linking P: 0.5111111111111111, R: 0.5111111111111111, micro F1: 0.5111111111111111, macro F1: 0.5111111111111111 @@ -637,8 +637,8 @@ Question 45: Where did Whitney Wistert debut? [INFO] Golden Answer: [http://dbpedia.org/resource/Cincinnati_Reds] [INFO] Predicted Answer: [http://dbpedia.org/resource/Cincinnati_Reds] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 46, P: 0.736, R: 0.776, macro F1: 0.739, macro F1*: 0.755 -[INFO] Current time: 958 ms, average: 3591.46 ms/ques, entity detection: 479.00 ms/ques (linking tool: 382.09 ms/ques), relation detection: 835.24 ms/ques (similarity: 603.04 ms/ques), EDG generation: 188.91 ms/ques +[INFO] Cumulative metrics, sample: 46, P: 0.736, R: 0.776, macro F1: 0.755 +[INFO] Current question consumed time: 950 ms, average: 3403.57 ms/ques, entity detection: 468.70 ms/ques (linking tool: 376.54 ms/ques), relation detection: 819.17 ms/ques (similarity: 601.96 ms/ques), EDG generation: 187.00 ms/ques [INFO] Entity Linking P: 0.9130434782608695, R: 0.8695652173913043, micro F1: 0.8840579710144929, macro F1: 0.8907741251325556 [INFO] Relation Linking P: 0.7717391304347826, R: 0.7608695652173914, micro F1: 0.7608695652173914, macro F1: 0.7662658032685785 [INFO] Type Linking P: 0.5217391304347826, R: 0.5217391304347826, micro F1: 0.5217391304347826, macro F1: 0.5217391304347826 @@ -651,8 +651,8 @@ Question 46: What is the major shrine of the monarchs whose successor is Dragimi [INFO] Golden Answer: [http://dbpedia.org/resource/Albania, http://dbpedia.org/resource/Elbasan, ,@en, Monastery of St Jovan Vladimir, near@en] [INFO] Predicted Answer: [http://dbpedia.org/resource/Albania, http://dbpedia.org/resource/Elbasan, ,@en, Monastery of St Jovan Vladimir, near@en] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 47, P: 0.742, R: 0.780, macro F1: 0.745, macro F1*: 0.761 -[INFO] Current time: 1330 ms, average: 3543.34 ms/ques, entity detection: 472.02 ms/ques (linking tool: 376.60 ms/ques), relation detection: 829.53 ms/ques (similarity: 599.45 ms/ques), EDG generation: 185.55 ms/ques +[INFO] Cumulative metrics, sample: 47, P: 0.742, R: 0.780, macro F1: 0.761 +[INFO] Current question consumed time: 1222 ms, average: 3357.15 ms/ques, entity detection: 461.47 ms/ques (linking tool: 370.74 ms/ques), relation detection: 813.23 ms/ques (similarity: 598.43 ms/ques), EDG generation: 183.62 ms/ques [INFO] Entity Linking P: 0.9148936170212766, R: 0.8723404255319149, micro F1: 0.8865248226950355, macro F1: 0.8931104356636271 [INFO] Relation Linking P: 0.776595744680851, R: 0.7659574468085106, micro F1: 0.7659574468085106, macro F1: 0.7712399119589142 [INFO] Type Linking P: 0.5106382978723404, R: 0.5106382978723404, micro F1: 0.5106382978723404, macro F1: 0.5106382978723404 @@ -665,8 +665,8 @@ Question 47: How many television shows have Fox Broadcasting Company as their c [INFO] Golden Answer: [160] [INFO] Predicted Answer: [163, 160] [INFO] P: 0.5, R: 1.0, F: 0.6666666666666666 -[INFO] Cumulative metrics, sample: 48, P: 0.737, R: 0.785, macro F1: 0.743, macro F1*: 0.760 -[INFO] Current time: 2275 ms, average: 3516.92 ms/ques, entity detection: 470.71 ms/ques (linking tool: 375.67 ms/ques), relation detection: 824.15 ms/ques (similarity: 597.50 ms/ques), EDG generation: 181.98 ms/ques +[INFO] Cumulative metrics, sample: 48, P: 0.737, R: 0.785, macro F1: 0.760 +[INFO] Current question consumed time: 2612 ms, average: 3341.62 ms/ques, entity detection: 462.94 ms/ques (linking tool: 372.67 ms/ques), relation detection: 811.46 ms/ques (similarity: 596.56 ms/ques), EDG generation: 180.10 ms/ques [INFO] Entity Linking P: 0.9166666666666666, R: 0.875, micro F1: 0.888888888888889, macro F1: 0.8953488372093024 [INFO] Relation Linking P: 0.78125, R: 0.7708333333333334, micro F1: 0.7708333333333334, macro F1: 0.776006711409396 [INFO] Type Linking P: 0.5, R: 0.5, micro F1: 0.5, macro F1: 0.5 @@ -679,8 +679,8 @@ Question 48: What is the river whose mouth locations are Arctic Ocean and Laptev [INFO] Golden Answer: [http://dbpedia.org/resource/Lena_River] [INFO] Predicted Answer: [http://dbpedia.org/resource/Lena_River] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 49, P: 0.742, R: 0.789, macro F1: 0.748, macro F1*: 0.765 -[INFO] Current time: 1942 ms, average: 3484.78 ms/ques, entity detection: 471.02 ms/ques (linking tool: 374.80 ms/ques), relation detection: 816.78 ms/ques (similarity: 595.94 ms/ques), EDG generation: 179.12 ms/ques +[INFO] Cumulative metrics, sample: 49, P: 0.742, R: 0.789, macro F1: 0.765 +[INFO] Current question consumed time: 1834 ms, average: 3310.86 ms/ques, entity detection: 462.57 ms/ques (linking tool: 371.47 ms/ques), relation detection: 804.10 ms/ques (similarity: 594.76 ms/ques), EDG generation: 177.39 ms/ques [INFO] Entity Linking P: 0.9183673469387755, R: 0.8775510204081632, micro F1: 0.8911564625850341, macro F1: 0.897495361781076 [INFO] Relation Linking P: 0.7857142857142857, R: 0.7755102040816326, micro F1: 0.7755102040816326, macro F1: 0.780578898225957 [INFO] Type Linking P: 0.4897959183673469, R: 0.4897959183673469, micro F1: 0.4897959183673469, macro F1: 0.4897959183673469 @@ -693,8 +693,8 @@ Question 49: What are some wars fought when Chung Il Kwon as the country's prime [INFO] Golden Answer: [http://dbpedia.org/resource/Vietnam_War, http://dbpedia.org/resource/Blue_House_Raid, http://dbpedia.org/resource/Korean_DMZ_Conflict_(1966–69), http://dbpedia.org/resource/209th_Detachment,_2325th_Group__Unit_684_Mutiny__1] [INFO] Predicted Answer: [http://dbpedia.org/resource/Park_Chung-hee] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 50, P: 0.727, R: 0.773, macro F1: 0.733, macro F1*: 0.750 -[INFO] Current time: 1464 ms, average: 3444.36 ms/ques, entity detection: 467.74 ms/ques (linking tool: 371.26 ms/ques), relation detection: 809.78 ms/ques (similarity: 594.86 ms/ques), EDG generation: 176.10 ms/ques +[INFO] Cumulative metrics, sample: 50, P: 0.727, R: 0.773, macro F1: 0.750 +[INFO] Current question consumed time: 1409 ms, average: 3272.84 ms/ques, entity detection: 458.82 ms/ques (linking tool: 367.64 ms/ques), relation detection: 797.24 ms/ques (similarity: 593.76 ms/ques), EDG generation: 174.36 ms/ques [INFO] Entity Linking P: 0.92, R: 0.88, micro F1: 0.8933333333333334, macro F1: 0.8995555555555556 [INFO] Relation Linking P: 0.77, R: 0.76, micro F1: 0.76, macro F1: 0.764967320261438 [INFO] Type Linking P: 0.48, R: 0.48, micro F1: 0.48, macro F1: 0.48 @@ -707,8 +707,8 @@ Question 50: Where was the person born who died in Bryn Mawr Hospital? [INFO] Golden Answer: [http://dbpedia.org/resource/Colorado] [INFO] Predicted Answer: [http://dbpedia.org/resource/Colorado] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 51, P: 0.733, R: 0.778, macro F1: 0.739, macro F1*: 0.755 -[INFO] Current time: 1106 ms, average: 3398.51 ms/ques, entity detection: 461.65 ms/ques (linking tool: 365.73 ms/ques), relation detection: 800.25 ms/ques (similarity: 587.37 ms/ques), EDG generation: 172.94 ms/ques +[INFO] Cumulative metrics, sample: 51, P: 0.733, R: 0.778, macro F1: 0.755 +[INFO] Current question consumed time: 1013 ms, average: 3228.53 ms/ques, entity detection: 452.71 ms/ques (linking tool: 362.08 ms/ques), relation detection: 787.80 ms/ques (similarity: 586.14 ms/ques), EDG generation: 171.25 ms/ques [INFO] Entity Linking P: 0.9215686274509803, R: 0.8823529411764706, micro F1: 0.8954248366013072, macro F1: 0.90153452685422 [INFO] Relation Linking P: 0.7549019607843137, R: 0.7450980392156863, micro F1: 0.7450980392156863, macro F1: 0.7499679610406255 [INFO] Type Linking P: 0.49019607843137253, R: 0.49019607843137253, micro F1: 0.49019607843137253, macro F1: 0.49019607843137253 @@ -721,8 +721,8 @@ Question 51: What is the faith of the Ajith Perera and religious affiliation of [INFO] Golden Answer: [http://dbpedia.org/resource/Buddhism] [INFO] Predicted Answer: [http://dbpedia.org/resource/Buddhism] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 52, P: 0.738, R: 0.782, macro F1: 0.744, macro F1*: 0.759 -[INFO] Current time: 1731 ms, average: 3366.44 ms/ques, entity detection: 464.21 ms/ques (linking tool: 366.81 ms/ques), relation detection: 789.92 ms/ques (similarity: 579.21 ms/ques), EDG generation: 170.63 ms/ques +[INFO] Cumulative metrics, sample: 52, P: 0.738, R: 0.782, macro F1: 0.759 +[INFO] Current question consumed time: 1645 ms, average: 3198.08 ms/ques, entity detection: 454.50 ms/ques (linking tool: 362.81 ms/ques), relation detection: 777.38 ms/ques (similarity: 578.04 ms/ques), EDG generation: 169.08 ms/ques [INFO] Entity Linking P: 0.9230769230769231, R: 0.8846153846153846, micro F1: 0.8974358974358976, macro F1: 0.9034369885433716 [INFO] Relation Linking P: 0.7596153846153846, R: 0.75, micro F1: 0.75, macro F1: 0.7547770700636942 [INFO] Type Linking P: 0.5, R: 0.5, micro F1: 0.5, macro F1: 0.5 @@ -735,8 +735,8 @@ Question 52: Who was in military unit which played the role of Air interdiction? [INFO] Golden Answer: [http://dbpedia.org/resource/John_Glenn, http://dbpedia.org/resource/Edward_Robert_Sellstrom, http://dbpedia.org/resource/Newton_Henry_Mason, http://dbpedia.org/resource/Christopher_Magee_(fighter_pilot), http://dbpedia.org/resource/William_Chesarek, http://dbpedia.org/resource/James_E._Swett, http://dbpedia.org/resource/Manuel_Rivera,_Jr., http://dbpedia.org/resource/Siegfried_Betke, http://dbpedia.org/resource/Rolf_Thomsen, http://dbpedia.org/resource/Rudolf_Schmidt_(Major), http://dbpedia.org/resource/Viktor_von_Loßberg, http://dbpedia.org/resource/Paul_Claas, http://dbpedia.org/resource/Dilbagh_Singh, http://dbpedia.org/resource/Paul_G._Baker, http://dbpedia.org/resource/Kenneth_A._Walsh, http://dbpedia.org/resource/John_F._Bolt, http://dbpedia.org/resource/Robert_M._Hanson, http://dbpedia.org/resource/Kenneth_L._Reusser, http://dbpedia.org/resource/Donald_Conroy, http://dbpedia.org/resource/James_N._Cupp, http://dbpedia.org/resource/Jefferson_J._DeBlanc, http://dbpedia.org/resource/Joe_Foss, http://dbpedia.org/resource/Jerry_Coleman__Jerry_Coleman__1, http://dbpedia.org/resource/Henry_T._Elrod, http://dbpedia.org/resource/Michael_J._Estocin, http://dbpedia.org/resource/Scott_Speicher, http://dbpedia.org/resource/Matt_Santos____1, http://dbpedia.org/resource/Oliver_Mitchell, http://dbpedia.org/resource/Lee_T._Lasseter, http://dbpedia.org/resource/Art_Nalls, http://dbpedia.org/resource/William_C._Lawe, http://dbpedia.org/resource/George_M._Campbell, http://dbpedia.org/resource/Oswald_J._Gaynier, http://dbpedia.org/resource/Harold_John_Ellison, http://dbpedia.org/resource/John_P._Gray, http://dbpedia.org/resource/George_H._Gay,_Jr., http://dbpedia.org/resource/William_W._Creamer, http://dbpedia.org/resource/Henry_R._Kenyon,_Jr., http://dbpedia.org/resource/Jack_W._Wilke] [INFO] Predicted Answer: [http://dbpedia.org/resource/Erhard_Milch, http://dbpedia.org/resource/Gustav_Francsi, http://dbpedia.org/resource/Erich_Kästner_(World_War_I_veteran), http://dbpedia.org/resource/Eugene_Seeley_Coler, http://dbpedia.org/resource/Russell_C._Davis, http://dbpedia.org/resource/Alfred_Masters, http://dbpedia.org/resource/Jacob_C._Vouza, http://dbpedia.org/resource/Walter_Carl_Wann,_Jr., http://dbpedia.org/resource/John_Fitzgerald_(Medal_of_Honor), http://dbpedia.org/resource/Kenneth_W._Durant, http://dbpedia.org/resource/Dennis_T._Gorski, http://dbpedia.org/resource/Carlton_W._Kent, http://dbpedia.org/resource/John_H._Ebersole, http://dbpedia.org/resource/Herschel_Rosenthal, http://dbpedia.org/resource/Robley_D._Evans] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 53, P: 0.724, R: 0.767, macro F1: 0.730, macro F1*: 0.745 -[INFO] Current time: 6235 ms, average: 3420.57 ms/ques, entity detection: 457.85 ms/ques (linking tool: 361.17 ms/ques), relation detection: 834.02 ms/ques (similarity: 596.38 ms/ques), EDG generation: 169.26 ms/ques +[INFO] Cumulative metrics, sample: 53, P: 0.724, R: 0.767, macro F1: 0.745 +[INFO] Current question consumed time: 6046 ms, average: 3251.83 ms/ques, entity detection: 447.98 ms/ques (linking tool: 357.06 ms/ques), relation detection: 821.34 ms/ques (similarity: 594.87 ms/ques), EDG generation: 167.66 ms/ques [INFO] Entity Linking P: 0.9245283018867925, R: 0.8867924528301887, micro F1: 0.8993710691823901, macro F1: 0.9052672955974843 [INFO] Relation Linking P: 0.7578616352201257, R: 0.7547169811320755, micro F1: 0.750943396226415, macro F1: 0.7562860393049072 [INFO] Type Linking P: 0.49056603773584906, R: 0.49056603773584906, micro F1: 0.49056603773584906, macro F1: 0.49056603773584906 @@ -749,8 +749,8 @@ Question 53: Who are the current members of Insects vs Robots? [INFO] Golden Answer: [http://dbpedia.org/resource/Willie_Nelson, Jeff Smith@en, Tony Peluso@en, Nikita Sorokin@en, Milo Gonzalez@en] [INFO] Predicted Answer: [http://dbpedia.org/resource/Willie_Nelson, Jeff Smith@en, Tony Peluso@en, Nikita Sorokin@en, Milo Gonzalez@en] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 54, P: 0.729, R: 0.772, macro F1: 0.735, macro F1*: 0.750 -[INFO] Current time: 656 ms, average: 3369.37 ms/ques, entity detection: 452.67 ms/ques (linking tool: 357.61 ms/ques), relation detection: 820.43 ms/ques (similarity: 586.67 ms/ques), EDG generation: 166.31 ms/ques +[INFO] Cumulative metrics, sample: 54, P: 0.729, R: 0.772, macro F1: 0.750 +[INFO] Current question consumed time: 655 ms, average: 3203.74 ms/ques, entity detection: 442.91 ms/ques (linking tool: 353.67 ms/ques), relation detection: 807.85 ms/ques (similarity: 585.02 ms/ques), EDG generation: 164.85 ms/ques [INFO] Entity Linking P: 0.9259259259259259, R: 0.8888888888888888, micro F1: 0.9012345679012347, macro F1: 0.9070294784580497 [INFO] Relation Linking P: 0.7623456790123456, R: 0.7592592592592593, micro F1: 0.7555555555555555, macro F1: 0.7607993388926452 [INFO] Type Linking P: 0.5, R: 0.5, micro F1: 0.5, macro F1: 0.5 @@ -763,8 +763,8 @@ Question 54: Is Ombla originate in Croatia? [INFO] Golden Answer: [true] [INFO] Predicted Answer: [true] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 55, P: 0.734, R: 0.776, macro F1: 0.740, macro F1*: 0.754 -[INFO] Current time: 4977 ms, average: 3398.62 ms/ques, entity detection: 465.18 ms/ques (linking tool: 369.64 ms/ques), relation detection: 858.82 ms/ques (similarity: 605.96 ms/ques), EDG generation: 163.44 ms/ques +[INFO] Cumulative metrics, sample: 55, P: 0.734, R: 0.776, macro F1: 0.754 +[INFO] Current question consumed time: 4906 ms, average: 3234.69 ms/ques, entity detection: 456.20 ms/ques (linking tool: 366.42 ms/ques), relation detection: 845.78 ms/ques (similarity: 603.96 ms/ques), EDG generation: 161.85 ms/ques [INFO] Entity Linking P: 0.9272727272727272, R: 0.8909090909090909, micro F1: 0.9030303030303031, macro F1: 0.9087272727272726 [INFO] Relation Linking P: 0.7484848484848484, R: 0.7454545454545455, micro F1: 0.7418181818181818, macro F1: 0.7469666236400516 [INFO] Type Linking P: 0.509090909090909, R: 0.509090909090909, micro F1: 0.509090909090909, macro F1: 0.509090909090909 @@ -777,8 +777,8 @@ Question 55: Greater Napanee is the home town of what people? [INFO] Golden Answer: [http://dbpedia.org/resource/Deryck_Whibley, http://dbpedia.org/resource/Avril_Lavigne] [INFO] Predicted Answer: [http://dbpedia.org/resource/St._Lawrence_Saints_women's_ice_hockey, http://dbpedia.org/resource/Deryck_Whibley, http://dbpedia.org/resource/Avril_Lavigne] [INFO] P: 0.6666666666666666, R: 1.0, F: 0.8 -[INFO] Cumulative metrics, sample: 56, P: 0.733, R: 0.780, macro F1: 0.741, macro F1*: 0.756 -[INFO] Current time: 2090 ms, average: 3375.25 ms/ques, entity detection: 460.14 ms/ques (linking tool: 365.23 ms/ques), relation detection: 858.66 ms/ques (similarity: 612.30 ms/ques), EDG generation: 160.84 ms/ques +[INFO] Cumulative metrics, sample: 56, P: 0.733, R: 0.780, macro F1: 0.756 +[INFO] Current question consumed time: 2078 ms, average: 3214.04 ms/ques, entity detection: 451.14 ms/ques (linking tool: 361.82 ms/ques), relation detection: 845.18 ms/ques (similarity: 610.46 ms/ques), EDG generation: 159.25 ms/ques [INFO] Entity Linking P: 0.9285714285714286, R: 0.8928571428571429, micro F1: 0.9047619047619049, macro F1: 0.9103641456582633 [INFO] Relation Linking P: 0.7529761904761905, R: 0.75, micro F1: 0.7464285714285713, macro F1: 0.7514851485148514 [INFO] Type Linking P: 0.5, R: 0.5, micro F1: 0.5, macro F1: 0.5 @@ -791,8 +791,8 @@ Question 56: What place did Edwin Adams die at, which gave birth to William A Pu [INFO] Golden Answer: [http://dbpedia.org/resource/Connecticut] [INFO] Predicted Answer: [http://dbpedia.org/resource/Connecticut] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 57, P: 0.737, R: 0.784, macro F1: 0.745, macro F1*: 0.760 -[INFO] Current time: 4740 ms, average: 3399.19 ms/ques, entity detection: 459.23 ms/ques (linking tool: 363.16 ms/ques), relation detection: 861.95 ms/ques (similarity: 622.98 ms/ques), EDG generation: 158.70 ms/ques +[INFO] Cumulative metrics, sample: 57, P: 0.737, R: 0.784, macro F1: 0.760 +[INFO] Current question consumed time: 4554 ms, average: 3237.54 ms/ques, entity detection: 450.16 ms/ques (linking tool: 359.74 ms/ques), relation detection: 848.70 ms/ques (similarity: 621.02 ms/ques), EDG generation: 157.32 ms/ques [INFO] Entity Linking P: 0.9298245614035088, R: 0.8947368421052632, micro F1: 0.9064327485380118, macro F1: 0.9119433198380568 [INFO] Relation Linking P: 0.7485380116959064, R: 0.7456140350877193, micro F1: 0.7421052631578947, macro F1: 0.7470731623579497 [INFO] Type Linking P: 0.49122807017543857, R: 0.49122807017543857, micro F1: 0.49122807017543857, macro F1: 0.49122807017543857 @@ -805,8 +805,8 @@ Question 57: Name the nationalteam of Trn Vit Hng ? [INFO] Golden Answer: [http://dbpedia.org/resource/Vietnam] [INFO] Predicted Answer: [Трн@en] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 58, P: 0.725, R: 0.770, macro F1: 0.732, macro F1*: 0.747 -[INFO] Current time: 758 ms, average: 3353.67 ms/ques, entity detection: 453.88 ms/ques (linking tool: 358.29 ms/ques), relation detection: 850.52 ms/ques (similarity: 614.50 ms/ques), EDG generation: 156.16 ms/ques +[INFO] Cumulative metrics, sample: 58, P: 0.725, R: 0.770, macro F1: 0.747 +[INFO] Current question consumed time: 745 ms, average: 3194.57 ms/ques, entity detection: 445.09 ms/ques (linking tool: 355.17 ms/ques), relation detection: 837.84 ms/ques (similarity: 612.71 ms/ques), EDG generation: 154.79 ms/ques [INFO] Entity Linking P: 0.9137931034482759, R: 0.8793103448275862, micro F1: 0.8908045977011495, macro F1: 0.8962201591511937 [INFO] Relation Linking P: 0.735632183908046, R: 0.7327586206896551, micro F1: 0.7293103448275862, macro F1: 0.7341925905931573 [INFO] Type Linking P: 0.5, R: 0.5, micro F1: 0.5, macro F1: 0.5 @@ -819,8 +819,8 @@ Question 58: Which designer of RENFE also owns Renaissance Center ? [INFO] Golden Answer: [http://dbpedia.org/resource/General_Motors] [INFO] Predicted Answer: [http://dbpedia.org/resource/General_Motors] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 59, P: 0.729, R: 0.774, macro F1: 0.737, macro F1*: 0.751 -[INFO] Current time: 3352 ms, average: 3353.64 ms/ques, entity detection: 454.66 ms/ques (linking tool: 357.95 ms/ques), relation detection: 854.93 ms/ques (similarity: 614.90 ms/ques), EDG generation: 153.78 ms/ques +[INFO] Cumulative metrics, sample: 59, P: 0.729, R: 0.774, macro F1: 0.751 +[INFO] Current question consumed time: 2873 ms, average: 3189.12 ms/ques, entity detection: 445.97 ms/ques (linking tool: 354.85 ms/ques), relation detection: 840.88 ms/ques (similarity: 612.97 ms/ques), EDG generation: 152.66 ms/ques [INFO] Entity Linking P: 0.9152542372881356, R: 0.8813559322033898, micro F1: 0.8926553672316385, macro F1: 0.8979852894147745 [INFO] Relation Linking P: 0.7401129943502824, R: 0.7372881355932204, micro F1: 0.7338983050847457, macro F1: 0.7386978643419646 [INFO] Type Linking P: 0.5084745762711864, R: 0.5084745762711864, micro F1: 0.5084745762711864, macro F1: 0.5084745762711864 @@ -833,8 +833,8 @@ Question 59: Whihc uni did Joshua A. Siegel attend ? [INFO] Golden Answer: [http://dbpedia.org/resource/Chicago, http://dbpedia.org/resource/University_at_Buffalo, http://dbpedia.org/resource/Northwestern_University] [INFO] Predicted Answer: [http://dbpedia.org/resource/Chicago, http://dbpedia.org/resource/University_at_Buffalo, http://dbpedia.org/resource/Northwestern_University] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 60, P: 0.734, R: 0.778, macro F1: 0.741, macro F1*: 0.755 -[INFO] Current time: 973 ms, average: 3313.97 ms/ques, entity detection: 450.03 ms/ques (linking tool: 353.10 ms/ques), relation detection: 842.75 ms/ques (similarity: 605.73 ms/ques), EDG generation: 151.37 ms/ques +[INFO] Cumulative metrics, sample: 60, P: 0.734, R: 0.778, macro F1: 0.755 +[INFO] Current question consumed time: 894 ms, average: 3150.87 ms/ques, entity detection: 441.63 ms/ques (linking tool: 350.23 ms/ques), relation detection: 828.70 ms/ques (similarity: 603.80 ms/ques), EDG generation: 150.13 ms/ques [INFO] Entity Linking P: 0.9166666666666666, R: 0.8833333333333333, micro F1: 0.8944444444444445, macro F1: 0.8996913580246914 [INFO] Relation Linking P: 0.7444444444444444, R: 0.7416666666666667, micro F1: 0.7383333333333333, macro F1: 0.7430529595015576 [INFO] Type Linking P: 0.5166666666666667, R: 0.5166666666666667, micro F1: 0.5166666666666667, macro F1: 0.5166666666666667 @@ -847,8 +847,8 @@ Question 60: How many books are written by the author who is famous for Novelas [INFO] Golden Answer: [4] [INFO] Predicted Answer: [1] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 61, P: 0.722, R: 0.765, macro F1: 0.729, macro F1*: 0.743 -[INFO] Current time: 1685 ms, average: 3287.26 ms/ques, entity detection: 444.72 ms/ques (linking tool: 348.48 ms/ques), relation detection: 837.51 ms/ques (similarity: 604.52 ms/ques), EDG generation: 149.46 ms/ques +[INFO] Cumulative metrics, sample: 61, P: 0.722, R: 0.765, macro F1: 0.743 +[INFO] Current question consumed time: 1716 ms, average: 3127.34 ms/ques, entity detection: 436.18 ms/ques (linking tool: 345.51 ms/ques), relation detection: 823.89 ms/ques (similarity: 602.87 ms/ques), EDG generation: 148.20 ms/ques [INFO] Entity Linking P: 0.9180327868852459, R: 0.8852459016393442, micro F1: 0.8961748633879782, macro F1: 0.9013412816691505 [INFO] Relation Linking P: 0.7404371584699453, R: 0.7377049180327869, micro F1: 0.7344262295081967, macro F1: 0.7390685130753614 [INFO] Type Linking P: 0.5081967213114754, R: 0.5081967213114754, micro F1: 0.5081967213114754, macro F1: 0.5081967213114754 @@ -861,8 +861,8 @@ Question 61: Name the rivers who originate from Essex? [INFO] Golden Answer: [http://dbpedia.org/resource/River_Stour,_Suffolk, http://dbpedia.org/resource/River_Thames] [INFO] Predicted Answer: [http://dbpedia.org/resource/Jimpster, http://dbpedia.org/resource/Symeon_Cosburn, http://dbpedia.org/resource/Disco_Inferno_(band), http://dbpedia.org/resource/Jamie_Robertson, http://dbpedia.org/resource/Ceephax_Acid_Crew, http://dbpedia.org/resource/I_Level, http://dbpedia.org/resource/Magistrates_(band), http://dbpedia.org/resource/Sonic_Violence, http://dbpedia.org/resource/Producers_with_Computers, http://dbpedia.org/resource/Darwyn_(band), http://dbpedia.org/resource/Seven_Summers, http://dbpedia.org/resource/Underworld_(band), http://dbpedia.org/resource/Modern_English_(band), http://dbpedia.org/resource/Warning_(UK_band), http://dbpedia.org/resource/Beans_On_Toast_(musician), http://dbpedia.org/resource/Blue_October_(British_band), http://dbpedia.org/resource/Jet_Black, http://dbpedia.org/resource/John_Farnham, http://dbpedia.org/resource/Nitzer_Ebb, http://dbpedia.org/resource/Nothing_But_Thieves, http://dbpedia.org/resource/Ernest_Lough, http://dbpedia.org/resource/Engerica, http://dbpedia.org/resource/The_Wrongstar_Society, http://dbpedia.org/resource/Absent_Kid, http://dbpedia.org/resource/Five_Star, http://dbpedia.org/resource/John_Smith_(musician), http://dbpedia.org/resource/Keith_West, http://dbpedia.org/resource/Morning_Parade, http://dbpedia.org/resource/Scarlet_Party, http://dbpedia.org/resource/Kate_Walsh_(singer), http://dbpedia.org/resource/Koopa_(band), http://dbpedia.org/resource/Twenty_Twenty_(band), http://dbpedia.org/resource/Harry_James_(entertainer), http://dbpedia.org/resource/N-Joi, http://dbpedia.org/resource/Baddies, http://dbpedia.org/resource/The_Symbols, http://dbpedia.org/resource/The_Ejected, http://dbpedia.org/resource/Dave_Gahan, http://dbpedia.org/resource/Norman_Watt-Roy, http://dbpedia.org/resource/Collapsed_Lung_(band), http://dbpedia.org/resource/Headswim, http://dbpedia.org/resource/Churchfitters, http://dbpedia.org/resource/David_Woodcock_(musician), http://dbpedia.org/resource/Rohan_By_Nightfall_(band), http://dbpedia.org/resource/The_Paramounts, http://dbpedia.org/resource/Keith_Christmas, http://dbpedia.org/resource/Fei_Comodo, http://dbpedia.org/resource/Empirion, http://dbpedia.org/resource/Shelly_Poole, http://dbpedia.org/resource/Kenai_(band), http://dbpedia.org/resource/Redjetson, http://dbpedia.org/resource/DJ_Cable] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 62, P: 0.710, R: 0.753, macro F1: 0.717, macro F1*: 0.731 -[INFO] Current time: 3466 ms, average: 3290.15 ms/ques, entity detection: 444.94 ms/ques (linking tool: 348.77 ms/ques), relation detection: 844.48 ms/ques (similarity: 617.45 ms/ques), EDG generation: 147.21 ms/ques +[INFO] Cumulative metrics, sample: 62, P: 0.710, R: 0.753, macro F1: 0.731 +[INFO] Current question consumed time: 3357 ms, average: 3131.05 ms/ques, entity detection: 435.84 ms/ques (linking tool: 345.32 ms/ques), relation detection: 829.81 ms/ques (similarity: 618.60 ms/ques), EDG generation: 145.81 ms/ques [INFO] Entity Linking P: 0.9193548387096774, R: 0.8870967741935484, micro F1: 0.8978494623655915, macro F1: 0.9029377880184332 [INFO] Relation Linking P: 0.7284946236559139, R: 0.7258064516129032, micro F1: 0.7225806451612903, macro F1: 0.7271480531870491 [INFO] Type Linking P: 0.5, R: 0.5, micro F1: 0.5, macro F1: 0.5 @@ -875,8 +875,8 @@ Question 62: Companies founded in Taiwan have typically which divisions? [INFO] Golden Answer: [http://dbpedia.org/resource/Asia, http://dbpedia.org/resource/Asia-Pacific, http://dbpedia.org/resource/Europe, http://dbpedia.org/resource/North_America] [INFO] Predicted Answer: [http://dbpedia.org/resource/Guishan_Island_(Yilan), http://dbpedia.org/resource/C215_Viaduct, http://dbpedia.org/resource/Dabajian_Mountain, http://dbpedia.org/resource/Nenggao_Mountain, http://dbpedia.org/resource/Tao_Mountain, http://dbpedia.org/resource/Mount_Banping, http://dbpedia.org/resource/Shoushan_(Kaohsiung), http://dbpedia.org/resource/Yushan_(mountain), http://dbpedia.org/resource/Hehuanshan, http://dbpedia.org/resource/Danjiang_Bridge, http://dbpedia.org/resource/Longteng_Bridge, http://dbpedia.org/resource/Xueshan, http://dbpedia.org/resource/Nanhu_Mountain, http://dbpedia.org/resource/Mount_Guanyin_(New_Taipei), http://dbpedia.org/resource/Taipei_Bridge] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 63, P: 0.699, R: 0.741, macro F1: 0.706, macro F1*: 0.719 -[INFO] Current time: 7963 ms, average: 3364.32 ms/ques, entity detection: 461.00 ms/ques (linking tool: 364.43 ms/ques), relation detection: 871.52 ms/ques (similarity: 651.02 ms/ques), EDG generation: 145.11 ms/ques +[INFO] Cumulative metrics, sample: 63, P: 0.699, R: 0.741, macro F1: 0.719 +[INFO] Current question consumed time: 7997 ms, average: 3208.29 ms/ques, entity detection: 453.37 ms/ques (linking tool: 362.37 ms/ques), relation detection: 858.05 ms/ques (similarity: 652.54 ms/ques), EDG generation: 143.70 ms/ques [INFO] Entity Linking P: 0.9206349206349206, R: 0.8888888888888888, micro F1: 0.8994708994708995, macro F1: 0.9044834307992203 [INFO] Relation Linking P: 0.7169312169312169, R: 0.7142857142857143, micro F1: 0.711111111111111, macro F1: 0.7156060205967784 [INFO] Type Linking P: 0.49206349206349204, R: 0.49206349206349204, micro F1: 0.49206349206349204, macro F1: 0.49206349206349204 @@ -889,8 +889,8 @@ Question 63: Did Timm Gunn guest in Sunrise (HIMYM)? [INFO] Golden Answer: [true] [INFO] Predicted Answer: [true] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 64, P: 0.704, R: 0.745, macro F1: 0.711, macro F1*: 0.724 -[INFO] Current time: 878 ms, average: 3325.47 ms/ques, entity detection: 456.58 ms/ques (linking tool: 359.91 ms/ques), relation detection: 861.58 ms/ques (similarity: 643.27 ms/ques), EDG generation: 143.05 ms/ques +[INFO] Cumulative metrics, sample: 64, P: 0.704, R: 0.745, macro F1: 0.724 +[INFO] Current question consumed time: 865 ms, average: 3171.67 ms/ques, entity detection: 448.89 ms/ques (linking tool: 357.62 ms/ques), relation detection: 848.55 ms/ques (similarity: 645.05 ms/ques), EDG generation: 141.66 ms/ques [INFO] Entity Linking P: 0.90625, R: 0.875, micro F1: 0.8854166666666667, macro F1: 0.8903508771929824 [INFO] Relation Linking P: 0.7057291666666666, R: 0.703125, micro F1: 0.7, macro F1: 0.7044246765249539 [INFO] Type Linking P: 0.5, R: 0.5, micro F1: 0.5, macro F1: 0.5 @@ -903,8 +903,8 @@ Question 64: to which university did both, the Stanford Cardinal and Gerald Reav [INFO] Golden Answer: [http://dbpedia.org/resource/Stanford_University] [INFO] Predicted Answer: [http://dbpedia.org/resource/Stanford_University] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 65, P: 0.708, R: 0.749, macro F1: 0.715, macro F1*: 0.728 -[INFO] Current time: 4273 ms, average: 3340.05 ms/ques, entity detection: 454.83 ms/ques (linking tool: 357.35 ms/ques), relation detection: 869.83 ms/ques (similarity: 657.75 ms/ques), EDG generation: 141.66 ms/ques +[INFO] Cumulative metrics, sample: 65, P: 0.708, R: 0.749, macro F1: 0.728 +[INFO] Current question consumed time: 4115 ms, average: 3186.18 ms/ques, entity detection: 446.48 ms/ques (linking tool: 354.72 ms/ques), relation detection: 856.58 ms/ques (similarity: 659.57 ms/ques), EDG generation: 140.45 ms/ques [INFO] Entity Linking P: 0.9076923076923077, R: 0.8769230769230769, micro F1: 0.8871794871794872, macro F1: 0.8920424403183024 [INFO] Relation Linking P: 0.7102564102564102, R: 0.7076923076923077, micro F1: 0.7046153846153845, macro F1: 0.7089720406176102 [INFO] Type Linking P: 0.49230769230769234, R: 0.49230769230769234, micro F1: 0.49230769230769234, macro F1: 0.49230769230769234 @@ -917,8 +917,8 @@ Question 65: Name the nearest city to David W. Brown House ? [INFO] Golden Answer: [http://dbpedia.org/resource/Colorado, http://dbpedia.org/resource/Arapahoe_County,_Colorado, http://dbpedia.org/resource/Englewood,_Colorado] [INFO] Predicted Answer: [] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 66, P: 0.697, R: 0.737, macro F1: 0.704, macro F1*: 0.717 -[INFO] Current time: 531 ms, average: 3297.48 ms/ques, entity detection: 449.74 ms/ques (linking tool: 353.02 ms/ques), relation detection: 858.00 ms/ques (similarity: 648.64 ms/ques), EDG generation: 139.67 ms/ques +[INFO] Cumulative metrics, sample: 66, P: 0.697, R: 0.737, macro F1: 0.717 +[INFO] Current question consumed time: 475 ms, average: 3145.11 ms/ques, entity detection: 441.50 ms/ques (linking tool: 350.39 ms/ques), relation detection: 844.94 ms/ques (similarity: 650.39 ms/ques), EDG generation: 138.44 ms/ques [INFO] Entity Linking P: 0.8939393939393939, R: 0.8636363636363636, micro F1: 0.8737373737373738, macro F1: 0.8785266457680251 [INFO] Relation Linking P: 0.6994949494949495, R: 0.696969696969697, micro F1: 0.6939393939393939, macro F1: 0.6982300400021918 [INFO] Type Linking P: 0.5, R: 0.5, micro F1: 0.5, macro F1: 0.5 @@ -931,8 +931,8 @@ Question 66: where did Deion Sanders debut? [INFO] Golden Answer: [http://dbpedia.org/resource/New_York_Yankees, Atlanta Falcons@en] [INFO] Predicted Answer: [http://dbpedia.org/resource/New_York_Yankees, Atlanta Falcons@en] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 67, P: 0.702, R: 0.741, macro F1: 0.709, macro F1*: 0.721 -[INFO] Current time: 1149 ms, average: 3265.42 ms/ques, entity detection: 444.85 ms/ques (linking tool: 349.45 ms/ques), relation detection: 853.04 ms/ques (similarity: 643.82 ms/ques), EDG generation: 137.70 ms/ques +[INFO] Cumulative metrics, sample: 67, P: 0.702, R: 0.741, macro F1: 0.721 +[INFO] Current question consumed time: 1080 ms, average: 3114.28 ms/ques, entity detection: 436.70 ms/ques (linking tool: 346.84 ms/ques), relation detection: 839.97 ms/ques (similarity: 645.46 ms/ques), EDG generation: 136.49 ms/ques [INFO] Entity Linking P: 0.8955223880597015, R: 0.8656716417910447, micro F1: 0.8756218905472638, macro F1: 0.8803440424993676 [INFO] Relation Linking P: 0.7039800995024875, R: 0.7014925373134329, micro F1: 0.6985074626865672, macro F1: 0.7027341170254919 [INFO] Type Linking P: 0.5074626865671642, R: 0.5074626865671642, micro F1: 0.5074626865671642, macro F1: 0.5074626865671642 @@ -945,8 +945,8 @@ Question 67: What cities are nearer to Fuountain Lake Farm? [INFO] Golden Answer: [http://dbpedia.org/resource/Montello,_Wisconsin, http://dbpedia.org/resource/North_America, http://dbpedia.org/resource/United_States, http://dbpedia.org/resource/Wisconsin] [INFO] Predicted Answer: [http://dbpedia.org/resource/Montello,_Wisconsin] [INFO] P: 1.0, R: 0.25, F: 0.4 -[INFO] Cumulative metrics, sample: 68, P: 0.706, R: 0.734, macro F1: 0.704, macro F1*: 0.720 -[INFO] Current time: 935 ms, average: 3231.15 ms/ques, entity detection: 439.78 ms/ques (linking tool: 345.18 ms/ques), relation detection: 844.50 ms/ques (similarity: 639.37 ms/ques), EDG generation: 135.82 ms/ques +[INFO] Cumulative metrics, sample: 68, P: 0.706, R: 0.734, macro F1: 0.720 +[INFO] Current question consumed time: 920 ms, average: 3082.01 ms/ques, entity detection: 431.76 ms/ques (linking tool: 342.56 ms/ques), relation detection: 831.60 ms/ques (similarity: 640.88 ms/ques), EDG generation: 134.62 ms/ques [INFO] Entity Linking P: 0.8970588235294118, R: 0.8676470588235294, micro F1: 0.877450980392157, macro F1: 0.882107843137255 [INFO] Relation Linking P: 0.7083333333333333, R: 0.7058823529411765, micro F1: 0.7029411764705882, macro F1: 0.707105719237435 [INFO] Type Linking P: 0.5, R: 0.5, micro F1: 0.5, macro F1: 0.5 @@ -959,8 +959,8 @@ Question 68: Who are the characters of Batman Live? [INFO] Golden Answer: [http://dbpedia.org/resource/Alfred_Pennyworth, http://dbpedia.org/resource/Batman, http://dbpedia.org/resource/Catwoman, http://dbpedia.org/resource/Harley_Quinn, http://dbpedia.org/resource/James_Gordon_(comics), http://dbpedia.org/resource/Joker_(comics), http://dbpedia.org/resource/Penguin_(comics), http://dbpedia.org/resource/Poison_Ivy_(comics), http://dbpedia.org/resource/Riddler, http://dbpedia.org/resource/Robin_(comics), http://dbpedia.org/resource/Scarecrow_(DC_Comics), http://dbpedia.org/resource/Two-Face] [INFO] Predicted Answer: [http://dbpedia.org/resource/List_of_DC_Comics_characters_named_Batman, http://dbpedia.org/resource/Batman_Live] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 69, P: 0.696, R: 0.724, macro F1: 0.694, macro F1*: 0.710 -[INFO] Current time: 3890 ms, average: 3240.70 ms/ques, entity detection: 441.26 ms/ques (linking tool: 345.64 ms/ques), relation detection: 855.86 ms/ques (similarity: 643.38 ms/ques), EDG generation: 134.03 ms/ques +[INFO] Cumulative metrics, sample: 69, P: 0.696, R: 0.724, macro F1: 0.710 +[INFO] Current question consumed time: 3535 ms, average: 3088.58 ms/ques, entity detection: 433.23 ms/ques (linking tool: 343.01 ms/ques), relation detection: 842.38 ms/ques (similarity: 644.61 ms/ques), EDG generation: 132.83 ms/ques [INFO] Entity Linking P: 0.8840579710144928, R: 0.855072463768116, micro F1: 0.8647342995169083, macro F1: 0.8693236714975845 [INFO] Relation Linking P: 0.7125603864734299, R: 0.7101449275362319, micro F1: 0.7072463768115942, macro F1: 0.7113506065303511 [INFO] Type Linking P: 0.5072463768115942, R: 0.5072463768115942, micro F1: 0.5072463768115942, macro F1: 0.5072463768115942 @@ -973,8 +973,8 @@ Question 69: Which artists trained in San Francisco Art Institute? [INFO] Golden Answer: [http://dbpedia.org/resource/Annie_Leibovitz, http://dbpedia.org/resource/Peter_Reginato, http://dbpedia.org/resource/Carlos_Vierra, http://dbpedia.org/resource/Gregory_Gillespie, http://dbpedia.org/resource/Helen_Hyde, http://dbpedia.org/resource/James_Zar, http://dbpedia.org/resource/Walter_Kuhlman, http://dbpedia.org/resource/Jason_Rhoades, http://dbpedia.org/resource/Jess_Collins, http://dbpedia.org/resource/Manuel_Neri, http://dbpedia.org/resource/Michael_Heizer, http://dbpedia.org/resource/Ernest_Peixotto, http://dbpedia.org/resource/Ralph_Stackpole, http://dbpedia.org/resource/Emily_Carr, http://dbpedia.org/resource/William_T._Wiley, http://dbpedia.org/resource/Robert_H._Hudson, http://dbpedia.org/resource/Edith_Heath, http://dbpedia.org/resource/Lynn_Marie_Kirby, http://dbpedia.org/resource/Sarah_Cain, http://dbpedia.org/resource/Paul_McCarthy, http://dbpedia.org/resource/Robert_Graham_(sculptor), http://dbpedia.org/resource/Carlos_Villa, http://dbpedia.org/resource/Deborah_Remington, http://dbpedia.org/resource/Victor_Moscoso, http://dbpedia.org/resource/Karl_Kasten, http://dbpedia.org/resource/Jim_Franklin_(artist), http://dbpedia.org/resource/Ray_Strong, http://dbpedia.org/resource/Joan_Brown, http://dbpedia.org/resource/Neil_Williams_(artist), http://dbpedia.org/resource/Ronald_Davis, http://dbpedia.org/resource/Ronnie_Landfield, http://dbpedia.org/resource/Tullio_DeSantis, http://dbpedia.org/resource/Peter_Forakis, http://dbpedia.org/resource/Reminisce_(artist), http://dbpedia.org/resource/Peter_Kovál, http://dbpedia.org/resource/Gladys_Rockmore_Davis, http://dbpedia.org/resource/Thom_Merrick, http://dbpedia.org/resource/Lin_Evola, http://dbpedia.org/resource/T._C._Cannon, http://dbpedia.org/resource/Arthur_Frank_Mathews, http://dbpedia.org/resource/Adelaide_Hanscom_Leeson, http://dbpedia.org/resource/Doris_Lee, http://dbpedia.org/resource/Daniela_Franco, http://dbpedia.org/resource/Vena_Kava] [INFO] Predicted Answer: [http://dbpedia.org/resource/Annie_Leibovitz, http://dbpedia.org/resource/Peter_Reginato, http://dbpedia.org/resource/Adolph_Treidler, http://dbpedia.org/resource/Carlos_Vierra, http://dbpedia.org/resource/Gregory_Gillespie, http://dbpedia.org/resource/David_Ireland_(artist), http://dbpedia.org/resource/Helen_Hyde, http://dbpedia.org/resource/James_Zar, http://dbpedia.org/resource/Walter_Kuhlman, http://dbpedia.org/resource/Jason_Rhoades, http://dbpedia.org/resource/Jess_Collins, http://dbpedia.org/resource/Manuel_Neri, http://dbpedia.org/resource/Michael_Heizer, http://dbpedia.org/resource/Ernest_Peixotto, http://dbpedia.org/resource/William_Anthony_(artist), http://dbpedia.org/resource/Honoré_Desmond_Sharrer, http://dbpedia.org/resource/Suzanne_Bocanegra, http://dbpedia.org/resource/Phoebe_Man, http://dbpedia.org/resource/Ralph_Stackpole, http://dbpedia.org/resource/Emily_Carr, http://dbpedia.org/resource/William_T._Wiley, http://dbpedia.org/resource/Robert_H._Hudson, http://dbpedia.org/resource/Linda_Lomahaftewa, http://dbpedia.org/resource/Edith_Heath, http://dbpedia.org/resource/Kevin_Larmee, http://dbpedia.org/resource/Lynn_Marie_Kirby, http://dbpedia.org/resource/Mario_Martinez_(painter), http://dbpedia.org/resource/Sarah_Cain, http://dbpedia.org/resource/Gutzon_Borglum, http://dbpedia.org/resource/Paul_McCarthy, http://dbpedia.org/resource/Rinaldo_Cuneo, http://dbpedia.org/resource/Robert_Graham_(sculptor), http://dbpedia.org/resource/Carlos_Villa, http://dbpedia.org/resource/Deborah_Remington, http://dbpedia.org/resource/Megan_Wilson, http://dbpedia.org/resource/Xavier_Martínez, http://dbpedia.org/resource/Karl_Kasten, http://dbpedia.org/resource/Sonia_Gechtoff, http://dbpedia.org/resource/Harrell_Fletcher, http://dbpedia.org/resource/Ian_Alan_Paul, http://dbpedia.org/resource/Jim_Franklin_(artist), http://dbpedia.org/resource/Jerome_Caja, http://dbpedia.org/resource/Ray_Strong, http://dbpedia.org/resource/David_Park_(painter), http://dbpedia.org/resource/Joan_Brown, http://dbpedia.org/resource/Neil_Williams_(artist), http://dbpedia.org/resource/Ronald_Davis, http://dbpedia.org/resource/Ronnie_Landfield, http://dbpedia.org/resource/Tullio_DeSantis, http://dbpedia.org/resource/Steve_Lambert, http://dbpedia.org/resource/Peter_Forakis, http://dbpedia.org/resource/Reminisce_(artist), http://dbpedia.org/resource/Peter_Kovál, http://dbpedia.org/resource/Gladys_Rockmore_Davis, http://dbpedia.org/resource/Thom_Merrick, http://dbpedia.org/resource/America_Meredith, http://dbpedia.org/resource/Renata_Bernal, http://dbpedia.org/resource/Lin_Evola, http://dbpedia.org/resource/T._C._Cannon, http://dbpedia.org/resource/Arthur_Frank_Mathews, http://dbpedia.org/resource/Rowena_Meeks_Abdy, http://dbpedia.org/resource/Adelaide_Hanscom_Leeson, http://dbpedia.org/resource/Doris_Lee, http://dbpedia.org/resource/Kenn_Davis, http://dbpedia.org/resource/Daniela_Franco, http://dbpedia.org/resource/Alice_Brown_Chittenden, http://dbpedia.org/resource/Vena_Kava, http://dbpedia.org/resource/Victor_Moscoso] [INFO] P: 0.6470588235294118, R: 1.0, F: 0.7857142857142858 -[INFO] Cumulative metrics, sample: 70, P: 0.695, R: 0.727, macro F1: 0.695, macro F1*: 0.711 -[INFO] Current time: 1817 ms, average: 3220.36 ms/ques, entity detection: 443.50 ms/ques (linking tool: 348.46 ms/ques), relation detection: 847.04 ms/ques (similarity: 637.79 ms/ques), EDG generation: 132.24 ms/ques +[INFO] Cumulative metrics, sample: 70, P: 0.695, R: 0.727, macro F1: 0.711 +[INFO] Current question consumed time: 1708 ms, average: 3068.86 ms/ques, entity detection: 435.47 ms/ques (linking tool: 345.70 ms/ques), relation detection: 833.77 ms/ques (similarity: 638.97 ms/ques), EDG generation: 131.06 ms/ques [INFO] Entity Linking P: 0.8857142857142857, R: 0.8571428571428571, micro F1: 0.8666666666666667, macro F1: 0.8711943793911006 [INFO] Relation Linking P: 0.7166666666666667, R: 0.7142857142857143, micro F1: 0.7114285714285714, macro F1: 0.7154742096505824 [INFO] Type Linking P: 0.5, R: 0.5, micro F1: 0.5, macro F1: 0.5 @@ -987,8 +987,8 @@ Question 70: What is the owning organisation of the Ford Kansas City Assembly Pl [INFO] Golden Answer: [http://dbpedia.org/resource/Ford_Motor_Company] [INFO] Predicted Answer: [http://dbpedia.org/resource/Ford_Motor_Company] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 71, P: 0.700, R: 0.731, macro F1: 0.699, macro F1*: 0.715 -[INFO] Current time: 1412 ms, average: 3194.89 ms/ques, entity detection: 441.51 ms/ques (linking tool: 346.10 ms/ques), relation detection: 839.25 ms/ques (similarity: 631.41 ms/ques), EDG generation: 131.85 ms/ques +[INFO] Cumulative metrics, sample: 71, P: 0.700, R: 0.731, macro F1: 0.715 +[INFO] Current question consumed time: 1339 ms, average: 3044.49 ms/ques, entity detection: 433.10 ms/ques (linking tool: 343.08 ms/ques), relation detection: 825.85 ms/ques (similarity: 632.52 ms/ques), EDG generation: 131.65 ms/ques [INFO] Entity Linking P: 0.8873239436619719, R: 0.8591549295774648, micro F1: 0.8685446009389672, macro F1: 0.873012267151295 [INFO] Relation Linking P: 0.7206572769953051, R: 0.7183098591549296, micro F1: 0.7154929577464788, macro F1: 0.7194816533786733 [INFO] Type Linking P: 0.5070422535211268, R: 0.5070422535211268, micro F1: 0.5070422535211268, macro F1: 0.5070422535211268 @@ -1001,8 +1001,8 @@ Question 71: Name the country with currency as Aureus ? [INFO] Golden Answer: [http://dbpedia.org/resource/History_of_the_Roman_Empire, http://dbpedia.org/resource/Roman_Empire] [INFO] Predicted Answer: [http://dbpedia.org/resource/History_of_the_Roman_Empire, http://dbpedia.org/resource/Roman_Empire] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 72, P: 0.704, R: 0.735, macro F1: 0.704, macro F1*: 0.719 -[INFO] Current time: 699 ms, average: 3160.22 ms/ques, entity detection: 437.49 ms/ques (linking tool: 342.39 ms/ques), relation detection: 830.14 ms/ques (similarity: 624.85 ms/ques), EDG generation: 130.19 ms/ques +[INFO] Cumulative metrics, sample: 72, P: 0.704, R: 0.735, macro F1: 0.719 +[INFO] Current question consumed time: 620 ms, average: 3010.82 ms/ques, entity detection: 428.90 ms/ques (linking tool: 339.26 ms/ques), relation detection: 816.68 ms/ques (similarity: 625.99 ms/ques), EDG generation: 129.99 ms/ques [INFO] Entity Linking P: 0.8888888888888888, R: 0.8611111111111112, micro F1: 0.8703703703703705, macro F1: 0.8747795414462081 [INFO] Relation Linking P: 0.724537037037037, R: 0.7222222222222222, micro F1: 0.7194444444444444, macro F1: 0.7233777777777778 [INFO] Type Linking P: 0.5, R: 0.5, micro F1: 0.5, macro F1: 0.5 @@ -1015,8 +1015,8 @@ Question 72: what awards have been given to A. Peter Dewey? [INFO] Golden Answer: [http://dbpedia.org/resource/Tunisia, http://dbpedia.org/resource/Legion_of_Honour, http://dbpedia.org/resource/Silver_Star, http://dbpedia.org/resource/Combatant's_Cross, http://dbpedia.org/resource/Order_of_Polonia_Restituta, http://dbpedia.org/resource/Croix_de_guerre_1939–1945_(France), http://dbpedia.org/resource/Legion_of_Merit] [INFO] Predicted Answer: [http://dbpedia.org/resource/Silver_Star, http://dbpedia.org/resource/Combatant's_Cross, http://dbpedia.org/resource/Order_of_Polonia_Restituta, http://dbpedia.org/resource/Legion_of_Merit] [INFO] P: 1.0, R: 0.5714285714285714, F: 0.7272727272727273 -[INFO] Cumulative metrics, sample: 73, P: 0.708, R: 0.733, macro F1: 0.704, macro F1*: 0.720 -[INFO] Current time: 831 ms, average: 3128.32 ms/ques, entity detection: 433.38 ms/ques (linking tool: 338.84 ms/ques), relation detection: 821.84 ms/ques (similarity: 619.38 ms/ques), EDG generation: 128.62 ms/ques +[INFO] Cumulative metrics, sample: 73, P: 0.708, R: 0.733, macro F1: 0.720 +[INFO] Current question consumed time: 745 ms, average: 2979.78 ms/ques, entity detection: 424.66 ms/ques (linking tool: 335.60 ms/ques), relation detection: 808.33 ms/ques (similarity: 620.51 ms/ques), EDG generation: 128.38 ms/ques [INFO] Entity Linking P: 0.8904109589041096, R: 0.863013698630137, micro F1: 0.8721461187214613, macro F1: 0.876498287671233 [INFO] Relation Linking P: 0.7146118721461187, R: 0.7123287671232876, micro F1: 0.7095890410958904, macro F1: 0.7134684931506848 [INFO] Type Linking P: 0.4931506849315068, R: 0.4931506849315068, micro F1: 0.4931506849315068, macro F1: 0.4931506849315068 @@ -1029,8 +1029,8 @@ Question 73: How many shows belong to the network one of whose sister station is [INFO] Golden Answer: [77] [INFO] Predicted Answer: [13] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 74, P: 0.698, R: 0.723, macro F1: 0.694, macro F1*: 0.710 -[INFO] Current time: 2991 ms, average: 3126.46 ms/ques, entity detection: 428.26 ms/ques (linking tool: 334.80 ms/ques), relation detection: 828.61 ms/ques (similarity: 626.22 ms/ques), EDG generation: 128.30 ms/ques +[INFO] Cumulative metrics, sample: 74, P: 0.698, R: 0.723, macro F1: 0.710 +[INFO] Current question consumed time: 2862 ms, average: 2978.20 ms/ques, entity detection: 419.57 ms/ques (linking tool: 331.55 ms/ques), relation detection: 815.09 ms/ques (similarity: 627.22 ms/ques), EDG generation: 128.20 ms/ques [INFO] Entity Linking P: 0.8918918918918919, R: 0.8648648648648649, micro F1: 0.8738738738738739, macro F1: 0.8781704781704782 [INFO] Relation Linking P: 0.7117117117117117, R: 0.7094594594594594, micro F1: 0.7067567567567568, macro F1: 0.710583800916606 [INFO] Type Linking P: 0.4864864864864865, R: 0.4864864864864865, micro F1: 0.4864864864864865, macro F1: 0.4864864864864865 @@ -1043,8 +1043,8 @@ Question 74: what has been developed by John Fanning? [INFO] Golden Answer: [http://dbpedia.org/resource/Napster] [INFO] Predicted Answer: [http://dbpedia.org/resource/Napster] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 75, P: 0.702, R: 0.727, macro F1: 0.698, macro F1*: 0.714 -[INFO] Current time: 593 ms, average: 3092.68 ms/ques, entity detection: 425.01 ms/ques (linking tool: 331.87 ms/ques), relation detection: 819.12 ms/ques (similarity: 618.88 ms/ques), EDG generation: 126.72 ms/ques +[INFO] Cumulative metrics, sample: 75, P: 0.702, R: 0.727, macro F1: 0.714 +[INFO] Current question consumed time: 550 ms, average: 2945.83 ms/ques, entity detection: 415.99 ms/ques (linking tool: 328.25 ms/ques), relation detection: 805.84 ms/ques (similarity: 619.85 ms/ques), EDG generation: 126.63 ms/ques [INFO] Entity Linking P: 0.8933333333333333, R: 0.8666666666666667, micro F1: 0.8755555555555556, macro F1: 0.8797979797979798 [INFO] Relation Linking P: 0.7155555555555555, R: 0.7133333333333334, micro F1: 0.7106666666666667, macro F1: 0.7144427164333851 [INFO] Type Linking P: 0.49333333333333335, R: 0.49333333333333335, micro F1: 0.49333333333333335, macro F1: 0.4933333333333334 @@ -1057,8 +1057,8 @@ Question 75: Muhammad Yunus has won how many awards? [INFO] Golden Answer: [9] [INFO] Predicted Answer: [9] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 76, P: 0.706, R: 0.730, macro F1: 0.702, macro F1*: 0.718 -[INFO] Current time: 1313 ms, average: 3069.26 ms/ques, entity detection: 421.50 ms/ques (linking tool: 328.78 ms/ques), relation detection: 813.91 ms/ques (similarity: 616.93 ms/ques), EDG generation: 125.20 ms/ques +[INFO] Cumulative metrics, sample: 76, P: 0.706, R: 0.730, macro F1: 0.718 +[INFO] Current question consumed time: 1254 ms, average: 2923.57 ms/ques, entity detection: 412.46 ms/ques (linking tool: 325.11 ms/ques), relation detection: 800.83 ms/ques (similarity: 617.88 ms/ques), EDG generation: 125.11 ms/ques [INFO] Entity Linking P: 0.8947368421052632, R: 0.868421052631579, micro F1: 0.8771929824561404, macro F1: 0.8813825608798115 [INFO] Relation Linking P: 0.706140350877193, R: 0.7039473684210527, micro F1: 0.7013157894736842, macro F1: 0.7050421543750511 [INFO] Type Linking P: 0.4868421052631579, R: 0.4868421052631579, micro F1: 0.4868421052631579, macro F1: 0.4868421052631579 @@ -1071,8 +1071,8 @@ Question 76: List the associates of bands which have a label by Motown ? [INFO] Golden Answer: [http://dbpedia.org/resource/Martha_and_the_Vandellas, http://dbpedia.org/resource/Marvin_Gaye, http://dbpedia.org/resource/The_Marvelettes, http://dbpedia.org/resource/Four_Tops, http://dbpedia.org/resource/Mary_Wells, http://dbpedia.org/resource/The_Supremes, The Downbeats, Sandra Mallett & The Vandellas@en, Richard Street & the Distants, The Temptations@en, http://dbpedia.org/resource/Elton_John, http://dbpedia.org/resource/Kylie_Minogue, http://dbpedia.org/resource/Teena_Marie, http://dbpedia.org/resource/The_Mooseheart_Faith_Stellar_Groove_Band, Kagny and the Dirty Rats@en, http://dbpedia.org/resource/Jimmy_Ruffin, http://dbpedia.org/resource/Edwin_Starr, http://dbpedia.org/resource/Stevie_Wonder, http://dbpedia.org/resource/David_Ruffin, http://dbpedia.org/resource/Bobby_Taylor_&_the_Vancouvers, http://dbpedia.org/resource/Syreeta_Wright, http://dbpedia.org/resource/Billy_Preston, The Endeavors@en, http://dbpedia.org/resource/Foo_Fighters, http://dbpedia.org/resource/New_Kids_on_the_Block, http://dbpedia.org/resource/Boyz_II_Men, Marvin Gaye and Tammi Terrell, Diana Ross, Stevie Wonder, Chaka Khan, Luther Vandross@en, http://dbpedia.org/resource/El_DeBarge, http://dbpedia.org/resource/Bunny_DeBarge, http://dbpedia.org/resource/Kristinia_DeBarge, http://dbpedia.org/resource/Chico_DeBarge, http://dbpedia.org/resource/Bobby_DeBarge, http://dbpedia.org/resource/Switch_(band), http://dbpedia.org/resource/The_Funk_Brothers, The Miracles, the Temptations, the Supremes, Martha and the Vandellas, the Marvelettes, Marvin Gaye, the Contours, the Jackson 5, the Four Tops, the Spinners, the Originals, the Andantes@en, Jack London & The Sparrows, Steppenwolf, Buffalo Springfield, The Sparrows, Sailorboys@en, http://dbpedia.org/resource/Dionne_Warwick, http://dbpedia.org/resource/Harvey_Fuqua, Mason's Cure@en, http://dbpedia.org/resource/Elle_Varner, The Future Tones, Blinky Williams@en, http://dbpedia.org/resource/Smokey_Robinson, http://dbpedia.org/resource/Dennis_Edwards, http://dbpedia.org/resource/Norman_Whitfield, http://dbpedia.org/resource/The_Original_7ven, http://dbpedia.org/resource/Klymaxx, http://dbpedia.org/resource/Brownmark, http://dbpedia.org/resource/Prince_(musician), http://dbpedia.org/resource/The_Wild_Pair, http://dbpedia.org/resource/The_Revolution_(band), http://dbpedia.org/resource/Rick_James, http://dbpedia.org/resource/Teddy_Riley, Gene Griffin@en, http://dbpedia.org/resource/Queen_Latifah, http://dbpedia.org/resource/Vivian_Green, http://dbpedia.org/resource/112_(band), http://dbpedia.org/resource/Mariah_Carey, http://dbpedia.org/resource/Jimmy_Jam_and_Terry_Lewis, http://dbpedia.org/resource/Babyface_(musician), http://dbpedia.org/resource/Brandy_Norwood, http://dbpedia.org/resource/New_Edition, http://dbpedia.org/resource/Sean_Combs, http://dbpedia.org/resource/Uncle_Sam_(singer), http://dbpedia.org/resource/L.A._Reid, http://dbpedia.org/resource/Brian_McKnight, http://dbpedia.org/resource/Dallas_Austin, http://dbpedia.org/resource/Tim_&_Bob, http://dbpedia.org/resource/The_Noel_Redding_Band, http://dbpedia.org/resource/Deniece_Williams, http://dbpedia.org/resource/Minnie_Riperton, http://dbpedia.org/resource/Ramsey_Lewis, http://dbpedia.org/resource/Curtis_Mayfield, http://dbpedia.org/resource/Earth,_Wind_&_Fire, http://dbpedia.org/resource/The_Temptations, http://dbpedia.org/resource/The_Falcons, http://dbpedia.org/resource/The_Jackson_5, http://dbpedia.org/resource/DeBarge, http://dbpedia.org/resource/Randy_DeBarge, http://dbpedia.org/resource/Mark_DeBarge, http://dbpedia.org/resource/James_DeBarge, http://dbpedia.org/resource/Bob_Seger, http://dbpedia.org/resource/Silkk_the_Shocker, http://dbpedia.org/resource/Mercedes_(singer), http://dbpedia.org/resource/Mia_X, http://dbpedia.org/resource/Allusion, http://dbpedia.org/resource/Michael_Bivins, http://dbpedia.org/resource/Missy_Elliott, http://dbpedia.org/resource/Timbaland, The Strikers, Strafe@en, http://dbpedia.org/resource/Naughty_by_Nature, http://dbpedia.org/resource/Road_Dawgs] [INFO] Predicted Answer: [http://dbpedia.org/resource/Pat_Lewis, http://dbpedia.org/resource/Damian_LeGassick, http://dbpedia.org/resource/Betty_Kelly, http://dbpedia.org/resource/Sandra_Tilley, http://dbpedia.org/resource/Del_Marquis, http://dbpedia.org/resource/Larry_Tee, http://dbpedia.org/resource/Paddy_Boom, http://dbpedia.org/resource/Jake_Shears, http://dbpedia.org/resource/Stuart_Price, http://dbpedia.org/resource/Ana_Matronic, http://dbpedia.org/resource/Babydaddy, http://dbpedia.org/resource/Ray_Singleton, http://dbpedia.org/resource/Freddie_Gorman, http://dbpedia.org/resource/C._P._Spencer, http://dbpedia.org/resource/Westlife, http://dbpedia.org/resource/New_Kids_on_the_Block, http://dbpedia.org/resource/Kenny_Greene, http://dbpedia.org/resource/Ullanda_McCullough, http://dbpedia.org/resource/Yogi_Horton, http://dbpedia.org/resource/Tammi_Terrell, http://dbpedia.org/resource/Chaka_Khan, http://dbpedia.org/resource/Jo_Armstead, http://dbpedia.org/resource/Blinky_(singer), http://dbpedia.org/resource/The_Andantes, http://dbpedia.org/resource/The_Velvelettes, http://dbpedia.org/resource/Ronald_Townson, http://dbpedia.org/resource/Marc_Gordon, http://dbpedia.org/resource/Billy_Davis,_Jr., http://dbpedia.org/resource/Florence_LaRue, http://dbpedia.org/resource/Bones_Howe, http://dbpedia.org/resource/Marilyn_McCoo, http://dbpedia.org/resource/Dennis_Coffey, http://dbpedia.org/resource/James_Jamerson, http://dbpedia.org/resource/Martha_and_the_Vandellas, http://dbpedia.org/resource/Robert_White_(guitarist), http://dbpedia.org/resource/Benny_Benjamin, http://dbpedia.org/resource/Bob_Babbitt, http://dbpedia.org/resource/Uriel_Jones, http://dbpedia.org/resource/Wah_Wah_Watson, http://dbpedia.org/resource/Richard_%22Pistol%22_Allen, http://dbpedia.org/resource/Paul_Riser, http://dbpedia.org/resource/Jack_Ashford, http://dbpedia.org/resource/Johnny_Griffith_(musician), http://dbpedia.org/resource/David_Townsend_(musician), http://dbpedia.org/resource/Rudolph_Isley, http://dbpedia.org/resource/Ronald_Isley, http://dbpedia.org/resource/O'Kelly_Isley,_Jr., http://dbpedia.org/resource/Marvin_Isley, http://dbpedia.org/resource/Chris_Jasper, http://dbpedia.org/resource/Tim_&_Bob, http://dbpedia.org/resource/Tonto's_Expanding_Head_Band, http://dbpedia.org/resource/Charles_Pitts, http://dbpedia.org/resource/Angela_Winbush, http://dbpedia.org/resource/René_Moore, http://dbpedia.org/resource/Steppenwolf_(band), http://dbpedia.org/resource/Jack_London_&_The_Sparrows, http://dbpedia.org/resource/Neil_Young, http://dbpedia.org/resource/Nick_St._Nicholas, http://dbpedia.org/resource/Neil_Merryweather, http://dbpedia.org/resource/Elle_Varner, http://dbpedia.org/resource/The_Originals_(band), http://dbpedia.org/resource/Barrett_Strong, http://dbpedia.org/resource/Norman_Whitfield, http://dbpedia.org/resource/Spike_Edney, http://dbpedia.org/resource/Florence_Ballard, http://dbpedia.org/resource/Sheldon_Reynolds_(guitarist), http://dbpedia.org/resource/Thomas_McClary_(musician), http://dbpedia.org/resource/James_Anthony_Carmichael, http://dbpedia.org/resource/Franne_Golde, http://dbpedia.org/resource/Skip_Miller, http://dbpedia.org/resource/The_Revolution_(band), http://dbpedia.org/resource/The_Weather_Girls, http://dbpedia.org/resource/Guy_(band), http://dbpedia.org/resource/Joe_(singer), http://dbpedia.org/resource/Unqle_Chriz, http://dbpedia.org/resource/Gwen_Gordy_Fuqua, http://dbpedia.org/resource/Bobby_Rogers, http://dbpedia.org/resource/Johnnie_Bassett, http://dbpedia.org/resource/Ronald_White, http://dbpedia.org/resource/Billy_Griffin, http://dbpedia.org/resource/Claudette_Rogers_Robinson, http://dbpedia.org/resource/Marv_Tarplin, http://dbpedia.org/resource/Warren_%22Pete%22_Moore, http://dbpedia.org/resource/Danny_Sembello, http://dbpedia.org/resource/Ruth_Pointer, http://dbpedia.org/resource/June_Pointer, http://dbpedia.org/resource/Anita_Pointer, http://dbpedia.org/resource/Bonnie_Pointer, http://dbpedia.org/resource/Davey_Johnstone, http://dbpedia.org/resource/Issa_Pointer, http://dbpedia.org/resource/Sadako_Pointer, http://dbpedia.org/resource/Melvin_Lee_Davis, http://dbpedia.org/resource/Ruby_Turner, http://dbpedia.org/resource/The_Temptations, http://dbpedia.org/resource/Cindy_Birdsong, http://dbpedia.org/resource/Levi_Stubbs, http://dbpedia.org/resource/Renaldo_Benson, http://dbpedia.org/resource/Maurice_White, http://dbpedia.org/resource/Deniece_Williams, http://dbpedia.org/resource/The_Chi-Lites, http://dbpedia.org/resource/List_of_Earth,_Wind_&_Fire_band_members, http://dbpedia.org/resource/Skip_Scarborough, http://dbpedia.org/resource/Earth,_Wind_&_Fire, http://dbpedia.org/resource/Garry_Glenn, http://dbpedia.org/resource/MC_Trouble, http://dbpedia.org/resource/Shaun_Murphy_(singer), http://dbpedia.org/resource/Reggie_McBride, http://dbpedia.org/resource/L._Russell_Brown, http://dbpedia.org/resource/Four_Lovers, http://dbpedia.org/resource/Tommy_DeVito_(musician), http://dbpedia.org/resource/Bob_Crewe, http://dbpedia.org/resource/Frankie_Valli, http://dbpedia.org/resource/The_Wonder_Who%3F, http://dbpedia.org/resource/Mary_Wilson_(singer), http://dbpedia.org/resource/Barbara_Martin_(singer), http://dbpedia.org/resource/Betty_McGlown, http://dbpedia.org/resource/Susaye_Greene, http://dbpedia.org/resource/Dionne_Warwick, http://dbpedia.org/resource/Johnny_Bristol, http://dbpedia.org/resource/Eddie_Kendricks, http://dbpedia.org/resource/Fontella_Bass, http://dbpedia.org/resource/Scherrie_Payne, http://dbpedia.org/resource/James_%22Boogaloo%22_Bolden, http://dbpedia.org/resource/The_Sweet_Inspirations, http://dbpedia.org/resource/Buddy_Miles, http://dbpedia.org/resource/Labelle, http://dbpedia.org/resource/Pino_Presti, http://dbpedia.org/resource/Elliot_Levine, http://dbpedia.org/resource/Jerome_Teasley, http://dbpedia.org/resource/Hubert_Johnson_(musician), http://dbpedia.org/resource/Santa_Barbera_Machine_Head, http://dbpedia.org/resource/Fairies_(British_band), http://dbpedia.org/resource/Dick_Taylor, http://dbpedia.org/resource/Mickey_Finn's_T-Rex, http://dbpedia.org/resource/Stars_(British_band), http://dbpedia.org/resource/Twink_(musician), http://dbpedia.org/resource/Jack_Green_(musician), http://dbpedia.org/resource/DeBarge, http://dbpedia.org/resource/El_DeBarge, http://dbpedia.org/resource/Phillip_Ingram, http://dbpedia.org/resource/High_Inergy, http://dbpedia.org/resource/Jermaine_Jackson, http://dbpedia.org/resource/Bunny_DeBarge, http://dbpedia.org/resource/Bob_Relf, http://dbpedia.org/resource/Bobby_DeBarge, http://dbpedia.org/resource/Teddy_Riley, http://dbpedia.org/resource/Soulshock_and_Karlin, http://dbpedia.org/resource/Naughty_by_Nature, http://dbpedia.org/resource/Treach, http://dbpedia.org/resource/Road_Dawgs, http://dbpedia.org/resource/Marvin_Gaye, http://dbpedia.org/resource/The_Marvelettes, http://dbpedia.org/resource/Four_Tops, http://dbpedia.org/resource/Mary_Wells, http://dbpedia.org/resource/The_Supremes, The Downbeats, Sandra Mallett & The Vandellas@en, Richard Street & the Distants, The Temptations@en, http://dbpedia.org/resource/Elton_John, http://dbpedia.org/resource/Kylie_Minogue, http://dbpedia.org/resource/Teena_Marie, http://dbpedia.org/resource/The_Mooseheart_Faith_Stellar_Groove_Band, Kagny and the Dirty Rats@en, http://dbpedia.org/resource/Jimmy_Ruffin, http://dbpedia.org/resource/Edwin_Starr, http://dbpedia.org/resource/Stevie_Wonder, http://dbpedia.org/resource/David_Ruffin, http://dbpedia.org/resource/Bobby_Taylor_&_the_Vancouvers, http://dbpedia.org/resource/Syreeta_Wright, http://dbpedia.org/resource/Billy_Preston, The Endeavors@en, http://dbpedia.org/resource/Foo_Fighters, http://dbpedia.org/resource/Boyz_II_Men, Marvin Gaye and Tammi Terrell, Diana Ross, Stevie Wonder, Chaka Khan, Luther Vandross@en, http://dbpedia.org/resource/The_Funk_Brothers, The Miracles, the Temptations, the Supremes, Martha and the Vandellas, the Marvelettes, Marvin Gaye, the Contours, the Jackson 5, the Four Tops, the Spinners, the Originals, the Andantes@en, Jack London & The Sparrows, Steppenwolf, Buffalo Springfield, The Sparrows, Sailorboys@en, The Future Tones, Blinky Williams@en, http://dbpedia.org/resource/The_Original_7ven, http://dbpedia.org/resource/Klymaxx, http://dbpedia.org/resource/Brownmark, http://dbpedia.org/resource/Prince_(musician), http://dbpedia.org/resource/The_Wild_Pair, http://dbpedia.org/resource/Rick_James, Gene Griffin@en, http://dbpedia.org/resource/Queen_Latifah, http://dbpedia.org/resource/Smokey_Robinson, http://dbpedia.org/resource/Switch_(band), http://dbpedia.org/resource/Minnie_Riperton, http://dbpedia.org/resource/Ramsey_Lewis, http://dbpedia.org/resource/Curtis_Mayfield, http://dbpedia.org/resource/The_Falcons, http://dbpedia.org/resource/Randy_DeBarge, http://dbpedia.org/resource/Mark_DeBarge, http://dbpedia.org/resource/James_DeBarge, http://dbpedia.org/resource/Bob_Seger, http://dbpedia.org/resource/Silkk_the_Shocker, http://dbpedia.org/resource/Mercedes_(singer), http://dbpedia.org/resource/Mia_X, http://dbpedia.org/resource/Allusion, http://dbpedia.org/resource/Michael_Bivins, http://dbpedia.org/resource/Missy_Elliott, http://dbpedia.org/resource/Timbaland, The Strikers, Strafe@en] [INFO] P: 0.3448275862068966, R: 0.7865168539325843, F: 0.47945205479452063 -[INFO] Cumulative metrics, sample: 77, P: 0.702, R: 0.731, macro F1: 0.700, macro F1*: 0.716 -[INFO] Current time: 2770 ms, average: 3065.38 ms/ques, entity detection: 419.65 ms/ques (linking tool: 327.23 ms/ques), relation detection: 818.77 ms/ques (similarity: 620.30 ms/ques), EDG generation: 123.90 ms/ques +[INFO] Cumulative metrics, sample: 77, P: 0.702, R: 0.731, macro F1: 0.716 +[INFO] Current question consumed time: 2602 ms, average: 2919.39 ms/ques, entity detection: 410.61 ms/ques (linking tool: 323.52 ms/ques), relation detection: 805.34 ms/ques (similarity: 621.13 ms/ques), EDG generation: 123.79 ms/ques [INFO] Entity Linking P: 0.8961038961038961, R: 0.8701298701298701, micro F1: 0.8787878787878789, macro F1: 0.88292589763178 [INFO] Relation Linking P: 0.7034632034632035, R: 0.7012987012987013, micro F1: 0.6987012987012987, macro F1: 0.7023792848137995 [INFO] Type Linking P: 0.4805194805194805, R: 0.4805194805194805, micro F1: 0.4805194805194805, macro F1: 0.4805194805194805 @@ -1085,8 +1085,8 @@ Question 77: Who wrote the musical whose composer is Emil Dean Zoghby? [INFO] Golden Answer: [http://dbpedia.org/resource/William_Shakespeare] [INFO] Predicted Answer: [http://dbpedia.org/resource/Alexander_Lernet-Holenia, http://dbpedia.org/resource/Curt_Johannes_Braun, http://dbpedia.org/resource/Felix_von_Eckardt, http://dbpedia.org/resource/Lotte_Neumann, http://dbpedia.org/resource/Paul_Josef_Cremers, http://dbpedia.org/resource/Gerhard_Menzel, http://dbpedia.org/resource/Walter_Wassermann, http://dbpedia.org/resource/Hellmuth_Unger, http://dbpedia.org/resource/Hanns_Kräly, http://dbpedia.org/resource/Rudolf_Stratz, Kurt Heuser@en, Harald Bratt@en, Gerhard Menzel@en, Lotte Neumann@en, Walter Wassermann@en, Paul Josef Cremers@en, Rudolf Stratz@en] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 78, P: 0.693, R: 0.722, macro F1: 0.691, macro F1*: 0.707 -[INFO] Current time: 1588 ms, average: 3046.44 ms/ques, entity detection: 415.87 ms/ques (linking tool: 324.05 ms/ques), relation detection: 815.81 ms/ques (similarity: 618.95 ms/ques), EDG generation: 122.49 ms/ques +[INFO] Cumulative metrics, sample: 78, P: 0.693, R: 0.722, macro F1: 0.707 +[INFO] Current question consumed time: 1467 ms, average: 2900.77 ms/ques, entity detection: 406.82 ms/ques (linking tool: 320.28 ms/ques), relation detection: 802.12 ms/ques (similarity: 619.69 ms/ques), EDG generation: 122.37 ms/ques [INFO] Entity Linking P: 0.8846153846153846, R: 0.8589743589743589, micro F1: 0.8675213675213675, macro F1: 0.8716063348416289 [INFO] Relation Linking P: 0.6944444444444444, R: 0.6923076923076923, micro F1: 0.6897435897435897, macro F1: 0.6933744221879815 [INFO] Type Linking P: 0.47435897435897434, R: 0.47435897435897434, micro F1: 0.47435897435897434, macro F1: 0.47435897435897434 @@ -1099,8 +1099,8 @@ Question 78: Count the units garrisoned at Arlington County, Virginia. [INFO] Golden Answer: [8] [INFO] Predicted Answer: [35, 20] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 79, P: 0.684, R: 0.712, macro F1: 0.682, macro F1*: 0.698 -[INFO] Current time: 4962 ms, average: 3070.68 ms/ques, entity detection: 419.77 ms/ques (linking tool: 328.48 ms/ques), relation detection: 832.33 ms/ques (similarity: 624.58 ms/ques), EDG generation: 121.13 ms/ques +[INFO] Cumulative metrics, sample: 79, P: 0.684, R: 0.712, macro F1: 0.698 +[INFO] Current question consumed time: 4600 ms, average: 2922.28 ms/ques, entity detection: 410.22 ms/ques (linking tool: 324.14 ms/ques), relation detection: 816.78 ms/ques (similarity: 625.00 ms/ques), EDG generation: 121.03 ms/ques [INFO] Entity Linking P: 0.8734177215189873, R: 0.8481012658227848, micro F1: 0.8565400843881857, macro F1: 0.8605733432613552 [INFO] Relation Linking P: 0.6983122362869199, R: 0.6962025316455697, micro F1: 0.6936708860759493, macro F1: 0.697255788123097 [INFO] Type Linking P: 0.46835443037974683, R: 0.46835443037974683, micro F1: 0.46835443037974683, macro F1: 0.46835443037974683 @@ -1113,8 +1113,8 @@ Question 79: What is the faith of Kwai Chang Caine ? [INFO] Golden Answer: [http://dbpedia.org/resource/Buddhism] [INFO] Predicted Answer: [http://dbpedia.org/resource/Buddhism] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 80, P: 0.688, R: 0.716, macro F1: 0.686, macro F1*: 0.702 -[INFO] Current time: 570 ms, average: 3039.43 ms/ques, entity detection: 416.23 ms/ques (linking tool: 325.31 ms/ques), relation detection: 823.58 ms/ques (similarity: 617.86 ms/ques), EDG generation: 119.72 ms/ques +[INFO] Cumulative metrics, sample: 80, P: 0.688, R: 0.716, macro F1: 0.702 +[INFO] Current question consumed time: 555 ms, average: 2892.70 ms/ques, entity detection: 406.81 ms/ques (linking tool: 321.02 ms/ques), relation detection: 808.21 ms/ques (similarity: 618.17 ms/ques), EDG generation: 119.64 ms/ques [INFO] Entity Linking P: 0.875, R: 0.85, micro F1: 0.8583333333333334, macro F1: 0.8623188405797101 [INFO] Relation Linking P: 0.7020833333333333, R: 0.7, micro F1: 0.6975, macro F1: 0.7010401188707281 [INFO] Type Linking P: 0.475, R: 0.475, micro F1: 0.475, macro F1: 0.47500000000000003 @@ -1127,8 +1127,8 @@ Question 80: Who is related to Kelly Osbourne & Ozzy Osbourne? [INFO] Golden Answer: [http://dbpedia.org/resource/Jack_Osbourne] [INFO] Predicted Answer: [http://dbpedia.org/resource/Jack_Osbourne] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 81, P: 0.692, R: 0.720, macro F1: 0.690, macro F1*: 0.705 -[INFO] Current time: 2364 ms, average: 3031.09 ms/ques, entity detection: 417.15 ms/ques (linking tool: 325.62 ms/ques), relation detection: 823.99 ms/ques (similarity: 616.86 ms/ques), EDG generation: 118.64 ms/ques +[INFO] Cumulative metrics, sample: 81, P: 0.692, R: 0.720, macro F1: 0.705 +[INFO] Current question consumed time: 2244 ms, average: 2884.69 ms/ques, entity detection: 407.35 ms/ques (linking tool: 321.05 ms/ques), relation detection: 808.78 ms/ques (similarity: 617.07 ms/ques), EDG generation: 118.52 ms/ques [INFO] Entity Linking P: 0.8765432098765432, R: 0.8518518518518519, micro F1: 0.860082304526749, macro F1: 0.864021164021164 [INFO] Relation Linking P: 0.7057613168724279, R: 0.7037037037037037, micro F1: 0.7012345679012345, macro F1: 0.7047310083806434 [INFO] Type Linking P: 0.48148148148148145, R: 0.48148148148148145, micro F1: 0.48148148148148145, macro F1: 0.48148148148148145 @@ -1141,8 +1141,8 @@ Question 81: What are the houses of the legislature where the leader is called P [INFO] Golden Answer: [http://dbpedia.org/resource/Rajya_Sabha, http://dbpedia.org/resource/Lok_Sabha] [INFO] Predicted Answer: [http://dbpedia.org/resource/Dr._Siva_Prasad] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 82, P: 0.683, R: 0.711, macro F1: 0.681, macro F1*: 0.697 -[INFO] Current time: 977 ms, average: 3006.04 ms/ques, entity detection: 412.46 ms/ques (linking tool: 321.93 ms/ques), relation detection: 819.46 ms/ques (similarity: 615.41 ms/ques), EDG generation: 117.57 ms/ques +[INFO] Cumulative metrics, sample: 82, P: 0.683, R: 0.711, macro F1: 0.697 +[INFO] Current question consumed time: 941 ms, average: 2860.99 ms/ques, entity detection: 402.73 ms/ques (linking tool: 317.40 ms/ques), relation detection: 804.21 ms/ques (similarity: 615.44 ms/ques), EDG generation: 117.43 ms/ques [INFO] Entity Linking P: 0.8780487804878049, R: 0.8536585365853658, micro F1: 0.8617886178861789, macro F1: 0.8656818962555823 [INFO] Relation Linking P: 0.6971544715447154, R: 0.6951219512195121, micro F1: 0.6926829268292682, macro F1: 0.6961367277906356 [INFO] Type Linking P: 0.47560975609756095, R: 0.47560975609756095, micro F1: 0.47560975609756095, macro F1: 0.47560975609756095 @@ -1155,8 +1155,8 @@ Question 82: Which serving railway line of the Daund Junction railway station wa [INFO] Golden Answer: [http://dbpedia.org/resource/Mumbai] [INFO] Predicted Answer: [http://dbpedia.org/resource/Mumbai] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 83, P: 0.687, R: 0.714, macro F1: 0.685, macro F1*: 0.700 -[INFO] Current time: 3022 ms, average: 3006.23 ms/ques, entity detection: 411.08 ms/ques (linking tool: 320.05 ms/ques), relation detection: 818.57 ms/ques (similarity: 617.78 ms/ques), EDG generation: 116.88 ms/ques +[INFO] Cumulative metrics, sample: 83, P: 0.687, R: 0.714, macro F1: 0.700 +[INFO] Current question consumed time: 2752 ms, average: 2859.67 ms/ques, entity detection: 401.69 ms/ques (linking tool: 315.66 ms/ques), relation detection: 802.87 ms/ques (similarity: 617.61 ms/ques), EDG generation: 117.63 ms/ques [INFO] Entity Linking P: 0.8795180722891566, R: 0.8554216867469879, micro F1: 0.8634538152610443, macro F1: 0.8673025435073627 [INFO] Relation Linking P: 0.7008032128514056, R: 0.6987951807228916, micro F1: 0.6963855421686747, macro F1: 0.6997977563049903 [INFO] Type Linking P: 0.46987951807228917, R: 0.46987951807228917, micro F1: 0.46987951807228917, macro F1: 0.46987951807228917 @@ -1169,8 +1169,8 @@ Question 83: List the ethnicity of Riley Reid? [INFO] Golden Answer: [http://dbpedia.org/resource/Dutch_Americans, http://dbpedia.org/resource/Cherokee, http://dbpedia.org/resource/Chickasaw, http://dbpedia.org/resource/German_Americans, http://dbpedia.org/resource/Irish_Americans, http://dbpedia.org/resource/Puerto_Ricans_in_the_United_States, http://dbpedia.org/resource/Dominican_Americans_(Dominican_Republic)] [INFO] Predicted Answer: [http://dbpedia.org/resource/Dutch_Americans, http://dbpedia.org/resource/Cherokee, http://dbpedia.org/resource/Chickasaw, http://dbpedia.org/resource/German_Americans, http://dbpedia.org/resource/Irish_Americans, http://dbpedia.org/resource/Puerto_Ricans_in_the_United_States, http://dbpedia.org/resource/Dominican_Americans_(Dominican_Republic)] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 84, P: 0.691, R: 0.718, macro F1: 0.689, macro F1*: 0.704 -[INFO] Current time: 628 ms, average: 2977.92 ms/ques, entity detection: 408.32 ms/ques (linking tool: 317.38 ms/ques), relation detection: 810.29 ms/ques (similarity: 611.32 ms/ques), EDG generation: 115.60 ms/ques +[INFO] Cumulative metrics, sample: 84, P: 0.691, R: 0.718, macro F1: 0.704 +[INFO] Current question consumed time: 534 ms, average: 2831.99 ms/ques, entity detection: 398.61 ms/ques (linking tool: 312.76 ms/ques), relation detection: 794.64 ms/ques (similarity: 611.12 ms/ques), EDG generation: 116.33 ms/ques [INFO] Entity Linking P: 0.8809523809523809, R: 0.8571428571428571, micro F1: 0.8650793650793651, macro F1: 0.8688845401174168 [INFO] Relation Linking P: 0.7043650793650793, R: 0.7023809523809523, micro F1: 0.7, macro F1: 0.7033716166297266 [INFO] Type Linking P: 0.47619047619047616, R: 0.47619047619047616, micro F1: 0.47619047619047616, macro F1: 0.47619047619047616 @@ -1183,8 +1183,8 @@ Question 84: Which shows are from a company whose subsidiary is Viacom Productio [INFO] Golden Answer: [http://dbpedia.org/resource/Family_Ties, http://dbpedia.org/resource/The_Tortellis, http://dbpedia.org/resource/Look_What's_Happened_to_Rosemary's_Baby, http://dbpedia.org/resource/Love_Story_(1973_TV_series), http://dbpedia.org/resource/Mission:_Impossible, http://dbpedia.org/resource/The_Brady_Bunch, http://dbpedia.org/resource/The_Lucy_Show, http://dbpedia.org/resource/Behind_the_Music, http://dbpedia.org/resource/The_Insider_(TV_series), http://dbpedia.org/resource/The_Parkers, http://dbpedia.org/resource/Webster_(TV_series), http://dbpedia.org/resource/Grease:_Live, http://dbpedia.org/resource/The_Sentinel_(TV_series), http://dbpedia.org/resource/Longstreet_(TV_series), http://dbpedia.org/resource/Sex,_Love_&_Secrets, http://dbpedia.org/resource/First_Monday, http://dbpedia.org/resource/Gung_Ho_(TV_series), http://dbpedia.org/resource/Kate_McShane, http://dbpedia.org/resource/The_Bad_News_Bears_(TV_series), http://dbpedia.org/resource/Goodtime_Girls, http://dbpedia.org/resource/LateLine, http://dbpedia.org/resource/Almost_Perfect, http://dbpedia.org/resource/The_Immortal_(TV_series), http://dbpedia.org/resource/Hudson_Street_(TV_series), http://dbpedia.org/resource/Flying_Blind_(TV_series), http://dbpedia.org/resource/Washington:_Behind_Closed_Doors, http://dbpedia.org/resource/Itsy_Bitsy_Spider_(TV_series), http://dbpedia.org/resource/Getting_Married_(film), http://dbpedia.org/resource/Big_Wave_Dave's, http://dbpedia.org/resource/Ghost_Whisperer, http://dbpedia.org/resource/Star_Trek:_The_Original_Series, http://dbpedia.org/resource/Star_Trek:_Voyager, http://dbpedia.org/resource/The_Young_Indiana_Jones_Chronicles, http://dbpedia.org/resource/MacGyver, http://dbpedia.org/resource/When_Things_Were_Rotten, http://dbpedia.org/resource/Cuts, http://dbpedia.org/resource/One_on_One_(TV_series), http://dbpedia.org/resource/Cybill, http://dbpedia.org/resource/Me_and_the_Chimp, http://dbpedia.org/resource/Kristin_(TV_series), http://dbpedia.org/resource/Hard_Knocks_(1987_TV_series), http://dbpedia.org/resource/The_Girl,_the_Gold_Watch_&_Everything_(film), http://dbpedia.org/resource/America_(U.S._TV_series), http://dbpedia.org/resource/Anything_for_Money, http://dbpedia.org/resource/Medium_(TV_series), http://dbpedia.org/resource/The_Adventures_of_Jimmy_Neutron:_Boy_Genius, http://dbpedia.org/resource/American_Dreamer_(TV_series), http://dbpedia.org/resource/Brooklyn_Bridge_(TV_series), http://dbpedia.org/resource/Laverne_&_Shirley, http://dbpedia.org/resource/Mork_&_Mindy, http://dbpedia.org/resource/Taxi_(TV_series), http://dbpedia.org/resource/That's_Life_(2000_TV_series), http://dbpedia.org/resource/Soul_Food_(TV_series), http://dbpedia.org/resource/Minority_Report_(TV_series), http://dbpedia.org/resource/Brothers_and_Sisters_(1979_TV_series), http://dbpedia.org/resource/Out_of_the_Blue_(1979_TV_series), http://dbpedia.org/resource/Hope_Island_(TV_series), http://dbpedia.org/resource/The_Busy_World_of_Richard_Scarry, http://dbpedia.org/resource/Wipeout_(1988_U.S._game_show), http://dbpedia.org/resource/Andy_Richter_Controls_the_Universe, http://dbpedia.org/resource/Busting_Loose_(TV_series), http://dbpedia.org/resource/Courting_Alex, http://dbpedia.org/resource/Joanie_Loves_Chachi, http://dbpedia.org/resource/The_Mommies_(TV_series), http://dbpedia.org/resource/The_Pursuit_of_Happiness_(TV_series), http://dbpedia.org/resource/Keen_Eddie, http://dbpedia.org/resource/Dog_and_Cat, http://dbpedia.org/resource/School_of_Rock_(TV_series), http://dbpedia.org/resource/Shooter_(TV_series), http://dbpedia.org/resource/The_Home_Court, http://dbpedia.org/resource/Seven_Days_(TV_series), http://dbpedia.org/resource/Barbary_Coast_(TV_series), http://dbpedia.org/resource/Diagnosis:_Murder, http://dbpedia.org/resource/Mannix, http://dbpedia.org/resource/Numbers_(TV_series), http://dbpedia.org/resource/Sons_and_Daughters_(1991_TV_series), http://dbpedia.org/resource/The_Magician_(U.S._TV_series), http://dbpedia.org/resource/Duckman, http://dbpedia.org/resource/Police_Squad!, http://dbpedia.org/resource/Shōgun_(miniseries), http://dbpedia.org/resource/The_Pentagon_Papers_(film), http://dbpedia.org/resource/Grandpa_Goes_to_Washington, http://dbpedia.org/resource/Love_&_Money, http://dbpedia.org/resource/Wolf_Lake, http://dbpedia.org/resource/Nightmare_on_the_13th_Floor, http://dbpedia.org/resource/Who's_Watching_the_Kids%3F, http://dbpedia.org/resource/Becker_(TV_series), http://dbpedia.org/resource/Cheers, http://dbpedia.org/resource/Love,_American_Style, http://dbpedia.org/resource/The_Untouchables_(1993_TV_series), http://dbpedia.org/resource/Brothers_(TV_series), http://dbpedia.org/resource/Deadwood_(TV_series), http://dbpedia.org/resource/Future_Cop_(TV_series), http://dbpedia.org/resource/The_Young_Lawyers, http://dbpedia.org/resource/Unexplained_Mysteries, http://dbpedia.org/resource/Bosom_Buddies, http://dbpedia.org/resource/The_Watcher_(TV_series), http://dbpedia.org/resource/The_Fonz_and_the_Happy_Days_Gang, http://dbpedia.org/resource/The_Firm_(2012_TV_series), http://dbpedia.org/resource/The_Odd_Couple_(1970_TV_series), http://dbpedia.org/resource/Paper_Moon_(TV_series), http://dbpedia.org/resource/Good_Company_(TV_series), http://dbpedia.org/resource/Charmed, http://dbpedia.org/resource/Frasier, http://dbpedia.org/resource/Here's_Boomer, http://dbpedia.org/resource/Star_Trek:_Enterprise, http://dbpedia.org/resource/NCIS_(TV_series), http://dbpedia.org/resource/Wings_(1990_TV_series), http://dbpedia.org/resource/Serpico_(TV_series), http://dbpedia.org/resource/Duet_(TV_series), http://dbpedia.org/resource/Fired_Up_(TV_series), http://dbpedia.org/resource/Girlfriends, http://dbpedia.org/resource/In-Laws, http://dbpedia.org/resource/Out_of_Practice, http://dbpedia.org/resource/Leeza, http://dbpedia.org/resource/Marblehead_Manor, http://dbpedia.org/resource/Call_to_Glory, http://dbpedia.org/resource/Clueless_(TV_series), http://dbpedia.org/resource/DiResta, http://dbpedia.org/resource/Marker_(TV_series), http://dbpedia.org/resource/Pig_Sty, http://dbpedia.org/resource/Platypus_Man, http://dbpedia.org/resource/The_Secret_Diary_of_Desmond_Pfeiffer, http://dbpedia.org/resource/Angie_(TV_series), http://dbpedia.org/resource/Sister,_Sister_(TV_series), http://dbpedia.org/resource/The_Jesse_Owens_Story, http://dbpedia.org/resource/The_Winds_of_War_(miniseries), http://dbpedia.org/resource/George_and_Leo, http://dbpedia.org/resource/Big_Apple_(TV_series), http://dbpedia.org/resource/Bram_&_Alice, http://dbpedia.org/resource/Dear_John_(U.S._TV_series), http://dbpedia.org/resource/Do_Over, http://dbpedia.org/resource/Dolphin_Cove_(TV_series), http://dbpedia.org/resource/E.A.R.T.H._Force, http://dbpedia.org/resource/Encore!_Encore!, http://dbpedia.org/resource/Flesh_'n'_Blood_(TV_series), http://dbpedia.org/resource/Grapevine_(TV_series), http://dbpedia.org/resource/His_&_Hers_(TV_series), http://dbpedia.org/resource/Hitz, http://dbpedia.org/resource/Hometown_(TV_series), http://dbpedia.org/resource/Jenny_(TV_series), http://dbpedia.org/resource/Life_on_a_Stick, http://dbpedia.org/resource/Maggie_(1998_TV_series), http://dbpedia.org/resource/Making_the_Grade_(TV_series), http://dbpedia.org/resource/Manhunt_(2001_TV_series), http://dbpedia.org/resource/Men,_Women_&_Dogs, http://dbpedia.org/resource/Mr._Smith_(TV_series), http://dbpedia.org/resource/Mulligan's_Stew, http://dbpedia.org/resource/Oh_Yeah!_Cartoons, http://dbpedia.org/resource/Raising_Dad, http://dbpedia.org/resource/Some_of_My_Best_Friends, http://dbpedia.org/resource/South_of_Sunset, http://dbpedia.org/resource/Struck_by_Lightning_(TV_series), http://dbpedia.org/resource/Szysznyk, http://dbpedia.org/resource/The_Bronx_Zoo_(TV_series), http://dbpedia.org/resource/The_Cavanaughs_(TV_series), http://dbpedia.org/resource/The_Marshal, http://dbpedia.org/resource/The_Random_Years, http://dbpedia.org/resource/The_Royal_Family_(TV_series), http://dbpedia.org/resource/The_Sandy_Duncan_Show, http://dbpedia.org/resource/All_is_Forgiven, http://dbpedia.org/resource/Vinyl_(TV_series), http://dbpedia.org/resource/Working_Stiffs_(TV_series), http://dbpedia.org/resource/Special_Unit_2, http://dbpedia.org/resource/A_Very_Brady_Christmas, http://dbpedia.org/resource/Day_by_Day_(TV_series), http://dbpedia.org/resource/Ferris_Bueller_(TV_series), http://dbpedia.org/resource/The_Brady_Kids, http://dbpedia.org/resource/The_Bradys, http://dbpedia.org/resource/Assault_on_the_Wayne, http://dbpedia.org/resource/Higher_Ground_(TV_series), http://dbpedia.org/resource/The_Powers_of_Matthew_Star, http://dbpedia.org/resource/Love,_Inc._(TV_series), http://dbpedia.org/resource/South_Beach_(2006_TV_series), http://dbpedia.org/resource/Second_Time_Around_(TV_series), http://dbpedia.org/resource/Barefoot_in_the_Park_(TV_series), http://dbpedia.org/resource/Blansky's_Beauties, http://dbpedia.org/resource/It's_All_Relative, http://dbpedia.org/resource/Madigan_Men, http://dbpedia.org/resource/Makin'_It_(TV_series), http://dbpedia.org/resource/Mr._Sunshine_(1986_TV_series), http://dbpedia.org/resource/Philly_(TV_series), http://dbpedia.org/resource/The_New_Odd_Couple, http://dbpedia.org/resource/The_Oddball_Couple, http://dbpedia.org/resource/The_Renegades, http://dbpedia.org/resource/The_Trouble_with_Normal_(TV_series), http://dbpedia.org/resource/Quarantined_(film), http://dbpedia.org/resource/Legend_(TV_series), http://dbpedia.org/resource/Open_House_(TV_series), http://dbpedia.org/resource/The_Brady_Bunch_Hour, http://dbpedia.org/resource/Viper_(TV_series), http://dbpedia.org/resource/Moesha, http://dbpedia.org/resource/Threshold_(TV_series), http://dbpedia.org/resource/Orleans_(TV_series), http://dbpedia.org/resource/Mission:_Impossible_(1988_TV_series), http://dbpedia.org/resource/Bob_(TV_series), http://dbpedia.org/resource/Petrocelli, http://dbpedia.org/resource/Medical_Investigation, http://dbpedia.org/resource/Hot_Ticket, http://dbpedia.org/resource/Foul_Play_(TV_series), http://dbpedia.org/resource/10,000_Black_Men_Named_George, http://dbpedia.org/resource/Ryan's_Four, http://dbpedia.org/resource/Down_Home_(TV_series)] [INFO] Predicted Answer: [] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 85, P: 0.683, R: 0.709, macro F1: 0.681, macro F1*: 0.696 -[INFO] Current time: 1829 ms, average: 2964.41 ms/ques, entity detection: 406.24 ms/ques (linking tool: 315.66 ms/ques), relation detection: 809.20 ms/ques (similarity: 611.05 ms/ques), EDG generation: 114.52 ms/ques +[INFO] Cumulative metrics, sample: 85, P: 0.683, R: 0.709, macro F1: 0.696 +[INFO] Current question consumed time: 1657 ms, average: 2818.16 ms/ques, entity detection: 396.15 ms/ques (linking tool: 310.71 ms/ques), relation detection: 793.33 ms/ques (similarity: 610.65 ms/ques), EDG generation: 115.22 ms/ques [INFO] Entity Linking P: 0.8705882352941177, R: 0.8470588235294118, micro F1: 0.8549019607843138, macro F1: 0.8586623690572119 [INFO] Relation Linking P: 0.696078431372549, R: 0.6941176470588235, micro F1: 0.691764705882353, macro F1: 0.6950966564340828 [INFO] Type Linking P: 0.47058823529411764, R: 0.47058823529411764, micro F1: 0.47058823529411764, macro F1: 0.47058823529411764 @@ -1197,8 +1197,8 @@ Question 85: Giuseppe Bertello and Pietro Parolin are leaders of which place ? [INFO] Golden Answer: [http://dbpedia.org/resource/Vatican_City] [INFO] Predicted Answer: [http://dbpedia.org/resource/Vatican_City] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 86, P: 0.686, R: 0.713, macro F1: 0.684, macro F1*: 0.699 -[INFO] Current time: 2351 ms, average: 2957.28 ms/ques, entity detection: 405.73 ms/ques (linking tool: 314.12 ms/ques), relation detection: 807.57 ms/ques (similarity: 613.21 ms/ques), EDG generation: 114.01 ms/ques +[INFO] Cumulative metrics, sample: 86, P: 0.686, R: 0.713, macro F1: 0.699 +[INFO] Current question consumed time: 2246 ms, average: 2811.51 ms/ques, entity detection: 395.83 ms/ques (linking tool: 309.27 ms/ques), relation detection: 791.58 ms/ques (similarity: 612.64 ms/ques), EDG generation: 114.71 ms/ques [INFO] Entity Linking P: 0.872093023255814, R: 0.8488372093023255, micro F1: 0.8565891472868218, macro F1: 0.8603079824010056 [INFO] Relation Linking P: 0.6996124031007752, R: 0.6976744186046512, micro F1: 0.6953488372093023, macro F1: 0.6986420668967519 [INFO] Type Linking P: 0.46511627906976744, R: 0.46511627906976744, micro F1: 0.46511627906976744, macro F1: 0.46511627906976744 @@ -1211,8 +1211,8 @@ Question 86: What is the university whose campus is Mount Moosilauke? [INFO] Golden Answer: [http://dbpedia.org/resource/Dartmouth_College] [INFO] Predicted Answer: [http://dbpedia.org/resource/Dartmouth_College] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 87, P: 0.690, R: 0.716, macro F1: 0.688, macro F1*: 0.703 -[INFO] Current time: 808 ms, average: 2932.57 ms/ques, entity detection: 402.77 ms/ques (linking tool: 311.38 ms/ques), relation detection: 800.93 ms/ques (similarity: 609.03 ms/ques), EDG generation: 112.89 ms/ques +[INFO] Cumulative metrics, sample: 87, P: 0.690, R: 0.716, macro F1: 0.703 +[INFO] Current question consumed time: 778 ms, average: 2788.14 ms/ques, entity detection: 393.02 ms/ques (linking tool: 306.54 ms/ques), relation detection: 785.07 ms/ques (similarity: 608.46 ms/ques), EDG generation: 113.55 ms/ques [INFO] Entity Linking P: 0.8735632183908046, R: 0.8505747126436781, micro F1: 0.8582375478927203, macro F1: 0.8619157088122607 [INFO] Relation Linking P: 0.7030651340996168, R: 0.7011494252873564, micro F1: 0.6988505747126437, macro F1: 0.7021059729344059 [INFO] Type Linking P: 0.45977011494252873, R: 0.45977011494252873, micro F1: 0.45977011494252873, macro F1: 0.45977011494252873 @@ -1225,8 +1225,8 @@ Question 87: How many religions are followed by the group whose architectural st [INFO] Golden Answer: [10] [INFO] Predicted Answer: [11] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 88, P: 0.682, R: 0.708, macro F1: 0.680, macro F1*: 0.695 -[INFO] Current time: 2903 ms, average: 2932.24 ms/ques, entity detection: 399.78 ms/ques (linking tool: 308.64 ms/ques), relation detection: 801.42 ms/ques (similarity: 612.19 ms/ques), EDG generation: 112.12 ms/ques +[INFO] Cumulative metrics, sample: 88, P: 0.682, R: 0.708, macro F1: 0.695 +[INFO] Current question consumed time: 2668 ms, average: 2786.77 ms/ques, entity detection: 390.05 ms/ques (linking tool: 303.88 ms/ques), relation detection: 785.45 ms/ques (similarity: 611.34 ms/ques), EDG generation: 112.76 ms/ques [INFO] Entity Linking P: 0.875, R: 0.8522727272727273, micro F1: 0.8598484848484849, macro F1: 0.8634868421052632 [INFO] Relation Linking P: 0.7064393939393939, R: 0.7045454545454546, micro F1: 0.7022727272727273, macro F1: 0.7054911531421598 [INFO] Type Linking P: 0.45454545454545453, R: 0.45454545454545453, micro F1: 0.45454545454545453, macro F1: 0.45454545454545453 @@ -1239,8 +1239,8 @@ Question 88: How many things are written in C++? [INFO] Golden Answer: [1001] [INFO] Predicted Answer: [] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 89, P: 0.674, R: 0.700, macro F1: 0.673, macro F1*: 0.687 -[INFO] Current time: 533 ms, average: 2905.28 ms/ques, entity detection: 395.30 ms/ques (linking tool: 305.17 ms/ques), relation detection: 796.51 ms/ques (similarity: 608.24 ms/ques), EDG generation: 111.03 ms/ques +[INFO] Cumulative metrics, sample: 89, P: 0.674, R: 0.700, macro F1: 0.687 +[INFO] Current question consumed time: 515 ms, average: 2761.25 ms/ques, entity detection: 385.66 ms/ques (linking tool: 300.46 ms/ques), relation detection: 780.69 ms/ques (similarity: 607.26 ms/ques), EDG generation: 111.61 ms/ques [INFO] Entity Linking P: 0.8651685393258427, R: 0.8426966292134831, micro F1: 0.8501872659176031, macro F1: 0.8537847427557658 [INFO] Relation Linking P: 0.698501872659176, R: 0.6966292134831461, micro F1: 0.69438202247191, macro F1: 0.6975642862529221 [INFO] Type Linking P: 0.4606741573033708, R: 0.4606741573033708, micro F1: 0.4606741573033708, macro F1: 0.4606741573033708 @@ -1253,8 +1253,8 @@ Question 89: Name the garrison of the Western Naval Command and line of the Vado [INFO] Golden Answer: [http://dbpedia.org/resource/Mumbai] [INFO] Predicted Answer: [http://dbpedia.org/resource/Mumbai] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 90, P: 0.678, R: 0.703, macro F1: 0.676, macro F1*: 0.690 -[INFO] Current time: 1347 ms, average: 2887.97 ms/ques, entity detection: 394.24 ms/ques (linking tool: 303.67 ms/ques), relation detection: 791.73 ms/ques (similarity: 604.07 ms/ques), EDG generation: 110.46 ms/ques +[INFO] Cumulative metrics, sample: 90, P: 0.678, R: 0.703, macro F1: 0.690 +[INFO] Current question consumed time: 1293 ms, average: 2744.93 ms/ques, entity detection: 384.69 ms/ques (linking tool: 299.07 ms/ques), relation detection: 775.87 ms/ques (similarity: 603.02 ms/ques), EDG generation: 110.79 ms/ques [INFO] Entity Linking P: 0.8666666666666667, R: 0.8444444444444444, micro F1: 0.8518518518518519, macro F1: 0.8554112554112554 [INFO] Relation Linking P: 0.7018518518518518, R: 0.7, micro F1: 0.6977777777777777, macro F1: 0.7009247027741082 [INFO] Type Linking P: 0.4666666666666667, R: 0.4666666666666667, micro F1: 0.4666666666666667, macro F1: 0.4666666666666667 @@ -1267,8 +1267,8 @@ Question 90: Who are the shareholder of the soccer club for whom Steve Holland p [INFO] Golden Answer: [http://dbpedia.org/resource/Roman_Abramovich] [INFO] Predicted Answer: [http://dbpedia.org/resource/Roman_Abramovich, http://dbpedia.org/resource/Gus_Mears] [INFO] P: 0.5, R: 1.0, F: 0.6666666666666666 -[INFO] Cumulative metrics, sample: 91, P: 0.676, R: 0.706, macro F1: 0.676, macro F1*: 0.691 -[INFO] Current time: 5949 ms, average: 2921.60 ms/ques, entity detection: 391.29 ms/ques (linking tool: 301.11 ms/ques), relation detection: 817.53 ms/ques (similarity: 614.04 ms/ques), EDG generation: 111.73 ms/ques +[INFO] Cumulative metrics, sample: 91, P: 0.676, R: 0.706, macro F1: 0.691 +[INFO] Current question consumed time: 5622 ms, average: 2776.55 ms/ques, entity detection: 381.84 ms/ques (linking tool: 296.55 ms/ques), relation detection: 800.97 ms/ques (similarity: 612.97 ms/ques), EDG generation: 109.89 ms/ques [INFO] Entity Linking P: 0.8681318681318682, R: 0.8461538461538461, micro F1: 0.8534798534798536, macro F1: 0.8570019723865877 [INFO] Relation Linking P: 0.7014652014652014, R: 0.7032967032967034, micro F1: 0.6989010989010989, macro F1: 0.70237975844234 [INFO] Type Linking P: 0.4725274725274725, R: 0.4725274725274725, micro F1: 0.4725274725274725, macro F1: 0.4725274725274725 @@ -1281,8 +1281,8 @@ Question 91: Which city located on Mediterranean Sea is also the stadium of Pani [INFO] Golden Answer: [http://dbpedia.org/resource/Athens] [INFO] Predicted Answer: [http://dbpedia.org/resource/Athens] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 92, P: 0.680, R: 0.710, macro F1: 0.680, macro F1*: 0.694 -[INFO] Current time: 4727 ms, average: 2941.23 ms/ques, entity detection: 395.63 ms/ques (linking tool: 305.15 ms/ques), relation detection: 823.32 ms/ques (similarity: 623.47 ms/ques), EDG generation: 111.76 ms/ques +[INFO] Cumulative metrics, sample: 92, P: 0.680, R: 0.710, macro F1: 0.694 +[INFO] Current question consumed time: 4695 ms, average: 2797.40 ms/ques, entity detection: 385.72 ms/ques (linking tool: 300.26 ms/ques), relation detection: 808.18 ms/ques (similarity: 622.55 ms/ques), EDG generation: 109.96 ms/ques [INFO] Entity Linking P: 0.8641304347826086, R: 0.842391304347826, micro F1: 0.8496376811594203, macro F1: 0.8531224037662698 [INFO] Relation Linking P: 0.7047101449275361, R: 0.7065217391304348, micro F1: 0.7021739130434782, macro F1: 0.7056147792599208 [INFO] Type Linking P: 0.4673913043478261, R: 0.4673913043478261, micro F1: 0.4673913043478261, macro F1: 0.4673913043478261 @@ -1295,8 +1295,8 @@ Question 92: Was Don't Bring Me Down recorded in West Germany? [INFO] Golden Answer: [true] [INFO] Predicted Answer: [true] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 93, P: 0.683, R: 0.713, macro F1: 0.683, macro F1*: 0.698 -[INFO] Current time: 9512 ms, average: 3011.88 ms/ques, entity detection: 443.20 ms/ques (linking tool: 353.16 ms/ques), relation detection: 831.01 ms/ques (similarity: 625.32 ms/ques), EDG generation: 110.73 ms/ques +[INFO] Cumulative metrics, sample: 93, P: 0.683, R: 0.713, macro F1: 0.698 +[INFO] Current question consumed time: 9247 ms, average: 2866.75 ms/ques, entity detection: 433.27 ms/ques (linking tool: 348.18 ms/ques), relation detection: 814.82 ms/ques (similarity: 624.39 ms/ques), EDG generation: 108.95 ms/ques [INFO] Entity Linking P: 0.8655913978494624, R: 0.8387096774193549, micro F1: 0.8476702508960574, macro F1: 0.8519385366846444 [INFO] Relation Linking P: 0.6971326164874552, R: 0.6989247311827957, micro F1: 0.6946236559139785, macro F1: 0.6980275235689539 [INFO] Type Linking P: 0.4731182795698925, R: 0.4731182795698925, micro F1: 0.4731182795698925, macro F1: 0.4731182795698925 @@ -1309,8 +1309,8 @@ Question 93: Through which cities do the rivers starting at the Montauk State Pa [INFO] Golden Answer: [http://dbpedia.org/resource/Van_Buren,_Missouri, http://dbpedia.org/resource/Missouri, http://dbpedia.org/resource/Doniphan,_Missouri] [INFO] Predicted Answer: [http://dbpedia.org/resource/Van_Buren,_Missouri, http://dbpedia.org/resource/Doniphan,_Missouri] [INFO] P: 1.0, R: 0.6666666666666666, F: 0.8 -[INFO] Cumulative metrics, sample: 94, P: 0.686, R: 0.712, macro F1: 0.684, macro F1*: 0.699 -[INFO] Current time: 2068 ms, average: 3001.84 ms/ques, entity detection: 440.17 ms/ques (linking tool: 350.35 ms/ques), relation detection: 829.40 ms/ques (similarity: 627.21 ms/ques), EDG generation: 110.50 ms/ques +[INFO] Cumulative metrics, sample: 94, P: 0.686, R: 0.712, macro F1: 0.699 +[INFO] Current question consumed time: 1937 ms, average: 2856.86 ms/ques, entity detection: 430.12 ms/ques (linking tool: 345.32 ms/ques), relation detection: 813.16 ms/ques (similarity: 626.31 ms/ques), EDG generation: 108.84 ms/ques [INFO] Entity Linking P: 0.8670212765957447, R: 0.8404255319148937, micro F1: 0.8492907801418441, macro F1: 0.8535162722873998 [INFO] Relation Linking P: 0.7003546099290779, R: 0.7021276595744681, micro F1: 0.6978723404255318, macro F1: 0.7012400139871425 [INFO] Type Linking P: 0.46808510638297873, R: 0.46808510638297873, micro F1: 0.46808510638297873, macro F1: 0.46808510638297873 @@ -1323,8 +1323,8 @@ Question 94: How many ideologies do the political party of Boonchu Rojanastien h [INFO] Golden Answer: [8] [INFO] Predicted Answer: [6] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 95, P: 0.679, R: 0.705, macro F1: 0.677, macro F1*: 0.692 -[INFO] Current time: 2816 ms, average: 2999.89 ms/ques, entity detection: 436.24 ms/ques (linking tool: 347.17 ms/ques), relation detection: 826.80 ms/ques (similarity: 625.98 ms/ques), EDG generation: 110.37 ms/ques +[INFO] Cumulative metrics, sample: 95, P: 0.679, R: 0.705, macro F1: 0.692 +[INFO] Current question consumed time: 2366 ms, average: 2851.69 ms/ques, entity detection: 426.12 ms/ques (linking tool: 342.04 ms/ques), relation detection: 810.00 ms/ques (similarity: 625.05 ms/ques), EDG generation: 108.68 ms/ques [INFO] Entity Linking P: 0.868421052631579, R: 0.8421052631578947, micro F1: 0.8508771929824562, macro F1: 0.8550607287449393 [INFO] Relation Linking P: 0.7035087719298245, R: 0.7052631578947368, micro F1: 0.7010526315789473, macro F1: 0.7043848725175329 [INFO] Type Linking P: 0.4631578947368421, R: 0.4631578947368421, micro F1: 0.4631578947368421, macro F1: 0.4631578947368421 @@ -1337,8 +1337,8 @@ Question 95: What are the hubs of PLUS Helicopter Services, a highway helicopter [INFO] Golden Answer: [http://dbpedia.org/resource/Selangor, http://dbpedia.org/resource/Malaysia, http://dbpedia.org/resource/Sultan_Abdul_Aziz_Shah_Airport, http://dbpedia.org/resource/Subang,_Indonesia] [INFO] Predicted Answer: [] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 96, P: 0.672, R: 0.697, macro F1: 0.670, macro F1*: 0.684 -[INFO] Current time: 3262 ms, average: 3002.62 ms/ques, entity detection: 436.17 ms/ques (linking tool: 347.26 ms/ques), relation detection: 831.07 ms/ques (similarity: 626.66 ms/ques), EDG generation: 109.66 ms/ques +[INFO] Cumulative metrics, sample: 96, P: 0.672, R: 0.697, macro F1: 0.684 +[INFO] Current question consumed time: 2913 ms, average: 2852.33 ms/ques, entity detection: 425.30 ms/ques (linking tool: 341.45 ms/ques), relation detection: 813.42 ms/ques (similarity: 625.36 ms/ques), EDG generation: 107.96 ms/ques [INFO] Entity Linking P: 0.859375, R: 0.8333333333333334, micro F1: 0.842013888888889, macro F1: 0.8461538461538461 [INFO] Relation Linking P: 0.6961805555555555, R: 0.6979166666666666, micro F1: 0.69375, macro F1: 0.6970475300954753 [INFO] Type Linking P: 0.46875, R: 0.46875, micro F1: 0.46875, macro F1: 0.46875 @@ -1351,8 +1351,8 @@ Question 96: Name the movies for which music was given by Geoffrey Burgon and d [INFO] Golden Answer: [http://dbpedia.org/resource/Monty_Python's_Life_of_Brian] [INFO] Predicted Answer: [http://dbpedia.org/resource/Monty_Python's_Life_of_Brian] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 97, P: 0.675, R: 0.701, macro F1: 0.674, macro F1*: 0.688 -[INFO] Current time: 1388 ms, average: 2985.98 ms/ques, entity detection: 435.48 ms/ques (linking tool: 345.71 ms/ques), relation detection: 824.44 ms/ques (similarity: 622.18 ms/ques), EDG generation: 109.25 ms/ques +[INFO] Cumulative metrics, sample: 97, P: 0.675, R: 0.701, macro F1: 0.688 +[INFO] Current question consumed time: 1204 ms, average: 2835.34 ms/ques, entity detection: 424.27 ms/ques (linking tool: 339.66 ms/ques), relation detection: 806.92 ms/ques (similarity: 620.90 ms/ques), EDG generation: 107.48 ms/ques [INFO] Entity Linking P: 0.8608247422680413, R: 0.8350515463917526, micro F1: 0.8436426116838489, macro F1: 0.8477422993764296 [INFO] Relation Linking P: 0.6941580756013745, R: 0.6958762886597938, micro F1: 0.6917525773195876, macro F1: 0.6950161201942069 [INFO] Type Linking P: 0.4639175257731959, R: 0.4639175257731959, micro F1: 0.4639175257731959, macro F1: 0.4639175257731959 @@ -1365,8 +1365,8 @@ Question 97: Is Henry David Thoreau interested in Politics? [INFO] Golden Answer: [true] [INFO] Predicted Answer: [true] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 98, P: 0.679, R: 0.704, macro F1: 0.677, macro F1*: 0.691 -[INFO] Current time: 1559 ms, average: 2971.42 ms/ques, entity detection: 435.90 ms/ques (linking tool: 345.53 ms/ques), relation detection: 820.92 ms/ques (similarity: 618.40 ms/ques), EDG generation: 108.21 ms/ques +[INFO] Cumulative metrics, sample: 98, P: 0.679, R: 0.704, macro F1: 0.691 +[INFO] Current question consumed time: 1442 ms, average: 2821.12 ms/ques, entity detection: 424.55 ms/ques (linking tool: 339.28 ms/ques), relation detection: 803.64 ms/ques (similarity: 617.09 ms/ques), EDG generation: 106.47 ms/ques [INFO] Entity Linking P: 0.8622448979591837, R: 0.8367346938775511, micro F1: 0.8452380952380953, macro F1: 0.8492982778697065 [INFO] Relation Linking P: 0.6972789115646258, R: 0.6989795918367347, micro F1: 0.6948979591836734, macro F1: 0.6981282159636083 [INFO] Type Linking P: 0.46938775510204084, R: 0.46938775510204084, micro F1: 0.46938775510204084, macro F1: 0.46938775510204084 @@ -1379,8 +1379,8 @@ Question 98: Among the companies who has product named Chinook who has a key per [INFO] Golden Answer: [http://dbpedia.org/resource/Boeing] [INFO] Predicted Answer: [http://dbpedia.org/resource/Boeing] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 99, P: 0.682, R: 0.707, macro F1: 0.680, macro F1*: 0.694 -[INFO] Current time: 3748 ms, average: 2979.26 ms/ques, entity detection: 434.81 ms/ques (linking tool: 343.72 ms/ques), relation detection: 830.23 ms/ques (similarity: 623.33 ms/ques), EDG generation: 107.57 ms/ques +[INFO] Cumulative metrics, sample: 99, P: 0.682, R: 0.707, macro F1: 0.694 +[INFO] Current question consumed time: 3527 ms, average: 2828.25 ms/ques, entity detection: 423.43 ms/ques (linking tool: 337.41 ms/ques), relation detection: 813.03 ms/ques (similarity: 621.94 ms/ques), EDG generation: 105.79 ms/ques [INFO] Entity Linking P: 0.8636363636363636, R: 0.8333333333333334, micro F1: 0.8434343434343435, macro F1: 0.8482142857142857 [INFO] Relation Linking P: 0.6902356902356902, R: 0.6919191919191919, micro F1: 0.6878787878787879, macro F1: 0.6910764158023598 [INFO] Type Linking P: 0.46464646464646464, R: 0.46464646464646464, micro F1: 0.46464646464646464, macro F1: 0.46464646464646464 @@ -1393,8 +1393,8 @@ Question 99: Which country's leader is Giuseppe Bertello? [INFO] Golden Answer: [http://dbpedia.org/resource/Vatican_City] [INFO] Predicted Answer: [http://dbpedia.org/resource/Vatican_City] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 100, P: 0.685, R: 0.709, macro F1: 0.683, macro F1*: 0.697 -[INFO] Current time: 1151 ms, average: 2960.98 ms/ques, entity detection: 432.79 ms/ques (linking tool: 341.38 ms/ques), relation detection: 825.55 ms/ques (similarity: 621.19 ms/ques), EDG generation: 106.63 ms/ques +[INFO] Cumulative metrics, sample: 100, P: 0.685, R: 0.709, macro F1: 0.697 +[INFO] Current question consumed time: 1008 ms, average: 2810.05 ms/ques, entity detection: 420.87 ms/ques (linking tool: 334.80 ms/ques), relation detection: 808.27 ms/ques (similarity: 619.73 ms/ques), EDG generation: 104.84 ms/ques [INFO] Entity Linking P: 0.865, R: 0.835, micro F1: 0.8450000000000002, macro F1: 0.849735294117647 [INFO] Relation Linking P: 0.6933333333333332, R: 0.695, micro F1: 0.691, macro F1: 0.6941656662665066 [INFO] Type Linking P: 0.46, R: 0.46, micro F1: 0.46, macro F1: 0.46 @@ -1407,8 +1407,8 @@ Question 100: Who wrote the subsequent work of One Day at a Time (Em's Version) [INFO] Golden Answer: [http://dbpedia.org/resource/Nate_Dogg, http://dbpedia.org/resource/Tupac_Shakur, http://dbpedia.org/resource/Eminem] [INFO] Predicted Answer: [http://dbpedia.org/resource/Easy_Mo_Bee, http://dbpedia.org/resource/Luis_Resto_(musician), http://dbpedia.org/resource/The_Notorious_B.I.G., http://dbpedia.org/resource/Tupac_Shakur, http://dbpedia.org/resource/Edgar_Winter, http://dbpedia.org/resource/Eminem, http://dbpedia.org/resource/Nate_Dogg, T. Shakur, F. Ross@en, C. Wallace, E. Winter, L. Resto, M. Mathers, O. Harvey, T. Shakur@en, T. Shakur, M. Mathers, N. Hale@en, T. Shakur, A. Thelusma, M. Greenidge, P. Brown@en] [INFO] P: 0.2727272727272727, R: 1.0, F: 0.42857142857142855 -[INFO] Cumulative metrics, sample: 101, P: 0.681, R: 0.712, macro F1: 0.681, macro F1*: 0.696 -[INFO] Current time: 2574 ms, average: 2957.15 ms/ques, entity detection: 433.64 ms/ques (linking tool: 342.34 ms/ques), relation detection: 823.78 ms/ques (similarity: 619.01 ms/ques), EDG generation: 106.39 ms/ques +[INFO] Cumulative metrics, sample: 101, P: 0.681, R: 0.712, macro F1: 0.696 +[INFO] Current question consumed time: 2345 ms, average: 2805.45 ms/ques, entity detection: 421.37 ms/ques (linking tool: 335.43 ms/ques), relation detection: 806.31 ms/ques (similarity: 617.46 ms/ques), EDG generation: 104.59 ms/ques [INFO] Entity Linking P: 0.8663366336633663, R: 0.8366336633663366, micro F1: 0.8465346534653466, macro F1: 0.8512261109831912 [INFO] Relation Linking P: 0.6963696369636964, R: 0.698019801980198, micro F1: 0.694059405940594, macro F1: 0.6971937430429432 [INFO] Type Linking P: 0.46534653465346537, R: 0.46534653465346537, micro F1: 0.46534653465346537, macro F1: 0.46534653465346537 @@ -1421,8 +1421,8 @@ Question 101: Name the TV shows with network as NBC and company as Paramount Tel [INFO] Golden Answer: [http://dbpedia.org/resource/LateLine, http://dbpedia.org/resource/Star_Trek:_The_Original_Series, http://dbpedia.org/resource/Kristin_(TV_series), http://dbpedia.org/resource/Medium_(TV_series), http://dbpedia.org/resource/Taxi_(TV_series), http://dbpedia.org/resource/The_Pursuit_of_Happiness_(TV_series), http://dbpedia.org/resource/The_Home_Court, http://dbpedia.org/resource/The_Magician_(U.S._TV_series), http://dbpedia.org/resource/Grandpa_Goes_to_Washington, http://dbpedia.org/resource/Who's_Watching_the_Kids%3F, http://dbpedia.org/resource/Cheers, http://dbpedia.org/resource/The_Firm_(2012_TV_series), http://dbpedia.org/resource/Frasier, http://dbpedia.org/resource/Wings_(1990_TV_series), http://dbpedia.org/resource/Fired_Up_(TV_series), http://dbpedia.org/resource/Leeza, http://dbpedia.org/resource/His_&_Hers_(TV_series), http://dbpedia.org/resource/The_Bronx_Zoo_(TV_series), http://dbpedia.org/resource/All_is_Forgiven, http://dbpedia.org/resource/Petrocelli, http://dbpedia.org/resource/Medical_Investigation, http://dbpedia.org/resource/Down_Home_(TV_series)] [INFO] Predicted Answer: [http://dbpedia.org/resource/LateLine, http://dbpedia.org/resource/Star_Trek:_The_Original_Series, http://dbpedia.org/resource/Kristin_(TV_series), http://dbpedia.org/resource/Medium_(TV_series), http://dbpedia.org/resource/Taxi_(TV_series), http://dbpedia.org/resource/The_Pursuit_of_Happiness_(TV_series), http://dbpedia.org/resource/The_Home_Court, http://dbpedia.org/resource/The_Magician_(U.S._TV_series), http://dbpedia.org/resource/Grandpa_Goes_to_Washington, http://dbpedia.org/resource/Who's_Watching_the_Kids%3F, http://dbpedia.org/resource/Cheers, http://dbpedia.org/resource/The_Firm_(2012_TV_series), http://dbpedia.org/resource/Frasier, http://dbpedia.org/resource/Wings_(1990_TV_series), http://dbpedia.org/resource/Fired_Up_(TV_series), http://dbpedia.org/resource/Leeza, http://dbpedia.org/resource/His_&_Hers_(TV_series), http://dbpedia.org/resource/The_Bronx_Zoo_(TV_series), http://dbpedia.org/resource/All_is_Forgiven, http://dbpedia.org/resource/Petrocelli, http://dbpedia.org/resource/Medical_Investigation, http://dbpedia.org/resource/Down_Home_(TV_series)] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 102, P: 0.684, R: 0.715, macro F1: 0.684, macro F1*: 0.699 -[INFO] Current time: 5052 ms, average: 2977.69 ms/ques, entity detection: 440.82 ms/ques (linking tool: 348.80 ms/ques), relation detection: 833.39 ms/ques (similarity: 621.47 ms/ques), EDG generation: 105.80 ms/ques +[INFO] Cumulative metrics, sample: 102, P: 0.684, R: 0.715, macro F1: 0.699 +[INFO] Current question consumed time: 4709 ms, average: 2824.11 ms/ques, entity detection: 427.86 ms/ques (linking tool: 341.27 ms/ques), relation detection: 815.20 ms/ques (similarity: 619.88 ms/ques), EDG generation: 103.97 ms/ques [INFO] Entity Linking P: 0.8676470588235294, R: 0.8382352941176471, micro F1: 0.8480392156862746, macro F1: 0.8526876267748479 [INFO] Relation Linking P: 0.6993464052287581, R: 0.7009803921568627, micro F1: 0.6970588235294117, macro F1: 0.7001624453748827 [INFO] Type Linking P: 0.46078431372549017, R: 0.46078431372549017, micro F1: 0.46078431372549017, macro F1: 0.46078431372549017 @@ -1435,8 +1435,8 @@ Question 102: Name the fictional character whose voice's animator is Hardie Gram [INFO] Golden Answer: [http://dbpedia.org/resource/Mickey_Mouse] [INFO] Predicted Answer: [http://dbpedia.org/resource/Mickey_Mouse] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 103, P: 0.687, R: 0.718, macro F1: 0.687, macro F1*: 0.702 -[INFO] Current time: 1350 ms, average: 2961.88 ms/ques, entity detection: 438.34 ms/ques (linking tool: 346.33 ms/ques), relation detection: 829.35 ms/ques (similarity: 618.99 ms/ques), EDG generation: 105.00 ms/ques +[INFO] Cumulative metrics, sample: 103, P: 0.687, R: 0.718, macro F1: 0.702 +[INFO] Current question consumed time: 1327 ms, average: 2809.57 ms/ques, entity detection: 425.41 ms/ques (linking tool: 338.78 ms/ques), relation detection: 811.34 ms/ques (similarity: 617.44 ms/ques), EDG generation: 103.17 ms/ques [INFO] Entity Linking P: 0.8689320388349514, R: 0.8398058252427184, micro F1: 0.8495145631067963, macro F1: 0.8541206972639012 [INFO] Relation Linking P: 0.7022653721682848, R: 0.7038834951456311, micro F1: 0.7, macro F1: 0.7030735026310793 [INFO] Type Linking P: 0.4563106796116505, R: 0.4563106796116505, micro F1: 0.4563106796116505, macro F1: 0.4563106796116505 @@ -1449,8 +1449,8 @@ Question 103: Which license provider of MSX basic is also the designer of Langua [INFO] Golden Answer: [http://dbpedia.org/resource/Microsoft] [INFO] Predicted Answer: [http://dbpedia.org/resource/Microsoft] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 104, P: 0.690, R: 0.721, macro F1: 0.690, macro F1*: 0.705 -[INFO] Current time: 1562 ms, average: 2948.42 ms/ques, entity detection: 437.32 ms/ques (linking tool: 344.87 ms/ques), relation detection: 825.65 ms/ques (similarity: 615.43 ms/ques), EDG generation: 104.95 ms/ques +[INFO] Cumulative metrics, sample: 104, P: 0.690, R: 0.721, macro F1: 0.705 +[INFO] Current question consumed time: 1496 ms, average: 2796.94 ms/ques, entity detection: 424.41 ms/ques (linking tool: 337.38 ms/ques), relation detection: 807.46 ms/ques (similarity: 613.83 ms/ques), EDG generation: 103.33 ms/ques [INFO] Entity Linking P: 0.8701923076923077, R: 0.8413461538461539, micro F1: 0.8509615384615385, macro F1: 0.8555261452031115 [INFO] Relation Linking P: 0.7051282051282051, R: 0.7067307692307693, micro F1: 0.7028846153846153, macro F1: 0.7059285776652405 [INFO] Type Linking P: 0.46153846153846156, R: 0.46153846153846156, micro F1: 0.46153846153846156, macro F1: 0.46153846153846156 @@ -1463,8 +1463,8 @@ Question 104: Who are the architect of the stadium whose one of the architect is [INFO] Golden Answer: [http://dbpedia.org/resource/Populous_(company), http://dbpedia.org/resource/Louis_D._Astorino, Architectural Innovations@en, Fukui Architects@en, Lami Grubb@en, http://dbpedia.org/resource/Pittsburgh, Fawley Bryant Architects, Bradenton, Florida@en] [INFO] Predicted Answer: [http://dbpedia.org/resource/Populous_(company), http://dbpedia.org/resource/Louis_D._Astorino, http://www.fawley-bryant.com, Architectural Innovations@en, Fukui Architects@en, Lami Grubb@en, http://dbpedia.org/resource/Pittsburgh, Fawley Bryant Architects, Bradenton, Florida@en] [INFO] P: 0.875, R: 1.0, F: 0.9333333333333333 -[INFO] Cumulative metrics, sample: 105, P: 0.692, R: 0.723, macro F1: 0.692, macro F1*: 0.707 -[INFO] Current time: 1579 ms, average: 2935.38 ms/ques, entity detection: 434.71 ms/ques (linking tool: 342.43 ms/ques), relation detection: 823.88 ms/ques (similarity: 613.72 ms/ques), EDG generation: 104.30 ms/ques +[INFO] Cumulative metrics, sample: 105, P: 0.692, R: 0.723, macro F1: 0.707 +[INFO] Current question consumed time: 1498 ms, average: 2784.57 ms/ques, entity detection: 421.73 ms/ques (linking tool: 334.90 ms/ques), relation detection: 806.57 ms/ques (similarity: 612.01 ms/ques), EDG generation: 102.68 ms/ques [INFO] Entity Linking P: 0.8714285714285714, R: 0.8428571428571429, micro F1: 0.8523809523809525, macro F1: 0.8569047619047618 [INFO] Relation Linking P: 0.7079365079365079, R: 0.7095238095238096, micro F1: 0.7057142857142856, macro F1: 0.7087292699834693 [INFO] Type Linking P: 0.45714285714285713, R: 0.45714285714285713, micro F1: 0.45714285714285713, macro F1: 0.45714285714285713 @@ -1477,8 +1477,8 @@ Question 105: Who are the astronauts associated with NASA? [INFO] Golden Answer: [http://dbpedia.org/resource/Michael_J._McCulley, http://dbpedia.org/resource/Bill_Dana_(pilot), http://dbpedia.org/resource/Donald_R._McMonagle, http://dbpedia.org/resource/Frank_L._Culbertson,_Jr., http://dbpedia.org/resource/David_M._Walker_(astronaut), http://dbpedia.org/resource/Kent_Rominger, http://dbpedia.org/resource/Michael_Foreman_(astronaut), http://dbpedia.org/resource/Kenneth_S._Reightler,_Jr., http://dbpedia.org/resource/Alan_Shepard, http://dbpedia.org/resource/Bruce_McCandless_II, http://dbpedia.org/resource/Buzz_Aldrin, http://dbpedia.org/resource/Charles_Duke, http://dbpedia.org/resource/David_Scott, http://dbpedia.org/resource/Deke_Slayton, http://dbpedia.org/resource/Don_L._Lind, http://dbpedia.org/resource/Edward_Higgins_White, http://dbpedia.org/resource/Eugene_Cernan, http://dbpedia.org/resource/Frank_Borman, http://dbpedia.org/resource/Fred_Haise, http://dbpedia.org/resource/Gerald_P._Carr, http://dbpedia.org/resource/Guion_Bluford, http://dbpedia.org/resource/James_McDivitt, http://dbpedia.org/resource/John_S._Bull, http://dbpedia.org/resource/John_Young_(astronaut), http://dbpedia.org/resource/Joseph_A._Walker, http://dbpedia.org/resource/Ken_Mattingly, http://dbpedia.org/resource/Michael_Collins_(astronaut), http://dbpedia.org/resource/Michael_J._Smith_(astronaut), http://dbpedia.org/resource/Neil_Armstrong, http://dbpedia.org/resource/Owen_K._Garriott, http://dbpedia.org/resource/Ronald_Evans_(astronaut), http://dbpedia.org/resource/Rusty_Schweickart, http://dbpedia.org/resource/Vance_D._Brand, http://dbpedia.org/resource/Walter_Cunningham, http://dbpedia.org/resource/William_Anders, http://dbpedia.org/resource/William_R._Pogue, http://dbpedia.org/resource/Donald_E._Williams, http://dbpedia.org/resource/Rick_Husband, http://dbpedia.org/resource/William_C._McCool, http://dbpedia.org/resource/Richard_A._Searfoss, http://dbpedia.org/resource/Bryan_D._O'Connor, http://dbpedia.org/resource/Terry_Hart, http://dbpedia.org/resource/Alan_Bean, http://dbpedia.org/resource/Alfred_Worden, http://dbpedia.org/resource/Anthony_W._England, http://dbpedia.org/resource/C._Gordon_Fullerton, http://dbpedia.org/resource/Clifton_Williams, http://dbpedia.org/resource/Curt_Michel, http://dbpedia.org/resource/Donald_H._Peterson, http://dbpedia.org/resource/Donn_F._Eisele, http://dbpedia.org/resource/Edgar_Mitchell, http://dbpedia.org/resource/Edward_Gibson, http://dbpedia.org/resource/Gordon_Cooper, http://dbpedia.org/resource/Henry_Hartsfield, http://dbpedia.org/resource/Jack_R._Lousma, http://dbpedia.org/resource/James_Irwin, http://dbpedia.org/resource/Joe_Engle, http://dbpedia.org/resource/Joseph_P._Allen, http://dbpedia.org/resource/Joseph_P._Kerwin, http://dbpedia.org/resource/Karl_Gordon_Henize, http://dbpedia.org/resource/Paul_J._Weitz, http://dbpedia.org/resource/Pete_Conrad, http://dbpedia.org/resource/Richard_F._Gordon,_Jr., http://dbpedia.org/resource/Robert_A._Parker, http://dbpedia.org/resource/Robert_F._Overmyer, http://dbpedia.org/resource/Roger_B._Chaffee, http://dbpedia.org/resource/Steven_R._Nagel, http://dbpedia.org/resource/Stuart_Roosa, http://dbpedia.org/resource/Theodore_Freeman, http://dbpedia.org/resource/Thomas_P._Stafford, http://dbpedia.org/resource/Wally_Schirra, http://dbpedia.org/resource/Robert_L._Gibson, http://dbpedia.org/resource/Jon_McBride, http://dbpedia.org/resource/Donald_Holmquest, http://dbpedia.org/resource/Frederick_W._Sturckow, http://dbpedia.org/resource/Frederick_D._Gregory, http://dbpedia.org/resource/Story_Musgrave, http://dbpedia.org/resource/Edward_Givens, http://dbpedia.org/resource/George_Nelson_(astronaut), http://dbpedia.org/resource/Michael_Fincke, http://dbpedia.org/resource/John_Casper, http://dbpedia.org/resource/John_M._Fabian, http://dbpedia.org/resource/Jim_Wetherbee, http://dbpedia.org/resource/Milton_Orville_Thompson, http://dbpedia.org/resource/Timothy_Kopra, http://dbpedia.org/resource/Mark_N._Brown, http://dbpedia.org/resource/Charles_Bassett, http://dbpedia.org/resource/Eileen_Collins, http://dbpedia.org/resource/Elliot_See, http://dbpedia.org/resource/Richard_H._Truly, http://dbpedia.org/resource/Piers_Sellers, http://dbpedia.org/resource/William_Shepherd, http://dbpedia.org/resource/Kevin_P._Chilton, http://dbpedia.org/resource/Brent_W._Jett,_Jr., http://dbpedia.org/resource/F._Drew_Gaffney, http://dbpedia.org/resource/Gregory_C._Johnson, http://dbpedia.org/resource/James_H._Newman, http://dbpedia.org/resource/John_B._McKay, http://dbpedia.org/resource/Kenneth_D._Cameron, http://dbpedia.org/resource/Mark_T._Vande_Hei, http://dbpedia.org/resource/Richard_N._Richards, http://dbpedia.org/resource/Ronald_J._Grabe, http://dbpedia.org/resource/Steven_Lindsey, http://dbpedia.org/resource/Scott_Carpenter, http://dbpedia.org/resource/William_B._Lenoir, http://dbpedia.org/resource/Brewster_H._Shaw, http://dbpedia.org/resource/Daniel_W._Bursch, http://dbpedia.org/resource/Daniel_Brandenstein, http://dbpedia.org/resource/Charles_L._Veach, http://dbpedia.org/resource/Michael_R._Clifford, http://dbpedia.org/resource/Robert_Crippen, http://dbpedia.org/resource/Bruce_E._Melnick, http://dbpedia.org/resource/James_van_Hoften, http://dbpedia.org/resource/Karol_J._Bobko] [INFO] Predicted Answer: [http://dbpedia.org/resource/Robert_M._Lightfoot,_Jr., http://dbpedia.org/resource/Robin_Henderson, http://dbpedia.org/resource/J._Wayne_Littles, http://dbpedia.org/resource/Patrick_Scheuermann, http://dbpedia.org/resource/Glynn_Lunney, http://dbpedia.org/resource/Ben_Parris, http://dbpedia.org/resource/Rob_Strain, http://dbpedia.org/resource/Christopher_C._Kraft,_Jr., http://dbpedia.org/resource/Edgar_Cortright, http://dbpedia.org/resource/Joseph_Francis_Shea, http://dbpedia.org/resource/Robert_R._Gilruth, http://dbpedia.org/resource/Norman_L._Crabill, http://dbpedia.org/resource/Joe_D._Dowdy, http://dbpedia.org/resource/David_Brin, http://dbpedia.org/resource/Daniel_O._Graham, http://dbpedia.org/resource/Roy_Marlin_Voris, http://dbpedia.org/resource/Dennis_M._Bushnell, http://dbpedia.org/resource/George_Diller, http://dbpedia.org/resource/Erkin_Sidick, http://dbpedia.org/resource/Addison_Bain, http://dbpedia.org/resource/Carolyn_Huntoon] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 106, P: 0.686, R: 0.716, macro F1: 0.686, macro F1*: 0.701 -[INFO] Current time: 2535 ms, average: 2931.60 ms/ques, entity detection: 434.97 ms/ques (linking tool: 342.63 ms/ques), relation detection: 826.35 ms/ques (similarity: 613.55 ms/ques), EDG generation: 103.43 ms/ques +[INFO] Cumulative metrics, sample: 106, P: 0.686, R: 0.716, macro F1: 0.701 +[INFO] Current question consumed time: 2487 ms, average: 2781.76 ms/ques, entity detection: 422.08 ms/ques (linking tool: 335.20 ms/ques), relation detection: 809.25 ms/ques (similarity: 611.82 ms/ques), EDG generation: 101.82 ms/ques [INFO] Entity Linking P: 0.8726415094339622, R: 0.8443396226415094, micro F1: 0.8537735849056605, macro F1: 0.8582573087290069 [INFO] Relation Linking P: 0.7012578616352201, R: 0.7028301886792453, micro F1: 0.6990566037735848, macro F1: 0.7020431447949461 [INFO] Type Linking P: 0.4528301886792453, R: 0.4528301886792453, micro F1: 0.4528301886792453, macro F1: 0.4528301886792453 @@ -1491,8 +1491,8 @@ Question 106: What is a common ingredient used in boyt Blodpalt and Kaszanka ? [INFO] Golden Answer: [http://dbpedia.org/resource/Blood] [INFO] Predicted Answer: [http://dbpedia.org/resource/Blood] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 107, P: 0.688, R: 0.719, macro F1: 0.689, macro F1*: 0.703 -[INFO] Current time: 550 ms, average: 2909.35 ms/ques, entity detection: 431.63 ms/ques (linking tool: 339.98 ms/ques), relation detection: 820.47 ms/ques (similarity: 609.11 ms/ques), EDG generation: 103.03 ms/ques +[INFO] Cumulative metrics, sample: 107, P: 0.688, R: 0.719, macro F1: 0.703 +[INFO] Current question consumed time: 516 ms, average: 2760.59 ms/ques, entity detection: 418.82 ms/ques (linking tool: 332.55 ms/ques), relation detection: 803.24 ms/ques (similarity: 607.14 ms/ques), EDG generation: 101.38 ms/ques [INFO] Entity Linking P: 0.8738317757009346, R: 0.8457943925233645, micro F1: 0.855140186915888, macro F1: 0.8595845184884193 [INFO] Relation Linking P: 0.7040498442367601, R: 0.705607476635514, micro F1: 0.7018691588785047, macro F1: 0.704827799865751 [INFO] Type Linking P: 0.45794392523364486, R: 0.45794392523364486, micro F1: 0.45794392523364486, macro F1: 0.45794392523364486 @@ -1505,8 +1505,8 @@ Question 107: Who is the partner of Rob Patterson? [INFO] Golden Answer: [http://dbpedia.org/resource/Carmen_Electra] [INFO] Predicted Answer: [http://dbpedia.org/resource/Carmen_Electra] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 108, P: 0.691, R: 0.722, macro F1: 0.692, macro F1*: 0.706 -[INFO] Current time: 665 ms, average: 2888.56 ms/ques, entity detection: 429.08 ms/ques (linking tool: 337.68 ms/ques), relation detection: 814.38 ms/ques (similarity: 604.43 ms/ques), EDG generation: 102.18 ms/ques +[INFO] Cumulative metrics, sample: 108, P: 0.691, R: 0.722, macro F1: 0.706 +[INFO] Current question consumed time: 609 ms, average: 2740.67 ms/ques, entity detection: 416.32 ms/ques (linking tool: 330.22 ms/ques), relation detection: 797.27 ms/ques (similarity: 602.44 ms/ques), EDG generation: 100.53 ms/ques [INFO] Entity Linking P: 0.875, R: 0.8472222222222222, micro F1: 0.8564814814814816, macro F1: 0.8608870967741935 [INFO] Relation Linking P: 0.70679012345679, R: 0.7083333333333334, micro F1: 0.7046296296296296, macro F1: 0.7075608869501999 [INFO] Type Linking P: 0.46296296296296297, R: 0.46296296296296297, micro F1: 0.46296296296296297, macro F1: 0.46296296296296297 @@ -1519,8 +1519,8 @@ Question 108: How many politicians are there from the city of Ganges? [INFO] Golden Answer: [9] [INFO] Predicted Answer: [148, 122] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 109, P: 0.685, R: 0.715, macro F1: 0.685, macro F1*: 0.700 -[INFO] Current time: 9929 ms, average: 2953.16 ms/ques, entity detection: 429.49 ms/ques (linking tool: 337.84 ms/ques), relation detection: 845.02 ms/ques (similarity: 639.94 ms/ques), EDG generation: 102.10 ms/ques +[INFO] Cumulative metrics, sample: 109, P: 0.685, R: 0.715, macro F1: 0.700 +[INFO] Current question consumed time: 9470 ms, average: 2802.40 ms/ques, entity detection: 415.24 ms/ques (linking tool: 328.96 ms/ques), relation detection: 827.94 ms/ques (similarity: 637.96 ms/ques), EDG generation: 100.41 ms/ques [INFO] Entity Linking P: 0.8761467889908257, R: 0.8486238532110092, micro F1: 0.8577981651376148, macro F1: 0.8621657232090572 [INFO] Relation Linking P: 0.709480122324159, R: 0.7064220183486238, micro F1: 0.7042813455657492, macro F1: 0.7079477678482969 [INFO] Type Linking P: 0.46788990825688076, R: 0.46788990825688076, micro F1: 0.46788990825688076, macro F1: 0.46788990825688076 @@ -1533,8 +1533,8 @@ Question 109: To which american football teams does the Heinz Field serve as hom [INFO] Golden Answer: [http://dbpedia.org/resource/Pittsburgh_Panthers, http://dbpedia.org/resource/Pittsburgh_Steelers] [INFO] Predicted Answer: [11] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 110, P: 0.679, R: 0.709, macro F1: 0.679, macro F1*: 0.693 -[INFO] Current time: 3862 ms, average: 2961.42 ms/ques, entity detection: 436.11 ms/ques (linking tool: 344.51 ms/ques), relation detection: 847.78 ms/ques (similarity: 637.87 ms/ques), EDG generation: 101.41 ms/ques +[INFO] Cumulative metrics, sample: 110, P: 0.679, R: 0.709, macro F1: 0.693 +[INFO] Current question consumed time: 3679 ms, average: 2810.37 ms/ques, entity detection: 421.68 ms/ques (linking tool: 335.38 ms/ques), relation detection: 830.22 ms/ques (similarity: 636.13 ms/ques), EDG generation: 99.72 ms/ques [INFO] Entity Linking P: 0.8681818181818182, R: 0.8409090909090909, micro F1: 0.8500000000000001, macro F1: 0.8543278529980659 [INFO] Relation Linking P: 0.703030303030303, R: 0.7, micro F1: 0.6978787878787879, macro F1: 0.701511879049676 [INFO] Type Linking P: 0.4727272727272727, R: 0.4727272727272727, micro F1: 0.4727272727272727, macro F1: 0.4727272727272727 @@ -1547,8 +1547,8 @@ Question 110: Which architect of the Marine Corps Air Station Kaneohe Bay is the [INFO] Golden Answer: [http://dbpedia.org/resource/United_States_Navy] [INFO] Predicted Answer: [http://dbpedia.org/resource/United_States_Navy] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 111, P: 0.682, R: 0.711, macro F1: 0.682, macro F1*: 0.696 -[INFO] Current time: 2166 ms, average: 2954.25 ms/ques, entity detection: 435.40 ms/ques (linking tool: 343.22 ms/ques), relation detection: 847.85 ms/ques (similarity: 637.14 ms/ques), EDG generation: 100.73 ms/ques +[INFO] Cumulative metrics, sample: 111, P: 0.682, R: 0.711, macro F1: 0.696 +[INFO] Current question consumed time: 2075 ms, average: 2803.75 ms/ques, entity detection: 421.05 ms/ques (linking tool: 334.07 ms/ques), relation detection: 830.62 ms/ques (similarity: 635.38 ms/ques), EDG generation: 99.05 ms/ques [INFO] Entity Linking P: 0.8693693693693694, R: 0.8423423423423423, micro F1: 0.8513513513513514, macro F1: 0.855642484589853 [INFO] Relation Linking P: 0.7057057057057057, R: 0.7027027027027027, micro F1: 0.7006006006006006, macro F1: 0.7042010027084653 [INFO] Type Linking P: 0.4774774774774775, R: 0.4774774774774775, micro F1: 0.4774774774774775, macro F1: 0.4774774774774775 @@ -1561,8 +1561,8 @@ Question 111: Which movies's screenplay is written by Akiva Goldsman [INFO] Golden Answer: [http://dbpedia.org/resource/The_Da_Vinci_Code_(film), http://dbpedia.org/resource/I,_Robot_(film), http://dbpedia.org/resource/Angels_&_Demons_(film), http://dbpedia.org/resource/Batman_Forever, http://dbpedia.org/resource/The_Client_(1994_film), http://dbpedia.org/resource/A_Beautiful_Mind_(film), http://dbpedia.org/resource/The_Divergent_Series:_Insurgent, http://dbpedia.org/resource/A_Time_to_Kill_(1996_film), http://dbpedia.org/resource/The_5th_Wave_(film)] [INFO] Predicted Answer: [http://dbpedia.org/resource/The_Da_Vinci_Code_(film), http://dbpedia.org/resource/I,_Robot_(film), http://dbpedia.org/resource/Angels_&_Demons_(film), http://dbpedia.org/resource/Batman_Forever, http://dbpedia.org/resource/The_Client_(1994_film), http://dbpedia.org/resource/A_Beautiful_Mind_(film), http://dbpedia.org/resource/The_Divergent_Series:_Insurgent, http://dbpedia.org/resource/A_Time_to_Kill_(1996_film), http://dbpedia.org/resource/The_5th_Wave_(film)] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 112, P: 0.685, R: 0.714, macro F1: 0.685, macro F1*: 0.699 -[INFO] Current time: 1122 ms, average: 2937.90 ms/ques, entity detection: 432.99 ms/ques (linking tool: 340.99 ms/ques), relation detection: 842.88 ms/ques (similarity: 634.12 ms/ques), EDG generation: 100.00 ms/ques +[INFO] Cumulative metrics, sample: 112, P: 0.685, R: 0.714, macro F1: 0.699 +[INFO] Current question consumed time: 1090 ms, average: 2788.45 ms/ques, entity detection: 418.80 ms/ques (linking tool: 331.90 ms/ques), relation detection: 825.59 ms/ques (similarity: 632.27 ms/ques), EDG generation: 98.31 ms/ques [INFO] Entity Linking P: 0.8705357142857143, R: 0.84375, micro F1: 0.8526785714285715, macro F1: 0.8569335937499999 [INFO] Relation Linking P: 0.7083333333333333, R: 0.7053571428571429, micro F1: 0.7032738095238095, macro F1: 0.7068421052631578 [INFO] Type Linking P: 0.4732142857142857, R: 0.4732142857142857, micro F1: 0.4732142857142857, macro F1: 0.4732142857142857 @@ -1575,8 +1575,8 @@ Question 112: How many TV shows are similar to the ones belonging to fantasy gen [INFO] Golden Answer: [20] [INFO] Predicted Answer: [] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 113, P: 0.678, R: 0.708, macro F1: 0.679, macro F1*: 0.693 -[INFO] Current time: 481 ms, average: 2916.17 ms/ques, entity detection: 430.13 ms/ques (linking tool: 338.57 ms/ques), relation detection: 837.78 ms/ques (similarity: 630.15 ms/ques), EDG generation: 99.36 ms/ques +[INFO] Cumulative metrics, sample: 113, P: 0.678, R: 0.708, macro F1: 0.693 +[INFO] Current question consumed time: 456 ms, average: 2767.81 ms/ques, entity detection: 415.82 ms/ques (linking tool: 329.34 ms/ques), relation detection: 820.59 ms/ques (similarity: 628.30 ms/ques), EDG generation: 97.69 ms/ques [INFO] Entity Linking P: 0.8628318584070797, R: 0.8362831858407079, micro F1: 0.845132743362832, macro F1: 0.8493501106194691 [INFO] Relation Linking P: 0.7020648967551621, R: 0.6991150442477876, micro F1: 0.6970501474926254, macro F1: 0.7005868653935724 [INFO] Type Linking P: 0.4690265486725664, R: 0.4690265486725664, micro F1: 0.4690265486725664, macro F1: 0.4690265486725664 @@ -1589,8 +1589,8 @@ Question 113: Give me all commanders which had an important role in both, the Ba [INFO] Golden Answer: [http://dbpedia.org/resource/Henry_Procter_(British_Army_officer), http://dbpedia.org/resource/Tecumseh] [INFO] Predicted Answer: [http://dbpedia.org/resource/Henry_Procter_(British_Army_officer), http://dbpedia.org/resource/Tecumseh] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 114, P: 0.681, R: 0.710, macro F1: 0.682, macro F1*: 0.695 -[INFO] Current time: 1871 ms, average: 2907.00 ms/ques, entity detection: 427.32 ms/ques (linking tool: 336.39 ms/ques), relation detection: 833.63 ms/ques (similarity: 628.19 ms/ques), EDG generation: 99.25 ms/ques +[INFO] Cumulative metrics, sample: 114, P: 0.681, R: 0.710, macro F1: 0.695 +[INFO] Current question consumed time: 1916 ms, average: 2760.33 ms/ques, entity detection: 413.11 ms/ques (linking tool: 327.23 ms/ques), relation detection: 816.47 ms/ques (similarity: 626.36 ms/ques), EDG generation: 98.30 ms/ques [INFO] Entity Linking P: 0.8640350877192983, R: 0.8377192982456141, micro F1: 0.8464912280701755, macro F1: 0.8506737203834329 [INFO] Relation Linking P: 0.7046783625730993, R: 0.7017543859649122, micro F1: 0.6997076023391813, macro F1: 0.7032133347922821 [INFO] Type Linking P: 0.47368421052631576, R: 0.47368421052631576, micro F1: 0.47368421052631576, macro F1: 0.47368421052631576 @@ -1603,8 +1603,8 @@ Question 114: How many developers were involved in creating games whose score is [INFO] Golden Answer: [11] [INFO] Predicted Answer: [7] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 115, P: 0.675, R: 0.704, macro F1: 0.676, macro F1*: 0.689 -[INFO] Current time: 1106 ms, average: 2891.34 ms/ques, entity detection: 425.04 ms/ques (linking tool: 334.33 ms/ques), relation detection: 828.14 ms/ques (similarity: 624.47 ms/ques), EDG generation: 98.73 ms/ques +[INFO] Cumulative metrics, sample: 115, P: 0.675, R: 0.704, macro F1: 0.689 +[INFO] Current question consumed time: 1069 ms, average: 2745.63 ms/ques, entity detection: 411.02 ms/ques (linking tool: 325.30 ms/ques), relation detection: 811.10 ms/ques (similarity: 622.67 ms/ques), EDG generation: 97.74 ms/ques [INFO] Entity Linking P: 0.8652173913043478, R: 0.8391304347826087, micro F1: 0.8478260869565218, macro F1: 0.8519742679680569 [INFO] Relation Linking P: 0.7072463768115942, R: 0.7, micro F1: 0.6994202898550725, macro F1: 0.7036045314109166 [INFO] Type Linking P: 0.4782608695652174, R: 0.4782608695652174, micro F1: 0.4782608695652174, macro F1: 0.4782608695652174 @@ -1617,8 +1617,8 @@ Question 115: Which sea is surrounded by Rostock and Kaliningrad? [INFO] Golden Answer: [http://dbpedia.org/resource/Baltic_Sea] [INFO] Predicted Answer: [http://dbpedia.org/resource/2012_Amputee_Football_World_Cup, http://dbpedia.org/resource/International_Slav_Institute, http://dbpedia.org/resource/Khrabrovo_Airport] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 116, P: 0.670, R: 0.698, macro F1: 0.670, macro F1*: 0.683 -[INFO] Current time: 6940 ms, average: 2926.24 ms/ques, entity detection: 428.28 ms/ques (linking tool: 335.48 ms/ques), relation detection: 842.52 ms/ques (similarity: 644.46 ms/ques), EDG generation: 98.11 ms/ques +[INFO] Cumulative metrics, sample: 116, P: 0.670, R: 0.698, macro F1: 0.683 +[INFO] Current question consumed time: 6641 ms, average: 2779.21 ms/ques, entity detection: 413.54 ms/ques (linking tool: 326.16 ms/ques), relation detection: 825.07 ms/ques (similarity: 642.66 ms/ques), EDG generation: 97.12 ms/ques [INFO] Entity Linking P: 0.8663793103448276, R: 0.8362068965517241, micro F1: 0.8462643678160922, macro F1: 0.8510257529463117 [INFO] Relation Linking P: 0.7011494252873562, R: 0.6939655172413793, micro F1: 0.6933908045977012, macro F1: 0.6975389751056501 [INFO] Type Linking P: 0.47413793103448276, R: 0.47413793103448276, micro F1: 0.47413793103448276, macro F1: 0.47413793103448276 @@ -1632,8 +1632,8 @@ Question 116: Which television show's developer is Brian Graden and voices to th [INFO] Predicted Answer: [http://dbpedia.org/resource/Debra_Hill, http://dbpedia.org/resource/Larry_J._Franco, http://dbpedia.org/resource/Kirk_Shaw, http://dbpedia.org/resource/Steven_Seagal, http://dbpedia.org/resource/Avi_Lerner, http://dbpedia.org/resource/Joe_Menosky, http://dbpedia.org/resource/Eric_Garcia, http://dbpedia.org/resource/Matt_Stone, http://dbpedia.org/resource/David_Perlmutter_(producer), http://dbpedia.org/resource/Robert_Hiltzik, http://dbpedia.org/resource/Mel_Stuart, http://dbpedia.org/resource/Albert_Band, http://dbpedia.org/resource/Peter_David, http://dbpedia.org/resource/Charles_Band, http://dbpedia.org/resource/David_T._Friendly, Steven Seagal@en, Avi Lerner@en, Lindsay MacAdam@en, Raymond Katz@en, Eric L. Gold@en, Albert Band@en, Debra Dion@en, Eric Garcia@en, Daniel J. Heffner@en, Fred Weintraub@en, Paul M. Heller@en, Carlos Vasallo@en, * Trey Parker *@en, Darren Higman@en, Robert Hiltzik@en, Michele Tatosian@en, Thomas E. van Dell@en, Mel Stuart@en, Larry Shaw@en, Vlad Paunescu@en, Oana Paunescu@en, Steve Greener@en, Charles Castaldi@en] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 117, P: 0.664, R: 0.692, macro F1: 0.664, macro F1*: 0.678 -[INFO] Current time: 2811 ms, average: 2925.26 ms/ques, entity detection: 426.55 ms/ques (linking tool: 333.88 ms/ques), relation detection: 843.79 ms/ques (similarity: 646.05 ms/ques), EDG generation: 98.38 ms/ques +[INFO] Cumulative metrics, sample: 117, P: 0.664, R: 0.692, macro F1: 0.678 +[INFO] Current question consumed time: 2653 ms, average: 2778.13 ms/ques, entity detection: 411.86 ms/ques (linking tool: 324.59 ms/ques), relation detection: 826.32 ms/ques (similarity: 644.20 ms/ques), EDG generation: 97.32 ms/ques [INFO] Entity Linking P: 0.8675213675213675, R: 0.8333333333333334, micro F1: 0.8447293447293449, macro F1: 0.8500837520938024 [INFO] Relation Linking P: 0.6951566951566951, R: 0.688034188034188, micro F1: 0.6874643874643875, macro F1: 0.6915771035235505 [INFO] Type Linking P: 0.4700854700854701, R: 0.4700854700854701, micro F1: 0.4700854700854701, macro F1: 0.4700854700854701 @@ -1646,8 +1646,8 @@ Question 117: How did the child of Stevens T. Mason die? [INFO] Golden Answer: [http://dbpedia.org/resource/Cholera] [INFO] Predicted Answer: [http://dbpedia.org/resource/Cholera] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 118, P: 0.667, R: 0.694, macro F1: 0.667, macro F1*: 0.680 -[INFO] Current time: 1381 ms, average: 2912.18 ms/ques, entity detection: 424.25 ms/ques (linking tool: 331.75 ms/ques), relation detection: 841.70 ms/ques (similarity: 643.71 ms/ques), EDG generation: 97.65 ms/ques +[INFO] Cumulative metrics, sample: 118, P: 0.667, R: 0.694, macro F1: 0.680 +[INFO] Current question consumed time: 1366 ms, average: 2766.16 ms/ques, entity detection: 409.77 ms/ques (linking tool: 322.58 ms/ques), relation detection: 824.47 ms/ques (similarity: 641.83 ms/ques), EDG generation: 96.60 ms/ques [INFO] Entity Linking P: 0.8601694915254238, R: 0.826271186440678, micro F1: 0.8375706214689268, macro F1: 0.8428796524997872 [INFO] Relation Linking P: 0.6977401129943502, R: 0.690677966101695, micro F1: 0.6901129943502825, macro F1: 0.694191078848906 [INFO] Type Linking P: 0.4745762711864407, R: 0.4745762711864407, micro F1: 0.4745762711864407, macro F1: 0.4745762711864407 @@ -1660,8 +1660,8 @@ Question 118: Does the Ontario International Airport serve the Inland Empire? [INFO] Golden Answer: [true] [INFO] Predicted Answer: [true] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 119, P: 0.669, R: 0.697, macro F1: 0.670, macro F1*: 0.683 -[INFO] Current time: 2479 ms, average: 2908.54 ms/ques, entity detection: 423.71 ms/ques (linking tool: 330.72 ms/ques), relation detection: 847.06 ms/ques (similarity: 645.96 ms/ques), EDG generation: 96.92 ms/ques +[INFO] Cumulative metrics, sample: 119, P: 0.669, R: 0.697, macro F1: 0.683 +[INFO] Current question consumed time: 2305 ms, average: 2762.29 ms/ques, entity detection: 409.22 ms/ques (linking tool: 321.60 ms/ques), relation detection: 829.37 ms/ques (similarity: 643.97 ms/ques), EDG generation: 95.87 ms/ques [INFO] Entity Linking P: 0.8613445378151261, R: 0.8277310924369747, micro F1: 0.8389355742296921, macro F1: 0.8442033529829842 [INFO] Relation Linking P: 0.6918767507002801, R: 0.6848739495798319, micro F1: 0.6843137254901961, macro F1: 0.6883575403711841 [INFO] Type Linking P: 0.4789915966386555, R: 0.4789915966386555, micro F1: 0.4789915966386555, macro F1: 0.4789915966386555 @@ -1674,8 +1674,8 @@ Question 119: Is semnani one of the languages spoken in Iran? [INFO] Golden Answer: [true] [INFO] Predicted Answer: [true] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 120, P: 0.672, R: 0.700, macro F1: 0.672, macro F1*: 0.686 -[INFO] Current time: 2397 ms, average: 2904.28 ms/ques, entity detection: 420.18 ms/ques (linking tool: 327.97 ms/ques), relation detection: 840.00 ms/ques (similarity: 641.67 ms/ques), EDG generation: 96.26 ms/ques +[INFO] Cumulative metrics, sample: 120, P: 0.672, R: 0.700, macro F1: 0.686 +[INFO] Current question consumed time: 2231 ms, average: 2757.86 ms/ques, entity detection: 405.81 ms/ques (linking tool: 318.92 ms/ques), relation detection: 822.46 ms/ques (similarity: 639.76 ms/ques), EDG generation: 95.21 ms/ques [INFO] Entity Linking P: 0.8583333333333333, R: 0.825, micro F1: 0.8361111111111114, macro F1: 0.8413366336633663 [INFO] Relation Linking P: 0.6861111111111111, R: 0.6791666666666667, micro F1: 0.6786111111111112, macro F1: 0.6826212275347576 [INFO] Type Linking P: 0.48333333333333334, R: 0.48333333333333334, micro F1: 0.48333333333333334, macro F1: 0.48333333333333334 @@ -1688,8 +1688,8 @@ Question 120: What is the nickname of the home stadium of Angels Toru? [INFO] Golden Answer: [http://dbpedia.org/resource/Nicolaus_Copernicus, http://dbpedia.org/resource/Toruń_gingerbread, http://dbpedia.org/resource/City_of_Angels] [INFO] Predicted Answer: [] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 121, P: 0.667, R: 0.694, macro F1: 0.667, macro F1*: 0.680 -[INFO] Current time: 354 ms, average: 2883.20 ms/ques, entity detection: 416.70 ms/ques (linking tool: 325.26 ms/ques), relation detection: 833.74 ms/ques (similarity: 636.84 ms/ques), EDG generation: 95.67 ms/ques +[INFO] Cumulative metrics, sample: 121, P: 0.667, R: 0.694, macro F1: 0.680 +[INFO] Current question consumed time: 349 ms, average: 2737.95 ms/ques, entity detection: 402.45 ms/ques (linking tool: 316.28 ms/ques), relation detection: 816.34 ms/ques (similarity: 634.93 ms/ques), EDG generation: 94.61 ms/ques [INFO] Entity Linking P: 0.8512396694214877, R: 0.8181818181818182, micro F1: 0.8292011019283748, macro F1: 0.8343834383438343 [INFO] Relation Linking P: 0.6804407713498623, R: 0.6735537190082644, micro F1: 0.6730027548209366, macro F1: 0.6769797297865364 [INFO] Type Linking P: 0.48760330578512395, R: 0.48760330578512395, micro F1: 0.48760330578512395, macro F1: 0.48760330578512395 @@ -1702,8 +1702,8 @@ Question 121: What is the television show whose executive producer is Douglas Ne [INFO] Golden Answer: [http://dbpedia.org/resource/Babylon_5, http://dbpedia.org/resource/Captain_Power_and_the_Soldiers_of_the_Future, http://dbpedia.org/resource/Hypernauts] [INFO] Predicted Answer: [http://dbpedia.org/resource/Babylon_5, http://dbpedia.org/resource/Captain_Power_and_the_Soldiers_of_the_Future, http://dbpedia.org/resource/Hypernauts] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 122, P: 0.669, R: 0.696, macro F1: 0.670, macro F1*: 0.683 -[INFO] Current time: 706 ms, average: 2865.35 ms/ques, entity detection: 414.81 ms/ques (linking tool: 323.32 ms/ques), relation detection: 828.11 ms/ques (similarity: 632.75 ms/ques), EDG generation: 95.03 ms/ques +[INFO] Cumulative metrics, sample: 122, P: 0.669, R: 0.696, macro F1: 0.683 +[INFO] Current question consumed time: 626 ms, average: 2720.64 ms/ques, entity detection: 400.42 ms/ques (linking tool: 314.34 ms/ques), relation detection: 810.75 ms/ques (similarity: 630.74 ms/ques), EDG generation: 93.98 ms/ques [INFO] Entity Linking P: 0.8524590163934426, R: 0.819672131147541, micro F1: 0.830601092896175, macro F1: 0.8357441337190613 [INFO] Relation Linking P: 0.6830601092896175, R: 0.6762295081967213, micro F1: 0.6756830601092897, macro F1: 0.6796276464288656 [INFO] Type Linking P: 0.48360655737704916, R: 0.48360655737704916, micro F1: 0.48360655737704916, macro F1: 0.48360655737704916 @@ -1716,8 +1716,8 @@ Question 122: List out the people who are related to the relatives of Jared Kush [INFO] Golden Answer: [http://dbpedia.org/resource/Maryanne_Trump_Barry, http://dbpedia.org/resource/Frederick_Trump, http://dbpedia.org/resource/Donald_Trump, http://dbpedia.org/resource/Ivanka_Trump, http://dbpedia.org/resource/Donald_Trump,_Jr., http://dbpedia.org/resource/Eric_Trump, http://dbpedia.org/resource/Jared_Kushner] [INFO] Predicted Answer: [http://dbpedia.org/resource/Ivanka_Trump, http://dbpedia.org/resource/Jared_Kushner, http://dbpedia.org/resource/Joshua_Kushner] [INFO] P: 0.6666666666666666, R: 0.2857142857142857, F: 0.4 -[INFO] Cumulative metrics, sample: 123, P: 0.669, R: 0.693, macro F1: 0.667, macro F1*: 0.681 -[INFO] Current time: 1799 ms, average: 2856.68 ms/ques, entity detection: 412.95 ms/ques (linking tool: 321.41 ms/ques), relation detection: 826.12 ms/ques (similarity: 633.58 ms/ques), EDG generation: 95.10 ms/ques +[INFO] Cumulative metrics, sample: 123, P: 0.669, R: 0.693, macro F1: 0.681 +[INFO] Current question consumed time: 1689 ms, average: 2712.25 ms/ques, entity detection: 398.47 ms/ques (linking tool: 312.41 ms/ques), relation detection: 808.62 ms/ques (similarity: 631.33 ms/ques), EDG generation: 94.12 ms/ques [INFO] Entity Linking P: 0.8536585365853658, R: 0.8211382113821138, micro F1: 0.831978319783198, macro F1: 0.8370826426710869 [INFO] Relation Linking P: 0.6815718157181572, R: 0.6788617886178862, micro F1: 0.675609756097561, macro F1: 0.680214102937842 [INFO] Type Linking P: 0.4878048780487805, R: 0.4878048780487805, micro F1: 0.4878048780487805, macro F1: 0.4878048780487805 @@ -1730,8 +1730,8 @@ Question 123: List all the doctoral student of the scientist who has advised joh [INFO] Golden Answer: [http://dbpedia.org/resource/Mary_Ainsworth, http://dbpedia.org/resource/Abraham_Maslow, http://dbpedia.org/resource/John_Bowlby] [INFO] Predicted Answer: [http://dbpedia.org/resource/Mary_Ainsworth, http://dbpedia.org/resource/Abraham_Maslow, http://dbpedia.org/resource/John_Bowlby] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 124, P: 0.672, R: 0.695, macro F1: 0.670, macro F1*: 0.684 -[INFO] Current time: 1312 ms, average: 2844.23 ms/ques, entity detection: 410.67 ms/ques (linking tool: 319.42 ms/ques), relation detection: 823.23 ms/ques (similarity: 631.88 ms/ques), EDG generation: 94.65 ms/ques +[INFO] Cumulative metrics, sample: 124, P: 0.672, R: 0.695, macro F1: 0.684 +[INFO] Current question consumed time: 1331 ms, average: 2701.11 ms/ques, entity detection: 396.33 ms/ques (linking tool: 310.49 ms/ques), relation detection: 806.08 ms/ques (similarity: 629.64 ms/ques), EDG generation: 93.67 ms/ques [INFO] Entity Linking P: 0.8548387096774194, R: 0.8225806451612904, micro F1: 0.8333333333333335, macro F1: 0.8383995037220844 [INFO] Relation Linking P: 0.6760752688172043, R: 0.6733870967741935, micro F1: 0.6701612903225808, macro F1: 0.6747285053335047 [INFO] Type Linking P: 0.4838709677419355, R: 0.4838709677419355, micro F1: 0.4838709677419355, macro F1: 0.4838709677419355 @@ -1744,8 +1744,8 @@ Question 124: To which series does the book belong which came after the Blade Ru [INFO] Golden Answer: [http://dbpedia.org/resource/Blade_Runner] [INFO] Predicted Answer: [http://dbpedia.org/resource/Rick_Deckard, Blade Runner #3@en, http://dbpedia.org/resource/Blade_Runner] [INFO] P: 0.3333333333333333, R: 1.0, F: 0.5 -[INFO] Cumulative metrics, sample: 125, P: 0.669, R: 0.698, macro F1: 0.669, macro F1*: 0.683 -[INFO] Current time: 2018 ms, average: 2837.62 ms/ques, entity detection: 408.77 ms/ques (linking tool: 317.65 ms/ques), relation detection: 822.06 ms/ques (similarity: 632.63 ms/ques), EDG generation: 94.09 ms/ques +[INFO] Cumulative metrics, sample: 125, P: 0.669, R: 0.698, macro F1: 0.683 +[INFO] Current question consumed time: 2070 ms, average: 2696.06 ms/ques, entity detection: 394.47 ms/ques (linking tool: 308.80 ms/ques), relation detection: 805.10 ms/ques (similarity: 630.34 ms/ques), EDG generation: 93.71 ms/ques [INFO] Entity Linking P: 0.856, R: 0.824, micro F1: 0.8346666666666669, macro F1: 0.839695238095238 [INFO] Relation Linking P: 0.6746666666666666, R: 0.672, micro F1: 0.6688000000000001, macro F1: 0.6733306930693069 [INFO] Type Linking P: 0.48, R: 0.48, micro F1: 0.48, macro F1: 0.48 @@ -1758,8 +1758,8 @@ Question 125: In which city did The Ultimate Fighter: Team Rousey vs. Team Tate [INFO] Golden Answer: [http://dbpedia.org/resource/Las_Vegas_Valley, http://dbpedia.org/resource/Nevada] [INFO] Predicted Answer: [http://dbpedia.org/resource/Las_Vegas_Valley] [INFO] P: 1.0, R: 0.5, F: 0.6666666666666666 -[INFO] Cumulative metrics, sample: 126, P: 0.672, R: 0.696, macro F1: 0.669, macro F1*: 0.684 -[INFO] Current time: 1642 ms, average: 2828.13 ms/ques, entity detection: 408.52 ms/ques (linking tool: 317.89 ms/ques), relation detection: 817.55 ms/ques (similarity: 630.11 ms/ques), EDG generation: 93.48 ms/ques +[INFO] Cumulative metrics, sample: 126, P: 0.672, R: 0.696, macro F1: 0.684 +[INFO] Current question consumed time: 1510 ms, average: 2686.65 ms/ques, entity detection: 394.06 ms/ques (linking tool: 308.81 ms/ques), relation detection: 800.79 ms/ques (similarity: 627.86 ms/ques), EDG generation: 93.10 ms/ques [INFO] Entity Linking P: 0.8571428571428571, R: 0.8253968253968254, micro F1: 0.8359788359788362, macro F1: 0.8409703504043127 [INFO] Relation Linking P: 0.6772486772486772, R: 0.6746031746031746, micro F1: 0.6714285714285715, macro F1: 0.6759233373714784 [INFO] Type Linking P: 0.47619047619047616, R: 0.47619047619047616, micro F1: 0.47619047619047616, macro F1: 0.47619047619047616 @@ -1772,8 +1772,8 @@ Question 126: Where was Sino-French War fought ? [INFO] Golden Answer: [http://dbpedia.org/resource/Annam_(French_protectorate), http://dbpedia.org/resource/Tonkin] [INFO] Predicted Answer: [http://dbpedia.org/resource/Joseph_Joffre, http://dbpedia.org/resource/Tonkin_Expeditionary_Corps, http://dbpedia.org/resource/French_Foreign_Legion, http://dbpedia.org/resource/Amédée_Courbet, http://dbpedia.org/resource/Sébastien_Lespès, http://dbpedia.org/resource/Joseph_Gaudérique_Aymerich, http://dbpedia.org/resource/Beiyang_Fleet, http://dbpedia.org/resource/Alfred-Amédée_Dodds, http://dbpedia.org/resource/Joseph_S._Skerrett, http://dbpedia.org/resource/Louis_Brière_de_l'Isle, http://dbpedia.org/resource/Gustave_Borgnis-Desbordes, http://dbpedia.org/resource/2nd_Foreign_Infantry_Regiment, http://dbpedia.org/resource/Jacques_Duchesne, http://dbpedia.org/resource/Liu_Mingchuan, http://dbpedia.org/resource/1st_Artillery_Regiment_(France), http://dbpedia.org/resource/Charles_Rondony] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 127, P: 0.667, R: 0.691, macro F1: 0.663, macro F1*: 0.679 -[INFO] Current time: 964 ms, average: 2813.45 ms/ques, entity detection: 406.82 ms/ques (linking tool: 316.48 ms/ques), relation detection: 812.72 ms/ques (similarity: 626.08 ms/ques), EDG generation: 92.82 ms/ques +[INFO] Cumulative metrics, sample: 127, P: 0.667, R: 0.691, macro F1: 0.679 +[INFO] Current question consumed time: 867 ms, average: 2672.32 ms/ques, entity detection: 392.25 ms/ques (linking tool: 307.27 ms/ques), relation detection: 795.92 ms/ques (similarity: 623.86 ms/ques), EDG generation: 92.44 ms/ques [INFO] Entity Linking P: 0.8582677165354331, R: 0.8267716535433071, micro F1: 0.8372703412073492, macro F1: 0.8422253293104717 [INFO] Relation Linking P: 0.6719160104986877, R: 0.6692913385826772, micro F1: 0.6661417322834646, macro F1: 0.6706011063685532 [INFO] Type Linking P: 0.48031496062992124, R: 0.48031496062992124, micro F1: 0.48031496062992124, macro F1: 0.48031496062992124 @@ -1786,8 +1786,8 @@ Question 127: Who did Daniel Gibson marry? [INFO] Golden Answer: [http://dbpedia.org/resource/Keyshia_Cole] [INFO] Predicted Answer: [http://dbpedia.org/resource/Keyshia_Cole] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 128, P: 0.669, R: 0.693, macro F1: 0.666, macro F1*: 0.681 -[INFO] Current time: 952 ms, average: 2798.91 ms/ques, entity detection: 404.87 ms/ques (linking tool: 314.68 ms/ques), relation detection: 808.95 ms/ques (similarity: 622.73 ms/ques), EDG generation: 92.15 ms/ques +[INFO] Cumulative metrics, sample: 128, P: 0.669, R: 0.693, macro F1: 0.681 +[INFO] Current question consumed time: 849 ms, average: 2658.08 ms/ques, entity detection: 390.34 ms/ques (linking tool: 305.52 ms/ques), relation detection: 792.09 ms/ques (similarity: 620.52 ms/ques), EDG generation: 91.77 ms/ques [INFO] Entity Linking P: 0.859375, R: 0.828125, micro F1: 0.8385416666666669, macro F1: 0.8434606481481481 [INFO] Relation Linking P: 0.6744791666666666, R: 0.671875, micro F1: 0.6687500000000001, macro F1: 0.6731745647969053 [INFO] Type Linking P: 0.484375, R: 0.484375, micro F1: 0.484375, macro F1: 0.484375 @@ -1800,8 +1800,8 @@ Question 128: What is the political party of the daughter of Jawaharlal Nehru ? [INFO] Golden Answer: [http://dbpedia.org/resource/Indian_National_Congress] [INFO] Predicted Answer: [http://dbpedia.org/resource/Indian_National_Congress] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 129, P: 0.672, R: 0.696, macro F1: 0.669, macro F1*: 0.684 -[INFO] Current time: 2298 ms, average: 2795.02 ms/ques, entity detection: 403.73 ms/ques (linking tool: 313.67 ms/ques), relation detection: 811.99 ms/ques (similarity: 623.57 ms/ques), EDG generation: 91.64 ms/ques +[INFO] Cumulative metrics, sample: 129, P: 0.672, R: 0.696, macro F1: 0.684 +[INFO] Current question consumed time: 2175 ms, average: 2654.33 ms/ques, entity detection: 388.97 ms/ques (linking tool: 304.37 ms/ques), relation detection: 794.78 ms/ques (similarity: 621.39 ms/ques), EDG generation: 91.23 ms/ques [INFO] Entity Linking P: 0.8604651162790697, R: 0.8294573643410853, micro F1: 0.839793281653747, macro F1: 0.8446767655216556 [INFO] Relation Linking P: 0.6770025839793281, R: 0.6744186046511628, micro F1: 0.6713178294573644, macro F1: 0.6757081239717194 [INFO] Type Linking P: 0.4883720930232558, R: 0.4883720930232558, micro F1: 0.4883720930232558, macro F1: 0.4883720930232558 @@ -1814,8 +1814,8 @@ Question 129: Name the river with source as Shannon Pot and its mouth is located [INFO] Golden Answer: [http://dbpedia.org/resource/River_Shannon] [INFO] Predicted Answer: [http://dbpedia.org/resource/River_Shannon] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 130, P: 0.674, R: 0.698, macro F1: 0.671, macro F1*: 0.686 -[INFO] Current time: 10278 ms, average: 2852.58 ms/ques, entity detection: 405.42 ms/ques (linking tool: 314.85 ms/ques), relation detection: 831.06 ms/ques (similarity: 662.68 ms/ques), EDG generation: 91.44 ms/ques +[INFO] Cumulative metrics, sample: 130, P: 0.674, R: 0.698, macro F1: 0.686 +[INFO] Current question consumed time: 10250 ms, average: 2712.76 ms/ques, entity detection: 390.54 ms/ques (linking tool: 305.45 ms/ques), relation detection: 813.80 ms/ques (similarity: 659.96 ms/ques), EDG generation: 91.01 ms/ques [INFO] Entity Linking P: 0.8615384615384616, R: 0.8307692307692308, micro F1: 0.8410256410256413, macro F1: 0.8458741258741259 [INFO] Relation Linking P: 0.6794871794871794, R: 0.676923076923077, micro F1: 0.6738461538461539, macro F1: 0.6782027046677329 [INFO] Type Linking P: 0.4846153846153846, R: 0.4846153846153846, micro F1: 0.4846153846153846, macro F1: 0.4846153846153846 @@ -1828,8 +1828,8 @@ Question 130: What is the organisation whose purposes are Environmentalism and P [INFO] Golden Answer: [http://dbpedia.org/resource/Greenpeace_Nordic, http://dbpedia.org/resource/Greenpeace] [INFO] Predicted Answer: [http://dbpedia.org/resource/Greenpeace_Nordic, http://dbpedia.org/resource/Greenpeace] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 131, P: 0.677, R: 0.700, macro F1: 0.674, macro F1*: 0.688 -[INFO] Current time: 3387 ms, average: 2856.66 ms/ques, entity detection: 405.50 ms/ques (linking tool: 314.36 ms/ques), relation detection: 833.42 ms/ques (similarity: 666.96 ms/ques), EDG generation: 90.93 ms/ques +[INFO] Cumulative metrics, sample: 131, P: 0.677, R: 0.700, macro F1: 0.688 +[INFO] Current question consumed time: 3436 ms, average: 2718.28 ms/ques, entity detection: 390.99 ms/ques (linking tool: 305.31 ms/ques), relation detection: 816.53 ms/ques (similarity: 664.03 ms/ques), EDG generation: 90.43 ms/ques [INFO] Entity Linking P: 0.8625954198473282, R: 0.8320610687022901, micro F1: 0.8422391857506363, macro F1: 0.8470531600302593 [INFO] Relation Linking P: 0.6819338422391857, R: 0.6793893129770993, micro F1: 0.6763358778625955, macro F1: 0.6806591995434116 [INFO] Type Linking P: 0.48091603053435117, R: 0.48091603053435117, micro F1: 0.48091603053435117, macro F1: 0.48091603053435117 @@ -1842,8 +1842,8 @@ Question 131: Count the number of sports team members which have player named M [INFO] Golden Answer: [157] [INFO] Predicted Answer: [1] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 132, P: 0.672, R: 0.695, macro F1: 0.669, macro F1*: 0.683 -[INFO] Current time: 2112 ms, average: 2851.02 ms/ques, entity detection: 405.28 ms/ques (linking tool: 314.22 ms/ques), relation detection: 832.74 ms/ques (similarity: 665.23 ms/ques), EDG generation: 90.43 ms/ques +[INFO] Cumulative metrics, sample: 132, P: 0.672, R: 0.695, macro F1: 0.683 +[INFO] Current question consumed time: 1939 ms, average: 2712.38 ms/ques, entity detection: 390.51 ms/ques (linking tool: 304.95 ms/ques), relation detection: 815.77 ms/ques (similarity: 662.39 ms/ques), EDG generation: 89.86 ms/ques [INFO] Entity Linking P: 0.8560606060606061, R: 0.8257575757575758, micro F1: 0.8358585858585861, macro F1: 0.8406360906360906 [INFO] Relation Linking P: 0.6843434343434343, R: 0.6818181818181818, micro F1: 0.6787878787878788, macro F1: 0.6830784742060158 [INFO] Type Linking P: 0.48484848484848486, R: 0.48484848484848486, micro F1: 0.48484848484848486, macro F1: 0.48484848484848486 @@ -1856,8 +1856,8 @@ Question 132: How many different people own the Timeform award winners? [INFO] Golden Answer: [27] [INFO] Predicted Answer: [251] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 133, P: 0.667, R: 0.690, macro F1: 0.664, macro F1*: 0.678 -[INFO] Current time: 523 ms, average: 2833.52 ms/ques, entity detection: 402.55 ms/ques (linking tool: 312.11 ms/ques), relation detection: 827.95 ms/ques (similarity: 661.74 ms/ques), EDG generation: 89.88 ms/ques +[INFO] Cumulative metrics, sample: 133, P: 0.667, R: 0.690, macro F1: 0.678 +[INFO] Current question consumed time: 512 ms, average: 2695.83 ms/ques, entity detection: 387.80 ms/ques (linking tool: 302.77 ms/ques), relation detection: 811.05 ms/ques (similarity: 659.15 ms/ques), EDG generation: 89.31 ms/ques [INFO] Entity Linking P: 0.8571428571428571, R: 0.8270676691729323, micro F1: 0.8370927318295741, macro F1: 0.8418367346938774 [INFO] Relation Linking P: 0.6791979949874687, R: 0.6766917293233082, micro F1: 0.6736842105263159, macro F1: 0.6779425458285269 [INFO] Type Linking P: 0.48120300751879697, R: 0.48120300751879697, micro F1: 0.48120300751879697, macro F1: 0.48120300751879697 @@ -1870,8 +1870,8 @@ Question 133: What religions do diplomats typically follow? [INFO] Golden Answer: [Greek Orthodox@en, http://dbpedia.org/resource/Maronite_Christianity_in_Lebanon, http://dbpedia.org/resource/Anglicanism, http://dbpedia.org/resource/Buddhism, http://dbpedia.org/resource/Jews, Roman Catholic@en, http://dbpedia.org/resource/Catholic_Church, http://dbpedia.org/resource/Episcopal_Church_(United_States), http://dbpedia.org/resource/Reform_Judaism] [INFO] Predicted Answer: [] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 134, P: 0.662, R: 0.685, macro F1: 0.659, macro F1*: 0.673 -[INFO] Current time: 447 ms, average: 2815.71 ms/ques, entity detection: 399.54 ms/ques (linking tool: 309.78 ms/ques), relation detection: 824.26 ms/ques (similarity: 658.52 ms/ques), EDG generation: 89.28 ms/ques +[INFO] Cumulative metrics, sample: 134, P: 0.662, R: 0.685, macro F1: 0.673 +[INFO] Current question consumed time: 441 ms, average: 2679.01 ms/ques, entity detection: 384.91 ms/ques (linking tool: 300.51 ms/ques), relation detection: 807.57 ms/ques (similarity: 655.75 ms/ques), EDG generation: 88.70 ms/ques [INFO] Entity Linking P: 0.8507462686567164, R: 0.8208955223880597, micro F1: 0.8308457711442788, macro F1: 0.8355543710021324 [INFO] Relation Linking P: 0.6741293532338308, R: 0.6716417910447762, micro F1: 0.6686567164179105, macro F1: 0.6728832730984634 [INFO] Type Linking P: 0.48507462686567165, R: 0.48507462686567165, micro F1: 0.48507462686567165, macro F1: 0.48507462686567165 @@ -1884,8 +1884,8 @@ Question 134: Guggenheim family is the cofounder of a company that invested stoc [INFO] Golden Answer: [http://dbpedia.org/resource/Los_Angeles_Dodgers] [INFO] Predicted Answer: [true] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 135, P: 0.657, R: 0.680, macro F1: 0.654, macro F1*: 0.668 -[INFO] Current time: 522 ms, average: 2798.72 ms/ques, entity detection: 396.59 ms/ques (linking tool: 307.48 ms/ques), relation detection: 818.16 ms/ques (similarity: 653.67 ms/ques), EDG generation: 88.93 ms/ques +[INFO] Cumulative metrics, sample: 135, P: 0.657, R: 0.680, macro F1: 0.668 +[INFO] Current question consumed time: 498 ms, average: 2662.85 ms/ques, entity detection: 382.06 ms/ques (linking tool: 298.29 ms/ques), relation detection: 801.59 ms/ques (similarity: 650.90 ms/ques), EDG generation: 88.27 ms/ques [INFO] Entity Linking P: 0.8444444444444444, R: 0.8148148148148148, micro F1: 0.8246913580246915, macro F1: 0.8293650793650793 [INFO] Relation Linking P: 0.6691358024691357, R: 0.6666666666666666, micro F1: 0.6637037037037038, macro F1: 0.6678989525569932 [INFO] Type Linking P: 0.48148148148148145, R: 0.48148148148148145, micro F1: 0.48148148148148145, macro F1: 0.48148148148148145 @@ -1898,8 +1898,8 @@ Question 135: Was John muir one of the first to ascend a mountain on Mt hoffmann [INFO] Golden Answer: [true] [INFO] Predicted Answer: [false] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 136, P: 0.652, R: 0.675, macro F1: 0.649, macro F1*: 0.663 -[INFO] Current time: 2185 ms, average: 2794.21 ms/ques, entity detection: 395.43 ms/ques (linking tool: 306.22 ms/ques), relation detection: 814.76 ms/ques (similarity: 650.32 ms/ques), EDG generation: 88.54 ms/ques +[INFO] Cumulative metrics, sample: 136, P: 0.652, R: 0.675, macro F1: 0.663 +[INFO] Current question consumed time: 2086 ms, average: 2658.61 ms/ques, entity detection: 380.96 ms/ques (linking tool: 296.99 ms/ques), relation detection: 798.21 ms/ques (similarity: 647.60 ms/ques), EDG generation: 87.89 ms/ques [INFO] Entity Linking P: 0.8382352941176471, R: 0.8088235294117647, micro F1: 0.8186274509803924, macro F1: 0.8232668067226891 [INFO] Relation Linking P: 0.6642156862745098, R: 0.6617647058823529, micro F1: 0.6588235294117648, macro F1: 0.6629879308470153 [INFO] Type Linking P: 0.4852941176470588, R: 0.4852941176470588, micro F1: 0.4852941176470588, macro F1: 0.4852941176470588 @@ -1912,8 +1912,8 @@ Question 136: Which city is known for the people working in the western penn hos [INFO] Golden Answer: [http://dbpedia.org/resource/Connecticut] [INFO] Predicted Answer: [http://dbpedia.org/resource/Royal_Court_Theatre] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 137, P: 0.647, R: 0.670, macro F1: 0.644, macro F1*: 0.658 -[INFO] Current time: 13345 ms, average: 2871.22 ms/ques, entity detection: 393.43 ms/ques (linking tool: 304.50 ms/ques), relation detection: 854.69 ms/ques (similarity: 685.07 ms/ques), EDG generation: 88.05 ms/ques +[INFO] Cumulative metrics, sample: 137, P: 0.647, R: 0.670, macro F1: 0.658 +[INFO] Current question consumed time: 13018 ms, average: 2734.23 ms/ques, entity detection: 379.09 ms/ques (linking tool: 295.39 ms/ques), relation detection: 838.04 ms/ques (similarity: 683.53 ms/ques), EDG generation: 87.39 ms/ques [INFO] Entity Linking P: 0.8321167883211679, R: 0.8029197080291971, micro F1: 0.8126520681265208, macro F1: 0.8172575599582899 [INFO] Relation Linking P: 0.6593673965936739, R: 0.656934306569343, micro F1: 0.6540145985401461, macro F1: 0.6581486028846284 [INFO] Type Linking P: 0.48175182481751827, R: 0.48175182481751827, micro F1: 0.48175182481751827, macro F1: 0.48175182481751827 @@ -1926,8 +1926,8 @@ Question 137: In which country does the publishing house exist which was cofound [INFO] Golden Answer: [http://dbpedia.org/resource/Cyprus] [INFO] Predicted Answer: [http://dbpedia.org/resource/2.5D, http://dbpedia.org/resource/Adal_Sultanate, http://dbpedia.org/resource/Ajuran_Sultanate, http://dbpedia.org/resource/Alania, http://dbpedia.org/resource/Alexander_Island, http://dbpedia.org/resource/American_Convention_on_Human_Rights, http://dbpedia.org/resource/Ancient_Carthage, http://dbpedia.org/resource/Anglo-Indian_cuisine, http://dbpedia.org/resource/Ante-Nicene_Fathers, http://dbpedia.org/resource/Arab_Monetary_Fund, http://dbpedia.org/resource/Archbishopric_of_Riga, http://dbpedia.org/resource/Areopagus_of_Eastern_Continental_Greece, http://dbpedia.org/resource/Astronomical_symbols, http://dbpedia.org/resource/Banu_Isam, http://dbpedia.org/resource/Bohemian, http://dbpedia.org/resource/Bolognese_Republic, http://dbpedia.org/resource/British_Antarctic_Survey, http://dbpedia.org/resource/British_Somaliland, http://dbpedia.org/resource/Cai_(state), http://dbpedia.org/resource/Canadian_cuisine, http://dbpedia.org/resource/Catholicism_in_Canada, http://dbpedia.org/resource/Chaoshan, http://dbpedia.org/resource/Chechen-Ingush_Autonomous_Soviet_Socialist_Republic, http://dbpedia.org/resource/Chechen_Republic_of_Ichkeria, http://dbpedia.org/resource/Chen_(state), http://dbpedia.org/resource/Chełmno_Land, http://dbpedia.org/resource/Chimor, http://dbpedia.org/resource/Chinese_poetry, http://dbpedia.org/resource/Cilicia_(Roman_province), http://dbpedia.org/resource/Cimmeria_(Conan), http://dbpedia.org/resource/Cincinnati_Bearcats_football, http://dbpedia.org/resource/Cincinnati_metropolitan_area, http://dbpedia.org/resource/Constitution_of_the_Year_VIII, http://dbpedia.org/resource/Contemporary_philosophy, http://dbpedia.org/resource/Counties_of_England, http://dbpedia.org/resource/County_of_Burgundy, http://dbpedia.org/resource/County_of_Holland, http://dbpedia.org/resource/County_of_Mark, http://dbpedia.org/resource/County_of_Zeeland, http://dbpedia.org/resource/Creta_et_Cyrenaica, http://dbpedia.org/resource/Darién_Gap, http://dbpedia.org/resource/Deconstructivism, http://dbpedia.org/resource/Democratic_Republic_of_Georgia, http://dbpedia.org/resource/Disallowance_and_reservation, http://dbpedia.org/resource/District_Six, http://dbpedia.org/resource/Division_of_Northern_Territory, http://dbpedia.org/resource/Diyu, http://dbpedia.org/resource/Don_Republic, http://dbpedia.org/resource/Downtown_Toronto, http://dbpedia.org/resource/Drimnagh, http://dbpedia.org/resource/Duchy_of_Courland_and_Semigallia_(1918), http://dbpedia.org/resource/Duchy_of_Greater_Poland, http://dbpedia.org/resource/Duchy_of_Legnica, http://dbpedia.org/resource/Duchy_of_Limburg, http://dbpedia.org/resource/Duchy_of_Masovia, http://dbpedia.org/resource/Duchy_of_Pomerania, http://dbpedia.org/resource/Dutch_Language_Union, http://dbpedia.org/resource/Dʿmt, http://dbpedia.org/resource/Eastern_United_States, http://dbpedia.org/resource/Economic_integration, http://dbpedia.org/resource/Edwardian_architecture, http://dbpedia.org/resource/Emirate_of_Jabal_Shammar, http://dbpedia.org/resource/European_Economic_Area, http://dbpedia.org/resource/European_cuisine, http://dbpedia.org/resource/Exarchate_of_Africa, http://dbpedia.org/resource/Fair_catch_kick, http://dbpedia.org/resource/Fier, http://dbpedia.org/resource/First_Red_Scare, http://dbpedia.org/resource/Fort_William,_Ontario, http://dbpedia.org/resource/Free_Imperial_City_of_Besançon, http://dbpedia.org/resource/Free_and_Independent_State_of_Cundinamarca, http://dbpedia.org/resource/Furstenberg_boundary, http://dbpedia.org/resource/General_Government, http://dbpedia.org/resource/German_East_Africa, http://dbpedia.org/resource/German_South-West_Africa, http://dbpedia.org/resource/Glenageary, http://dbpedia.org/resource/Glywysing, http://dbpedia.org/resource/Gog_Magog_Downs, http://dbpedia.org/resource/Goryeo, http://dbpedia.org/resource/Goss_Moor, http://dbpedia.org/resource/Grand_Duchy_of_Finland, http://dbpedia.org/resource/Greek_East_and_Latin_West, http://dbpedia.org/resource/Guelders, http://dbpedia.org/resource/Hariphunchai, http://dbpedia.org/resource/Homeland, http://dbpedia.org/resource/Hong_Kong_cuisine, http://dbpedia.org/resource/Humanitas, http://dbpedia.org/resource/Hyborian_Age, http://dbpedia.org/resource/Industrialisation, http://dbpedia.org/resource/Ingushetia, http://dbpedia.org/resource/International_Energy_Agency, http://dbpedia.org/resource/Iranian_cuisine, http://dbpedia.org/resource/Islamic_Courts_Union, http://dbpedia.org/resource/Islamic_schools_and_branches, http://dbpedia.org/resource/Italian_Somaliland, http://dbpedia.org/resource/Jazz_Age, http://dbpedia.org/resource/Jeffersonian_architecture, http://dbpedia.org/resource/Jin_(Chinese_state), http://dbpedia.org/resource/Kakongo, http://dbpedia.org/resource/Kansas_Territory, http://dbpedia.org/resource/Kiautschou_Bay_concession, http://dbpedia.org/resource/Kilnamanagh, http://dbpedia.org/resource/King_of_Albania, http://dbpedia.org/resource/Kingdom_of_Altava, http://dbpedia.org/resource/Kingdom_of_Cusco, http://dbpedia.org/resource/Kingdom_of_Etruria, http://dbpedia.org/resource/Kingdom_of_Finland_(1918), http://dbpedia.org/resource/Kingdom_of_Lithuania_(1918), http://dbpedia.org/resource/Kingdom_of_Meath, http://dbpedia.org/resource/Kongo_dia_Nlaza, http://dbpedia.org/resource/Kovno_Governorate, http://dbpedia.org/resource/Kuroda_normal_form, http://dbpedia.org/resource/Landport, http://dbpedia.org/resource/Leopardstown, http://dbpedia.org/resource/Lesson, http://dbpedia.org/resource/List_of_living_legitimate_male_Capetians, http://dbpedia.org/resource/Llŷn_Peninsula, http://dbpedia.org/resource/Los_Altos,_Central_America, http://dbpedia.org/resource/Lu_(state), http://dbpedia.org/resource/Malay_world, http://dbpedia.org/resource/Member_states_of_the_Council_of_Europe, http://dbpedia.org/resource/Mental_process, http://dbpedia.org/resource/Metapolitefsi, http://dbpedia.org/resource/Mid-Atlantic_states, http://dbpedia.org/resource/Middle_Pleistocene, http://dbpedia.org/resource/Minyue, http://dbpedia.org/resource/Mongolian_cuisine, http://dbpedia.org/resource/Mormon_Corridor, http://dbpedia.org/resource/Mormon_Reformation, http://dbpedia.org/resource/Mountainous_Republic_of_the_Northern_Caucasus, http://dbpedia.org/resource/Nasrid_dynasty, http://dbpedia.org/resource/National_Football_League_preseason, http://dbpedia.org/resource/Nativity_Fast, http://dbpedia.org/resource/New_York_City_Fire_Department, http://dbpedia.org/resource/Nuclear_Energy_Agency, http://dbpedia.org/resource/Occupation_of_the_Ruhr, http://dbpedia.org/resource/Ohio_Country, http://dbpedia.org/resource/Old_Cairo, http://dbpedia.org/resource/PR_interval, http://dbpedia.org/resource/Panthays, http://dbpedia.org/resource/Peirce's_criterion, http://dbpedia.org/resource/People's_Republic_of_the_Congo, http://dbpedia.org/resource/Piast_dynasty, http://dbpedia.org/resource/Pipex, http://dbpedia.org/resource/Polish_Cathedral_style, http://dbpedia.org/resource/Politics_of_Belgium, http://dbpedia.org/resource/Politics_of_Finland, http://dbpedia.org/resource/Postmodern_architecture, http://dbpedia.org/resource/Prince-Bishopric_of_Warmia, http://dbpedia.org/resource/Principality_of_Taranto, http://dbpedia.org/resource/Province_of_Jülich-Cleves-Berg, http://dbpedia.org/resource/Province_of_Pomerania_(1815–1945), http://dbpedia.org/resource/Province_of_Silesia, http://dbpedia.org/resource/Province_of_Tierra_Firme, http://dbpedia.org/resource/Pukapuka, http://dbpedia.org/resource/Qi_(state), http://dbpedia.org/resource/Radnorshire, http://dbpedia.org/resource/Real_Audiencia_of_Guatemala, http://dbpedia.org/resource/Reichskommissariat_Ukraine, http://dbpedia.org/resource/Renaissance_Papacy, http://dbpedia.org/resource/Renaissance_of_the_12th_century, http://dbpedia.org/resource/Republics_of_Russia, http://dbpedia.org/resource/Rockfleet_Castle, http://dbpedia.org/resource/Roman_Republic_(18th_century), http://dbpedia.org/resource/Roman_Republic_(19th_century), http://dbpedia.org/resource/Romanian_Revolution, http://dbpedia.org/resource/Rutland, http://dbpedia.org/resource/Saar_(League_of_Nations), http://dbpedia.org/resource/Samborides, http://dbpedia.org/resource/Santry, http://dbpedia.org/resource/Saybrook_Colony, http://dbpedia.org/resource/Second_Hellenic_Republic, http://dbpedia.org/resource/Soconusco, http://dbpedia.org/resource/Song_(state), http://dbpedia.org/resource/State_of_the_Teutonic_Order, http://dbpedia.org/resource/Suburbicarian_diocese, http://dbpedia.org/resource/Sukhothai_Kingdom, http://dbpedia.org/resource/Sultanate_of_Darfur, http://dbpedia.org/resource/Television_in_the_United_States, http://dbpedia.org/resource/Tian, http://dbpedia.org/resource/Tijuca_Forest, http://dbpedia.org/resource/Tlaxcala_(Nahua_state), http://dbpedia.org/resource/Togoland, http://dbpedia.org/resource/Transitional_Federal_Government, http://dbpedia.org/resource/Trinidad_and_Tobago_cuisine, http://dbpedia.org/resource/Tsim_Sha_Tsui, http://dbpedia.org/resource/Tudor_period, http://dbpedia.org/resource/Tyrconnell, http://dbpedia.org/resource/Ukrainian_State, http://dbpedia.org/resource/United_States_Bobsled_and_Skeleton_Federation, http://dbpedia.org/resource/Upper_Mesopotamia, http://dbpedia.org/resource/Uí_Fiachrach, http://dbpedia.org/resource/Uí_Maine, http://dbpedia.org/resource/Uí_Néill, http://dbpedia.org/resource/Vedic_period, http://dbpedia.org/resource/Vermilion_Bird, http://dbpedia.org/resource/West_Friesland_(region), http://dbpedia.org/resource/Wey_(state), http://dbpedia.org/resource/1830_Constitution_of_Ecuador, http://dbpedia.org/resource/Aceh_Sultanate, http://dbpedia.org/resource/Afterfeast, http://dbpedia.org/resource/Airy_disk, http://dbpedia.org/resource/Alawite_State, http://dbpedia.org/resource/Alodia, http://dbpedia.org/resource/Ancient_Hawaii, http://dbpedia.org/resource/Anglo-Saxon_architecture, http://dbpedia.org/resource/Anglo-Scottish_border, http://dbpedia.org/resource/Arab_Federation, http://dbpedia.org/resource/Baltic_Assembly, http://dbpedia.org/resource/Bamana_Empire, http://dbpedia.org/resource/Barotseland, http://dbpedia.org/resource/British_literature, http://dbpedia.org/resource/Brockworth,_Gloucestershire, http://dbpedia.org/resource/Bunyoro, http://dbpedia.org/resource/Busoga, http://dbpedia.org/resource/Butch_and_femme, http://dbpedia.org/resource/California_Republic, http://dbpedia.org/resource/Catholic_Church_in_England_and_Wales, http://dbpedia.org/resource/Cherokee_Nation, http://dbpedia.org/resource/Chūzan, http://dbpedia.org/resource/Circassia, http://dbpedia.org/resource/Cisalpine_Republic, http://dbpedia.org/resource/Cispadane_Republic, http://dbpedia.org/resource/Civic-military_dictatorship_of_Uruguay, http://dbpedia.org/resource/Colony_of_Aden, http://dbpedia.org/resource/Common_Market_for_Eastern_and_Southern_Africa, http://dbpedia.org/resource/Confederation_of_the_Rhine, http://dbpedia.org/resource/Cook_Islands_Federation, http://dbpedia.org/resource/Crimean_Autonomous_Soviet_Socialist_Republic, http://dbpedia.org/resource/Decapolis, http://dbpedia.org/resource/December_2001_riots_in_Argentina, http://dbpedia.org/resource/Democratic_Republic_of_Yemen, http://dbpedia.org/resource/Demographics_of_Cameroon, http://dbpedia.org/resource/Demographics_of_Liberia, http://dbpedia.org/resource/Demographics_of_Mozambique, http://dbpedia.org/resource/Dendi_Kingdom, http://dbpedia.org/resource/Dervish_state, http://dbpedia.org/resource/Des_Moines_metropolitan_area, http://dbpedia.org/resource/Devolution_(biology), http://dbpedia.org/resource/Direct_market, http://dbpedia.org/resource/Divine_call, http://dbpedia.org/resource/Dominion_of_Ceylon, http://dbpedia.org/resource/Dominion_of_Fiji, http://dbpedia.org/resource/Donbass, http://dbpedia.org/resource/Duchy_of_Courland_and_Semigallia, http://dbpedia.org/resource/Duchy_of_Lucca, http://dbpedia.org/resource/Duchy_of_Massa_and_Carrara, http://dbpedia.org/resource/Duchy_of_Parma, http://dbpedia.org/resource/Dutch_Slave_Coast, http://dbpedia.org/resource/Early_modern_Europe, http://dbpedia.org/resource/East_Jersey, http://dbpedia.org/resource/East_New_York,_Brooklyn, http://dbpedia.org/resource/Ecuadorians, http://dbpedia.org/resource/Egyptian_Armed_Forces, http://dbpedia.org/resource/Egyptian_nationality_law, http://dbpedia.org/resource/Electorate_of_Hesse, http://dbpedia.org/resource/Emergency_Government_of_the_Republic_of_Indonesia, http://dbpedia.org/resource/Emirate_of_Diriyah, http://dbpedia.org/resource/Emirate_of_Nejd, http://dbpedia.org/resource/Emirate_of_Nejd_and_Hasa, http://dbpedia.org/resource/Emirate_of_Sicily, http://dbpedia.org/resource/Empire_of_Great_Fulo, http://dbpedia.org/resource/English_Gothic_architecture, http://dbpedia.org/resource/English_national_identity, http://dbpedia.org/resource/Eocene_Thermal_Maximum_2, http://dbpedia.org/resource/Epistemological_rupture, http://dbpedia.org/resource/Era_of_Silence, http://dbpedia.org/resource/Far_Eastern_Republic, http://dbpedia.org/resource/Federal_Government_of_Somalia, http://dbpedia.org/resource/Federated_Malay_States, http://dbpedia.org/resource/Federation_of_Arab_Emirates_of_the_South, http://dbpedia.org/resource/Federation_of_South_Arabia, http://dbpedia.org/resource/Fin, http://dbpedia.org/resource/Finnish_Democratic_Republic, http://dbpedia.org/resource/Finnish_Socialist_Workers'_Republic, http://dbpedia.org/resource/First_Hellenic_Republic, http://dbpedia.org/resource/First_Hungarian_Republic, http://dbpedia.org/resource/Fourth_Chinese_domination_of_Vietnam, http://dbpedia.org/resource/Franceville,_New_Hebrides, http://dbpedia.org/resource/Free_City_of_Kraków, http://dbpedia.org/resource/Free_State_of_Fiume, http://dbpedia.org/resource/Futurist_architecture, http://dbpedia.org/resource/Galician_Soviet_Socialist_Republic, http://dbpedia.org/resource/Galápagos_Time, http://dbpedia.org/resource/Gaza_Empire, http://dbpedia.org/resource/General_position, http://dbpedia.org/resource/German_Samoa, http://dbpedia.org/resource/Government_in_exile, http://dbpedia.org/resource/Governorate, http://dbpedia.org/resource/Governorate_of_Estonia, http://dbpedia.org/resource/Gozo_(independent_state), http://dbpedia.org/resource/Grand_Duchy_of_Mecklenburg-Schwerin, http://dbpedia.org/resource/Grand_Duchy_of_Mecklenburg-Strelitz, http://dbpedia.org/resource/Grand_Principality_of_Serbia, http://dbpedia.org/resource/Great_Britain_Davis_Cup_team, http://dbpedia.org/resource/Gupta_script, http://dbpedia.org/resource/Habesh_Eyalet, http://dbpedia.org/resource/Hakata-ku,_Fukuoka, http://dbpedia.org/resource/Himara, http://dbpedia.org/resource/Himyarite_Kingdom, http://dbpedia.org/resource/History_of_Germany_during_World_War_I, http://dbpedia.org/resource/History_of_Portugal_(1415–1578), http://dbpedia.org/resource/Hohenzollern-Hechingen, http://dbpedia.org/resource/Hohenzollern-Sigmaringen, http://dbpedia.org/resource/Holstein, http://dbpedia.org/resource/Idrisid_Emirate_of_Asir, http://dbpedia.org/resource/Imamate_of_Futa_Jallon, http://dbpedia.org/resource/Imamate_of_Futa_Toro, http://dbpedia.org/resource/Impact_winter, http://dbpedia.org/resource/Iowa_Territory, http://dbpedia.org/resource/Islands_of_Refreshment, http://dbpedia.org/resource/Italian_Empire, http://dbpedia.org/resource/Italian_Islands_of_the_Aegean, http://dbpedia.org/resource/Italian_Regency_of_Carnaro, http://dbpedia.org/resource/Italian_Republic_(Napoleonic), http://dbpedia.org/resource/Jabal_al-Druze, http://dbpedia.org/resource/Jolof_Empire, http://dbpedia.org/resource/Kaabu, http://dbpedia.org/resource/Kaffa_Province, http://dbpedia.org/resource/Kazakh_Khanate, http://dbpedia.org/resource/Kazembe, http://dbpedia.org/resource/Kingdom_of_Aksum, http://dbpedia.org/resource/Kingdom_of_Bora_Bora, http://dbpedia.org/resource/Kingdom_of_Brittany, http://dbpedia.org/resource/Kingdom_of_Burundi, http://dbpedia.org/resource/Kingdom_of_Butua, http://dbpedia.org/resource/Kingdom_of_Corsica_(1736), http://dbpedia.org/resource/Kingdom_of_Hejaz, http://dbpedia.org/resource/Kingdom_of_Kurdistan, http://dbpedia.org/resource/Kingdom_of_Loango, http://dbpedia.org/resource/Kingdom_of_Makuria, http://dbpedia.org/resource/Kingdom_of_Mutapa, http://dbpedia.org/resource/Kingdom_of_Northumbria, http://dbpedia.org/resource/Kingdom_of_Pajang, http://dbpedia.org/resource/Kingdom_of_Rwanda, http://dbpedia.org/resource/Kingdom_of_Sine, http://dbpedia.org/resource/Kingdom_of_Tahiti, http://dbpedia.org/resource/Kingdom_of_Uí_Failghe, http://dbpedia.org/resource/Kingdom_of_Whydah, http://dbpedia.org/resource/Kokura, http://dbpedia.org/resource/Konbaung_Dynasty, http://dbpedia.org/resource/Kuybyshev,_Russia, http://dbpedia.org/resource/Landgraviate_of_Hesse-Darmstadt, http://dbpedia.org/resource/Lanfang_Republic, http://dbpedia.org/resource/List_of_Bronze_Age_states, http://dbpedia.org/resource/List_of_Iron_Age_states, http://dbpedia.org/resource/List_of_sovereign_states_and_dependent_territories_in_Europe, http://dbpedia.org/resource/Lê_dynasty, http://dbpedia.org/resource/MENA, http://dbpedia.org/resource/Magadha_Kingdom, http://dbpedia.org/resource/Mahan_confederacy, http://dbpedia.org/resource/Mahdist_Sudan, http://dbpedia.org/resource/Majeerteen_Sultanate, http://dbpedia.org/resource/Mali_Federation, http://dbpedia.org/resource/Manipur_(princely_state), http://dbpedia.org/resource/Marinoan_glaciation, http://dbpedia.org/resource/Marstrand_Free_Port, http://dbpedia.org/resource/Mataram_Sultanate, http://dbpedia.org/resource/Merina_Kingdom, http://dbpedia.org/resource/Michigan_Territory, http://dbpedia.org/resource/Mongolian_People's_Republic, http://dbpedia.org/resource/Mthwakazi, http://dbpedia.org/resource/Mudaito_Dynasty, http://dbpedia.org/resource/Muhammad_Ali_dynasty, http://dbpedia.org/resource/Mutawakkilite_Kingdom_of_Yemen, http://dbpedia.org/resource/National_symbols_of_Ecuador, http://dbpedia.org/resource/Netherlands_New_Guinea, http://dbpedia.org/resource/Nigerians, http://dbpedia.org/resource/Nogai_Horde, http://dbpedia.org/resource/Norman_architecture, http://dbpedia.org/resource/North_Sea_Empire, http://dbpedia.org/resource/Nottingham_Urban_Area, http://dbpedia.org/resource/Orange_River_Sovereignty, http://dbpedia.org/resource/Ottomanism, http://dbpedia.org/resource/Pahlavi_dynasty, http://dbpedia.org/resource/Parish_church, http://dbpedia.org/resource/People's_Democratic_Republic_of_Ethiopia, http://dbpedia.org/resource/Peripatric_speciation, http://dbpedia.org/resource/Principality_of_Lippe, http://dbpedia.org/resource/Principality_of_Serbia_(medieval), http://dbpedia.org/resource/Principality_of_Trinidad, http://dbpedia.org/resource/Principality_of_Zeta, http://dbpedia.org/resource/Probation, http://dbpedia.org/resource/Provisional_Government_of_Hawaii, http://dbpedia.org/resource/Provisional_Revolutionary_Government_of_the_Republic_of_South_Vietnam, http://dbpedia.org/resource/Republic_of_Acre, http://dbpedia.org/resource/Republic_of_Central_Lithuania, http://dbpedia.org/resource/Republic_of_Cospaia, http://dbpedia.org/resource/Republic_of_Ezo, http://dbpedia.org/resource/Republic_of_Formosa, http://dbpedia.org/resource/Republic_of_Indian_Stream, http://dbpedia.org/resource/Republic_of_Kuwait, http://dbpedia.org/resource/Republic_of_Maryland, http://dbpedia.org/resource/Republic_of_Mountainous_Armenia, http://dbpedia.org/resource/Republic_of_North_Peru, http://dbpedia.org/resource/Republic_of_Prekmurje, http://dbpedia.org/resource/Republic_of_Rose_Island, http://dbpedia.org/resource/Republic_of_Sonora, http://dbpedia.org/resource/Republic_of_Yucatán, http://dbpedia.org/resource/Republic_of_the_Rio_Grande, http://dbpedia.org/resource/Reuss_(state), http://dbpedia.org/resource/Revocation, http://dbpedia.org/resource/Rif_Republic, http://dbpedia.org/resource/Rule_of_St._Augustine, http://dbpedia.org/resource/Ryukyu_Kingdom, http://dbpedia.org/resource/Samoa_Time_Zone, http://dbpedia.org/resource/San_Lorenzo,_Ecuador, http://dbpedia.org/resource/Saudeleur_Dynasty, http://dbpedia.org/resource/Saxe-Coburg-Saalfeld, http://dbpedia.org/resource/Saxe-Gotha, http://dbpedia.org/resource/Saxe-Hildburghausen, http://dbpedia.org/resource/Schwarzburg-Rudolstadt, http://dbpedia.org/resource/Schwarzburg-Sondershausen, http://dbpedia.org/resource/Sennar_(sultanate), http://dbpedia.org/resource/Serbian_Despotate, http://dbpedia.org/resource/Shandong_Peninsula, http://dbpedia.org/resource/Shotgun_house, http://dbpedia.org/resource/Single_Euro_Payments_Area, http://dbpedia.org/resource/Slovak_Soviet_Republic, http://dbpedia.org/resource/Slovak_euro_coins, http://dbpedia.org/resource/South-West_Africa, http://dbpedia.org/resource/South_Arabia, http://dbpedia.org/resource/Southeast_Michigan, http://dbpedia.org/resource/Soviet_Republic_of_Naissaar, http://dbpedia.org/resource/State_of_Aleppo, http://dbpedia.org/resource/State_of_Damascus, http://dbpedia.org/resource/State_of_Muskogee, http://dbpedia.org/resource/Sudd, http://dbpedia.org/resource/Sultanate_of_Cirebon, http://dbpedia.org/resource/Sultanate_of_Hobyo, http://dbpedia.org/resource/Sultanate_of_Maguindanao, http://dbpedia.org/resource/Sultanate_of_Nejd, http://dbpedia.org/resource/Supreme_Being, http://dbpedia.org/resource/Surinam_(Dutch_colony), http://dbpedia.org/resource/Tamilakam, http://dbpedia.org/resource/Tanganyika_(territory), http://dbpedia.org/resource/Taungoo_Dynasty, http://dbpedia.org/resource/Terai, http://dbpedia.org/resource/Terra_nullius, http://dbpedia.org/resource/Thames_Gateway, http://dbpedia.org/resource/Tibet_(1912–51), http://dbpedia.org/resource/Toro_Kingdom, http://dbpedia.org/resource/Toucouleur_Empire, http://dbpedia.org/resource/Transitional_National_Government, http://dbpedia.org/resource/Transkei, http://dbpedia.org/resource/Transpadane_Republic, http://dbpedia.org/resource/Travancore, http://dbpedia.org/resource/Trust_Territory_of_the_Pacific_Islands, http://dbpedia.org/resource/Tudor_architecture, http://dbpedia.org/resource/Tuʻi_Tonga_Empire, http://dbpedia.org/resource/UTC+05:00, http://dbpedia.org/resource/UTC+06:00, http://dbpedia.org/resource/UTC−01:00, http://dbpedia.org/resource/Unfederated_Malay_States, http://dbpedia.org/resource/Union_Defence_Force_(UAE), http://dbpedia.org/resource/Union_between_Sweden_and_Norway, http://dbpedia.org/resource/United_Arab_Republic, http://dbpedia.org/resource/United_Belgian_States, http://dbpedia.org/resource/United_Principalities, http://dbpedia.org/resource/United_States_Ambassador_to_Pakistan, http://dbpedia.org/resource/United_Suvadive_Republic, http://dbpedia.org/resource/Villafranchian, http://dbpedia.org/resource/Volga_Bulgaria, http://dbpedia.org/resource/Waldeck_(state), http://dbpedia.org/resource/Walt_Disney_Studios_(Burbank), http://dbpedia.org/resource/Warsangali_Sultanate, http://dbpedia.org/resource/Wassoulou_Empire, http://dbpedia.org/resource/Wattasid_dynasty, http://dbpedia.org/resource/West_Jersey, http://dbpedia.org/resource/Wikt:Anglo-French, http://dbpedia.org/resource/Württemberg, http://dbpedia.org/resource/Yang_di-Pertuan_Agong, http://dbpedia.org/resource/Years_of_Lead_(Italy), http://dbpedia.org/resource/Yeke_Kingdom, http://dbpedia.org/resource/Yemen_Arab_Republic, http://dbpedia.org/resource/Zayyanid_dynasty, http://dbpedia.org/resource/Zimbabwe_Rhodesia, http://dbpedia.org/resource/Zulu_Kingdom, http://dbpedia.org/resource/Éire, http://dbpedia.org/resource/1938_British_Empire_Games, http://dbpedia.org/resource/Adjectival_noun_(Japanese), http://dbpedia.org/resource/Adverbial_genitive, http://dbpedia.org/resource/Age_of_the_Sturlungs, http://dbpedia.org/resource/Amali_dynasty, http://dbpedia.org/resource/Australian_nationality_law, http://dbpedia.org/resource/Beirut_Vilayet, http://dbpedia.org/resource/Bissagos_Islands, http://dbpedia.org/resource/Calymmian, http://dbpedia.org/resource/Caminha, http://dbpedia.org/resource/Cape_Finisterre, http://dbpedia.org/resource/Cappadocia_(Roman_province), http://dbpedia.org/resource/County_of_Geneva, http://dbpedia.org/resource/Cryogenian, http://dbpedia.org/resource/Demographics_of_Syria, http://dbpedia.org/resource/Despotate_of_the_Morea, http://dbpedia.org/resource/Duchy_of_Brunswick-Lüneburg, http://dbpedia.org/resource/Duchy_of_Magdeburg, http://dbpedia.org/resource/Eastern_Settlement, http://dbpedia.org/resource/Ectasian, http://dbpedia.org/resource/Eemian, http://dbpedia.org/resource/Eidgenossenschaft, http://dbpedia.org/resource/Elmet, http://dbpedia.org/resource/FIBA_Europe, http://dbpedia.org/resource/Fort_Monmouth, http://dbpedia.org/resource/Fouta_Djallon, http://dbpedia.org/resource/French_Guinea, http://dbpedia.org/resource/Fugger, http://dbpedia.org/resource/Gram_(Denmark), http://dbpedia.org/resource/Grand_unification_epoch, http://dbpedia.org/resource/Great_Britain_at_the_Olympics, http://dbpedia.org/resource/Greece_national_basketball_team, http://dbpedia.org/resource/Greek_economic_miracle, http://dbpedia.org/resource/Guatemalan_Revolution, http://dbpedia.org/resource/Hanover_Square,_Westminster, http://dbpedia.org/resource/High-tech_architecture, http://dbpedia.org/resource/History_of_Iceland, http://dbpedia.org/resource/History_of_the_Roman_Empire, http://dbpedia.org/resource/Kafana, http://dbpedia.org/resource/Kingdom_of_Dagbon, http://dbpedia.org/resource/Kingdom_of_Poland_(1916–18), http://dbpedia.org/resource/Kingdom_of_the_Suebi, http://dbpedia.org/resource/LGBT_community, http://dbpedia.org/resource/Lancashire_County_Cricket_Club, http://dbpedia.org/resource/Leave_the_gate_as_you_found_it, http://dbpedia.org/resource/Lee_shore, http://dbpedia.org/resource/List_of_Bailiffs_of_Guernsey, http://dbpedia.org/resource/Little_Ice_Age, http://dbpedia.org/resource/Lordship_of_Groningen, http://dbpedia.org/resource/Maniots, http://dbpedia.org/resource/Metropolitan_Borough_of_St_Pancras, http://dbpedia.org/resource/Monera, http://dbpedia.org/resource/Montefeltro, http://dbpedia.org/resource/Nieuwe_Zakelijkheid, http://dbpedia.org/resource/North_German_Constitution, http://dbpedia.org/resource/North_Waziristan, http://dbpedia.org/resource/Northern_Liang, http://dbpedia.org/resource/Nueva_Galicia, http://dbpedia.org/resource/Ober_Ost, http://dbpedia.org/resource/Ochlocracy, http://dbpedia.org/resource/Oium, http://dbpedia.org/resource/Older_Dryas, http://dbpedia.org/resource/Oldest_Dryas, http://dbpedia.org/resource/Orosirian, http://dbpedia.org/resource/Ottoman_Greeks, http://dbpedia.org/resource/Palladian_architecture, http://dbpedia.org/resource/Peter_Badmayev, http://dbpedia.org/resource/Phoenice_(Roman_province), http://dbpedia.org/resource/Pie_rule, http://dbpedia.org/resource/Pokrovskoye,_Tyumen_Oblast, http://dbpedia.org/resource/Politics_of_Guatemala, http://dbpedia.org/resource/Pregnancy_(mammals), http://dbpedia.org/resource/Province_of_Hanover, http://dbpedia.org/resource/Province_of_Lower_Silesia, http://dbpedia.org/resource/Province_of_Posen, http://dbpedia.org/resource/Province_of_Schleswig-Holstein, http://dbpedia.org/resource/Rangitata_Orogeny, http://dbpedia.org/resource/Regency_architecture, http://dbpedia.org/resource/Republic_of_Cuba_(1902–59), http://dbpedia.org/resource/Rexurdimento, http://dbpedia.org/resource/Rhyacian, http://dbpedia.org/resource/Sakastan, http://dbpedia.org/resource/Santo_Tomás_de_Castilla, http://dbpedia.org/resource/Saxe-Weissenfels, http://dbpedia.org/resource/Sellafield, http://dbpedia.org/resource/Semigroup_action, http://dbpedia.org/resource/Shu_(state), http://dbpedia.org/resource/Siderian, http://dbpedia.org/resource/Singhasari, http://dbpedia.org/resource/Somali_Democratic_Republic, http://dbpedia.org/resource/Southern_Han, http://dbpedia.org/resource/Statherian, http://dbpedia.org/resource/Statistical_regularity, http://dbpedia.org/resource/Stenian, http://dbpedia.org/resource/Swiss_cuisine, http://dbpedia.org/resource/Symbel, http://dbpedia.org/resource/Tirtha_and_Kshetra, http://dbpedia.org/resource/Tobolsk_Governorate, http://dbpedia.org/resource/Tonian, http://dbpedia.org/resource/Twenty-sixth_Dynasty_of_Egypt, http://dbpedia.org/resource/United_Provinces_of_Central_Italy, http://dbpedia.org/resource/United_States_territory, http://dbpedia.org/resource/Universal_Transverse_Mercator_coordinate_system, http://dbpedia.org/resource/Upper_Galilee, http://dbpedia.org/resource/Vormärz, http://dbpedia.org/resource/West_African_cuisine, http://dbpedia.org/resource/Western_Settlement, http://dbpedia.org/resource/World_of_A_Song_of_Ice_and_Fire, http://dbpedia.org/resource/Xiguan, http://dbpedia.org/resource/Îles_des_Saintes, http://dbpedia.org/resource/Afghan_Transitional_Administration, http://dbpedia.org/resource/Akhand_Bharat, http://dbpedia.org/resource/Al-Faw_Peninsula, http://dbpedia.org/resource/Amadora, http://dbpedia.org/resource/American_(word), http://dbpedia.org/resource/Arkansas_Territory, http://dbpedia.org/resource/Baixa_de_Cassanje, http://dbpedia.org/resource/Bangladeshis, http://dbpedia.org/resource/Barrio, http://dbpedia.org/resource/Bergslagen, http://dbpedia.org/resource/Boukólos_rule, http://dbpedia.org/resource/British_Cyprus, http://dbpedia.org/resource/British_invasion_of_Manila, http://dbpedia.org/resource/Brulé, http://dbpedia.org/resource/Chinese_Taipei, http://dbpedia.org/resource/Colony_of_Jamaica, http://dbpedia.org/resource/Colony_of_Santiago, http://dbpedia.org/resource/Constitution_of_Jamaica, http://dbpedia.org/resource/Dakshinayana, http://dbpedia.org/resource/Demarcation_point, http://dbpedia.org/resource/Dutch_Bengal, http://dbpedia.org/resource/Dutch_Ceylon, http://dbpedia.org/resource/Dutch_Malacca, http://dbpedia.org/resource/Dál_nAraidi, http://dbpedia.org/resource/EUCLID_(university), http://dbpedia.org/resource/Earldom_of_Ulster, http://dbpedia.org/resource/Ednam, http://dbpedia.org/resource/Essequibo_(colony), http://dbpedia.org/resource/External_relations_of_the_Isle_of_Man, http://dbpedia.org/resource/Finland_men's_national_ice_hockey_team, http://dbpedia.org/resource/Flight_information_region, http://dbpedia.org/resource/Foreign_relations_of_Nicaragua, http://dbpedia.org/resource/Foreign_relations_of_Poland, http://dbpedia.org/resource/French_Madagascar, http://dbpedia.org/resource/Fulfillment_house, http://dbpedia.org/resource/Hawaii–Aleutian_Time_Zone, http://dbpedia.org/resource/Help_the_Aged, http://dbpedia.org/resource/Higher_education_in_the_United_States, http://dbpedia.org/resource/Hispania_Ulterior, http://dbpedia.org/resource/History_of_Sindh, http://dbpedia.org/resource/History_of_Sudan_(1956–69), http://dbpedia.org/resource/Hockey_Canada, http://dbpedia.org/resource/Houthis, http://dbpedia.org/resource/Imperial_Japanese_Navy_General_Staff, http://dbpedia.org/resource/Industrial_Age, http://dbpedia.org/resource/Insular_Government_of_the_Philippine_Islands, http://dbpedia.org/resource/Israeli_Civil_Administration, http://dbpedia.org/resource/Israeli_Military_Governorate, http://dbpedia.org/resource/Japanese_Paleolithic, http://dbpedia.org/resource/Japanese_occupation_of_Burma, http://dbpedia.org/resource/Jewish_Quarter_(Jerusalem), http://dbpedia.org/resource/Kamakura_period, http://dbpedia.org/resource/Kazakh_Autonomous_Socialist_Soviet_Republic, http://dbpedia.org/resource/Kenmu_Restoration, http://dbpedia.org/resource/Kirghiz_Autonomous_Socialist_Soviet_Republic_(1926–36), http://dbpedia.org/resource/Koryū, http://dbpedia.org/resource/Kunashir_Island, http://dbpedia.org/resource/Launch_window, http://dbpedia.org/resource/Liaodong_Peninsula, http://dbpedia.org/resource/Lihyan, http://dbpedia.org/resource/List_of_Jamaicans, http://dbpedia.org/resource/List_of_Serb_countries_and_regions, http://dbpedia.org/resource/List_of_islands_of_the_Bailiwick_of_Jersey, http://dbpedia.org/resource/Local_government_in_the_United_States, http://dbpedia.org/resource/Lorne,_Scotland, http://dbpedia.org/resource/Madurai_Sultanate, http://dbpedia.org/resource/Marinid_dynasty, http://dbpedia.org/resource/Moldavian_Soviet_Socialist_Republic, http://dbpedia.org/resource/Montana_Territory, http://dbpedia.org/resource/Nebraska_Territory, http://dbpedia.org/resource/Neo-Inca_State, http://dbpedia.org/resource/New_Zealanders, http://dbpedia.org/resource/North-East_Frontier_Agency, http://dbpedia.org/resource/Novorossiya, http://dbpedia.org/resource/Okinawa_Islands, http://dbpedia.org/resource/Paramara_dynasty, http://dbpedia.org/resource/Plus_and_minus_signs, http://dbpedia.org/resource/Prince-Bishopric_of_Strasbourg, http://dbpedia.org/resource/Protectorate_of_South_Arabia, http://dbpedia.org/resource/Qataban, http://dbpedia.org/resource/Quadrennium, http://dbpedia.org/resource/Reichskommissariat_Norwegen, http://dbpedia.org/resource/Rhu, http://dbpedia.org/resource/River_crab_(Internet_slang), http://dbpedia.org/resource/Russia_men's_national_ice_hockey_team, http://dbpedia.org/resource/Samudera_Pasai_Sultanate, http://dbpedia.org/resource/Sanjak_of_Alexandretta, http://dbpedia.org/resource/Second_East_Turkestan_Republic, http://dbpedia.org/resource/Serafina_Corrêa, http://dbpedia.org/resource/Setúbal, http://dbpedia.org/resource/Sheikhdom_of_Kuwait, http://dbpedia.org/resource/Shiga_Prefecture, http://dbpedia.org/resource/Shiva_(Judaism), http://dbpedia.org/resource/Solanki_dynasty, http://dbpedia.org/resource/State_of_Vietnam, http://dbpedia.org/resource/Student_orientation, http://dbpedia.org/resource/Subterranea_(comics), http://dbpedia.org/resource/Sweden_men's_national_ice_hockey_team, http://dbpedia.org/resource/Taiheiyō_Belt, http://dbpedia.org/resource/Tajik_Autonomous_Soviet_Socialist_Republic, http://dbpedia.org/resource/Tiflis_Governorate, http://dbpedia.org/resource/Tobata-ku,_Kitakyūshū, http://dbpedia.org/resource/Turkestan_Autonomous_Soviet_Socialist_Republic, http://dbpedia.org/resource/Turkmen_Soviet_Socialist_Republic, http://dbpedia.org/resource/United_Nations_Temporary_Executive_Authority, http://dbpedia.org/resource/United_Provinces_of_British_India, http://dbpedia.org/resource/United_States_and_weapons_of_mass_destruction, http://dbpedia.org/resource/Uzbek_Soviet_Socialist_Republic, http://dbpedia.org/resource/Weihaiwei_under_British_rule, http://dbpedia.org/resource/Whyalla, http://dbpedia.org/resource/Yamatai, http://dbpedia.org/resource/Zamorin_of_Calicut, http://dbpedia.org/resource/ASEAN_Summit, http://dbpedia.org/resource/Aishvarya, http://dbpedia.org/resource/Aki_District,_Hiroshima, http://dbpedia.org/resource/Beylik_of_Dulkadir, http://dbpedia.org/resource/Beyliks_of_Canik, http://dbpedia.org/resource/Burma_National_Army, http://dbpedia.org/resource/Florida_Territory, http://dbpedia.org/resource/Fort_Peck_Indian_Reservation, http://dbpedia.org/resource/France_national_rugby_league_team, http://dbpedia.org/resource/Friedman_rule, http://dbpedia.org/resource/Fur_trade_in_Montana, http://dbpedia.org/resource/Grand_Palace, http://dbpedia.org/resource/Guobiao_Majiang, http://dbpedia.org/resource/Hadley_Centre_for_Climate_Prediction_and_Research, http://dbpedia.org/resource/Hanthawaddy_Kingdom, http://dbpedia.org/resource/History_of_Myanmar, http://dbpedia.org/resource/Isfendiyarids, http://dbpedia.org/resource/Junagadh_State, http://dbpedia.org/resource/Kathiawar_Agency, http://dbpedia.org/resource/Kingdom_of_Commagene, http://dbpedia.org/resource/Kingdom_of_Mrauk_U, http://dbpedia.org/resource/Kodori_Valley, http://dbpedia.org/resource/Kuratowski_convergence, http://dbpedia.org/resource/Landscape_mythology, http://dbpedia.org/resource/Later_Shu, http://dbpedia.org/resource/Later_Tang, http://dbpedia.org/resource/Lushan_District, http://dbpedia.org/resource/Meagher_County,_Montana, http://dbpedia.org/resource/Memphis_metropolitan_area, http://dbpedia.org/resource/Minnesota_Chippewa_Tribe, http://dbpedia.org/resource/Minnesota_Territory, http://dbpedia.org/resource/Myanmar_Standard_Time, http://dbpedia.org/resource/Nanzhao, http://dbpedia.org/resource/Neomercantilism, http://dbpedia.org/resource/Newington_Butts, http://dbpedia.org/resource/Nusantara, http://dbpedia.org/resource/Ottoman_Tunisia, http://dbpedia.org/resource/Palmela, http://dbpedia.org/resource/Peloponnesian_Senate, http://dbpedia.org/resource/People's_History_Museum, http://dbpedia.org/resource/Postpartum_period, http://dbpedia.org/resource/Powder_River_Country, http://dbpedia.org/resource/Prome_Kingdom, http://dbpedia.org/resource/Pyu_city-states, http://dbpedia.org/resource/Reorganized_National_Government_of_the_Republic_of_China, http://dbpedia.org/resource/Republic_of_Madawaska, http://dbpedia.org/resource/Restored_Hanthawaddy_Kingdom, http://dbpedia.org/resource/Second_Republic_of_South_Korea, http://dbpedia.org/resource/Shun_dynasty, http://dbpedia.org/resource/State_of_Burma, http://dbpedia.org/resource/Tagalog_Republic, http://dbpedia.org/resource/The_Flathead, http://dbpedia.org/resource/United_States_Ambassador_to_Costa_Rica, http://dbpedia.org/resource/United_States_of_America_Mathematical_Olympiad, http://dbpedia.org/resource/Verethragna, http://dbpedia.org/resource/White_Earth_Band_of_Ojibwe, http://dbpedia.org/resource/Abkhaz_Autonomous_Soviet_Socialist_Republic, http://dbpedia.org/resource/Alexander_Palace, http://dbpedia.org/resource/Artois, http://dbpedia.org/resource/Ba_(town), http://dbpedia.org/resource/Beverwijck, http://dbpedia.org/resource/Black_Hawk_State_Historic_Site, http://dbpedia.org/resource/British_West_Africa, http://dbpedia.org/resource/Byrd_Rule, http://dbpedia.org/resource/Carlton,_Victoria, http://dbpedia.org/resource/Chahar_Province, http://dbpedia.org/resource/Chelyabinsk_Oblast, http://dbpedia.org/resource/Constitution_of_the_Republic_of_Texas, http://dbpedia.org/resource/Cool_Britannia, http://dbpedia.org/resource/Cylon_(Battlestar_Galactica), http://dbpedia.org/resource/Demographics_of_Eritrea, http://dbpedia.org/resource/Design_life, http://dbpedia.org/resource/Disentis_Abbey, http://dbpedia.org/resource/Dnipropetrovsk_Oblast, http://dbpedia.org/resource/Dolgans, http://dbpedia.org/resource/Donetsk_People's_Republic, http://dbpedia.org/resource/Enets_people, http://dbpedia.org/resource/Federal_administration_of_Switzerland, http://dbpedia.org/resource/Foreign_relations_of_Laos, http://dbpedia.org/resource/French_nobility, http://dbpedia.org/resource/Government_of_the_Autonomous_Republic_of_Abkhazia, http://dbpedia.org/resource/Harbor_City,_Los_Angeles, http://dbpedia.org/resource/Hertsa_Raion, http://dbpedia.org/resource/Homestead_exemption, http://dbpedia.org/resource/Huai_Army, http://dbpedia.org/resource/Kamchatka_Oblast, http://dbpedia.org/resource/Kherson_Oblast, http://dbpedia.org/resource/Kirov_Oblast, http://dbpedia.org/resource/Law_of_Russia, http://dbpedia.org/resource/List_of_emperors_of_the_Qing_dynasty, http://dbpedia.org/resource/Majha, http://dbpedia.org/resource/Media_of_Russia, http://dbpedia.org/resource/Morton_Arboretum, http://dbpedia.org/resource/Mykolaiv_Oblast, http://dbpedia.org/resource/Najd_Sanjak, http://dbpedia.org/resource/Neomodern, http://dbpedia.org/resource/Nganasan_people, http://dbpedia.org/resource/Nineveh_plains, http://dbpedia.org/resource/North_Ossetia-Alania, http://dbpedia.org/resource/Northern_Region,_Nigeria, http://dbpedia.org/resource/Oranienbaum,_Russia, http://dbpedia.org/resource/Orenburg_Oblast, http://dbpedia.org/resource/Particle_size, http://dbpedia.org/resource/Pavlovsk_Palace, http://dbpedia.org/resource/Piedmontese_Republic, http://dbpedia.org/resource/Politics_of_Italy, http://dbpedia.org/resource/Politics_of_Qatar, http://dbpedia.org/resource/Principality_of_Jersika, http://dbpedia.org/resource/Principality_of_Tver, http://dbpedia.org/resource/Prisoner_in_the_Vatican, http://dbpedia.org/resource/Province_of_Vercelli, http://dbpedia.org/resource/Psychrometrics, http://dbpedia.org/resource/Relationship_between_string_theory_and_quantum_field_theory, http://dbpedia.org/resource/Republic_of_Alba, http://dbpedia.org/resource/Romania_men's_national_handball_team, http://dbpedia.org/resource/Romania_national_rugby_union_team, http://dbpedia.org/resource/Romania_women's_national_handball_team, http://dbpedia.org/resource/Rwandan_Revolution, http://dbpedia.org/resource/Saratov_Oblast, http://dbpedia.org/resource/Selwyn_Mountains, http://dbpedia.org/resource/Strelna, http://dbpedia.org/resource/Strong_cryptography, http://dbpedia.org/resource/Subalpine_Republic, http://dbpedia.org/resource/Taiwan_under_Qing_rule, http://dbpedia.org/resource/Television_in_Australia, http://dbpedia.org/resource/Television_in_Mexico, http://dbpedia.org/resource/Television_in_New_Zealand, http://dbpedia.org/resource/Texians, http://dbpedia.org/resource/Three_Leagues, http://dbpedia.org/resource/Tibet_under_Qing_rule, http://dbpedia.org/resource/Time_in_Russia, http://dbpedia.org/resource/Ulyanovsk_Oblast, http://dbpedia.org/resource/United_Nations_Special_Commission, http://dbpedia.org/resource/Usfurids, http://dbpedia.org/resource/Volgograd_Oblast, http://dbpedia.org/resource/World_Summit_of_Nobel_Peace_Laureates, http://dbpedia.org/resource/Xinjiang_under_Qing_rule, http://dbpedia.org/resource/Zaporizhia_Oblast, http://dbpedia.org/resource/Ōmiya,_Saitama, http://dbpedia.org/resource/Afghan_Interim_Administration, http://dbpedia.org/resource/Aisleless_church, http://dbpedia.org/resource/An_der_Etsch, http://dbpedia.org/resource/Bishopric_of_Worms, http://dbpedia.org/resource/Borduria, http://dbpedia.org/resource/Bremen-Verden, http://dbpedia.org/resource/Canary_girls, http://dbpedia.org/resource/Cape_Verde_Time, http://dbpedia.org/resource/Denaby, http://dbpedia.org/resource/Essen_Abbey, http://dbpedia.org/resource/Etchū_Province, http://dbpedia.org/resource/European_Table_Tennis_Union, http://dbpedia.org/resource/Freedom_of_religion_in_Saudi_Arabia, http://dbpedia.org/resource/Great_Law_of_Peace, http://dbpedia.org/resource/Grodno_Governorate, http://dbpedia.org/resource/Gørlev, http://dbpedia.org/resource/Hever_Castle, http://dbpedia.org/resource/Human_trafficking_in_the_United_States, http://dbpedia.org/resource/Indiana_Democratic_Party, http://dbpedia.org/resource/Irish_revolutionary_period, http://dbpedia.org/resource/Kaga_Province, http://dbpedia.org/resource/Kai_Province, http://dbpedia.org/resource/Kavajë, http://dbpedia.org/resource/Khemed, http://dbpedia.org/resource/Kishiwada,_Osaka, http://dbpedia.org/resource/Landgraviate_of_Hesse, http://dbpedia.org/resource/Lists_of_time_zones, http://dbpedia.org/resource/Malaysian_cuisine, http://dbpedia.org/resource/Newfoundland_Time_Zone, http://dbpedia.org/resource/Roman_Umbria, http://dbpedia.org/resource/Rule_of_twelfths, http://dbpedia.org/resource/Saxe-Eisenach, http://dbpedia.org/resource/Saxe-Jena, http://dbpedia.org/resource/Shinano_Province, http://dbpedia.org/resource/Shëngjergj, http://dbpedia.org/resource/Slack_water, http://dbpedia.org/resource/Syldavia, http://dbpedia.org/resource/Székely_Land, http://dbpedia.org/resource/Tidal_diamond, http://dbpedia.org/resource/Time_in_Argentina, http://dbpedia.org/resource/Time_in_Georgia_(country), http://dbpedia.org/resource/Time_in_India, http://dbpedia.org/resource/Time_in_Mexico, http://dbpedia.org/resource/Time_in_New_Zealand, http://dbpedia.org/resource/Time_in_Venezuela, http://dbpedia.org/resource/Trams_in_Melbourne, http://dbpedia.org/resource/Trecena, http://dbpedia.org/resource/Tyrian_shekel, http://dbpedia.org/resource/Tōtōmi_Province, http://dbpedia.org/resource/UTC+03:30, http://dbpedia.org/resource/UTC+04:30, http://dbpedia.org/resource/UTC+06:30, http://dbpedia.org/resource/UTC−02:00, http://dbpedia.org/resource/UTC−08:00, http://dbpedia.org/resource/Ursberg_Abbey, http://dbpedia.org/resource/Wadiyar_dynasty, http://dbpedia.org/resource/Yasnaya_Polyana, http://dbpedia.org/resource/Yeouido, http://dbpedia.org/resource/1931_Yugoslav_Constitution, http://dbpedia.org/resource/Ag_Qoyunlu, http://dbpedia.org/resource/Arminiya, http://dbpedia.org/resource/Bagratid_Armenia, http://dbpedia.org/resource/Barotziland-North-Western_Rhodesia, http://dbpedia.org/resource/Benzingerode, http://dbpedia.org/resource/Bishopric_of_Trent, http://dbpedia.org/resource/Business_rule, http://dbpedia.org/resource/Cinema_of_South_Africa, http://dbpedia.org/resource/Constitution_of_Zimbabwe, http://dbpedia.org/resource/County_of_Wernigerode, http://dbpedia.org/resource/Daylight_saving_time_in_Canada, http://dbpedia.org/resource/Dir_(princely_state), http://dbpedia.org/resource/Duchy_of_Guastalla, http://dbpedia.org/resource/Duchy_of_Modena_and_Reggio, http://dbpedia.org/resource/Ephemeris_day, http://dbpedia.org/resource/Erivan_Governorate, http://dbpedia.org/resource/Flag_of_Bhutan, http://dbpedia.org/resource/Gladstone,_Manitoba, http://dbpedia.org/resource/Glossary_of_the_Weimar_Republic, http://dbpedia.org/resource/Gévaudan, http://dbpedia.org/resource/Hashid, http://dbpedia.org/resource/History_of_the_Ming_dynasty, http://dbpedia.org/resource/Hyolmo_people, http://dbpedia.org/resource/Hyperinflation_in_Zimbabwe, http://dbpedia.org/resource/Interim_Peace, http://dbpedia.org/resource/Kaikoura_Ranges, http://dbpedia.org/resource/Koch_dynasty, http://dbpedia.org/resource/Komarno_(Hasidic_dynasty), http://dbpedia.org/resource/Lacs_Region, http://dbpedia.org/resource/Later_Stone_Age, http://dbpedia.org/resource/Libyan_Arab_Republic, http://dbpedia.org/resource/List_of_Dragonlance_deities, http://dbpedia.org/resource/Lordship_of_Wilmington, http://dbpedia.org/resource/Madh'hij, http://dbpedia.org/resource/Mahdids, http://dbpedia.org/resource/Marquisate_of_Finale, http://dbpedia.org/resource/Mid-Western_Region,_Nigeria, http://dbpedia.org/resource/Mid-century_modern, http://dbpedia.org/resource/Najahid_dynasty, http://dbpedia.org/resource/New_Zealand_Cricket, http://dbpedia.org/resource/North-Eastern_Rhodesia, http://dbpedia.org/resource/Otaki,_New_Zealand, http://dbpedia.org/resource/Park_County,_Montana, http://dbpedia.org/resource/Pelkosenniemi, http://dbpedia.org/resource/Persian_Armenia, http://dbpedia.org/resource/Porirua, http://dbpedia.org/resource/Princess_Anne_County,_Virginia, http://dbpedia.org/resource/Quadling_Country, http://dbpedia.org/resource/Russian_Armenia, http://dbpedia.org/resource/Satrapy_of_Armenia, http://dbpedia.org/resource/Stolberg-Wernigerode, http://dbpedia.org/resource/Suhma_Kingdom, http://dbpedia.org/resource/Sulayhid_dynasty, http://dbpedia.org/resource/Sultanate_of_Lahej, http://dbpedia.org/resource/System_partition_and_boot_partition, http://dbpedia.org/resource/Tahirids_(Yemen), http://dbpedia.org/resource/Tahltan, http://dbpedia.org/resource/Tambov_Governorate, http://dbpedia.org/resource/Te_Āti_Awa, http://dbpedia.org/resource/Travancore-Cochin, http://dbpedia.org/resource/Tulip_period, http://dbpedia.org/resource/Vaspurakan, http://dbpedia.org/resource/Viipuri_Province, http://dbpedia.org/resource/Wellington_(New_Zealand_electorate), http://dbpedia.org/resource/Were, http://dbpedia.org/resource/Wernigerode_Castle, http://dbpedia.org/resource/Wing_Chun_terms, http://dbpedia.org/resource/Yemen_Eyalet, http://dbpedia.org/resource/Yoga_physiology, http://dbpedia.org/resource/Yotvingia, http://dbpedia.org/resource/Yufirids, http://dbpedia.org/resource/Zakarid_Armenia, http://dbpedia.org/resource/Ziyadid_dynasty, http://dbpedia.org/resource/Zurayids, http://dbpedia.org/resource/Alsace_wine, http://dbpedia.org/resource/Articles_of_Leith, http://dbpedia.org/resource/Bettani, http://dbpedia.org/resource/Birmingham,_Erie_County,_Ohio, http://dbpedia.org/resource/Bunte_Republik_Neustadt, http://dbpedia.org/resource/Cantre'r_Gwaelod, http://dbpedia.org/resource/Constitution_of_Yucatán, http://dbpedia.org/resource/Cotnari, http://dbpedia.org/resource/Dalton_Minimum, http://dbpedia.org/resource/Duchy_of_Estonia_(1561–1721), http://dbpedia.org/resource/Eastern_Ganga_dynasty, http://dbpedia.org/resource/Education_for_sustainable_development, http://dbpedia.org/resource/Electoral_district_of_Hastings, http://dbpedia.org/resource/Electoral_district_of_Nepean, http://dbpedia.org/resource/Ethnocracy, http://dbpedia.org/resource/First_Mexican_Republic, http://dbpedia.org/resource/Friuli-Venezia_Giulia_wine, http://dbpedia.org/resource/Government_of_Florida, http://dbpedia.org/resource/Haryanka_dynasty, http://dbpedia.org/resource/Ituri_Rainforest, http://dbpedia.org/resource/Jundiaí, http://dbpedia.org/resource/Kadamba_dynasty, http://dbpedia.org/resource/Kalabhras_dynasty, http://dbpedia.org/resource/Kalachuri_dynasty, http://dbpedia.org/resource/Kamarupa, http://dbpedia.org/resource/Kantoli, http://dbpedia.org/resource/Karlani]... [INFO] P: 7.257947452460444E-5, R: 1.0, F: 1.4514841425357426E-4 -[INFO] Cumulative metrics, sample: 138, P: 0.643, R: 0.672, macro F1: 0.640, macro F1*: 0.657 -[INFO] Current time: 1486 ms, average: 2861.18 ms/ques, entity detection: 391.73 ms/ques (linking tool: 302.88 ms/ques), relation detection: 850.25 ms/ques (similarity: 681.31 ms/ques), EDG generation: 87.64 ms/ques +[INFO] Cumulative metrics, sample: 138, P: 0.643, R: 0.672, macro F1: 0.657 +[INFO] Current question consumed time: 1412 ms, average: 2724.64 ms/ques, entity detection: 377.57 ms/ques (linking tool: 293.82 ms/ques), relation detection: 833.60 ms/ques (similarity: 679.63 ms/ques), EDG generation: 86.90 ms/ques [INFO] Entity Linking P: 0.8260869565217391, R: 0.7971014492753623, micro F1: 0.8067632850241547, macro F1: 0.8113354037267081 [INFO] Relation Linking P: 0.6545893719806763, R: 0.6521739130434783, micro F1: 0.6492753623188406, macro F1: 0.6533794101101021 [INFO] Type Linking P: 0.4782608695652174, R: 0.4782608695652174, micro F1: 0.4782608695652174, macro F1: 0.4782608695652174 @@ -1940,8 +1940,8 @@ Question 138: Name the team which is owned by Dafenham wind turbines and stood f [INFO] Golden Answer: [http://dbpedia.org/resource/Ford_Motor_Company] [INFO] Predicted Answer: [http://dbpedia.org/resource/Ford_Motor_Company] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 139, P: 0.645, R: 0.674, macro F1: 0.642, macro F1*: 0.659 -[INFO] Current time: 3204 ms, average: 2863.65 ms/ques, entity detection: 392.05 ms/ques (linking tool: 302.49 ms/ques), relation detection: 850.19 ms/ques (similarity: 682.85 ms/ques), EDG generation: 87.32 ms/ques +[INFO] Cumulative metrics, sample: 139, P: 0.645, R: 0.674, macro F1: 0.659 +[INFO] Current question consumed time: 2906 ms, average: 2725.95 ms/ques, entity detection: 377.53 ms/ques (linking tool: 293.27 ms/ques), relation detection: 833.22 ms/ques (similarity: 681.06 ms/ques), EDG generation: 86.50 ms/ques [INFO] Entity Linking P: 0.8273381294964028, R: 0.7949640287769785, micro F1: 0.8057553956834534, macro F1: 0.8108280559587807 [INFO] Relation Linking P: 0.6570743405275778, R: 0.6510791366906474, micro F1: 0.6494004796163071, macro F1: 0.6540630007451417 [INFO] Type Linking P: 0.48201438848920863, R: 0.48201438848920863, micro F1: 0.48201438848920863, macro F1: 0.48201438848920863 @@ -1954,8 +1954,8 @@ Question 139: Which education center Sidney Catlin Partrodge attended which was [INFO] Golden Answer: [http://dbpedia.org/resource/Yale_University] [INFO] Predicted Answer: [http://dbpedia.org/resource/Yale_University] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 140, P: 0.648, R: 0.677, macro F1: 0.645, macro F1*: 0.662 -[INFO] Current time: 2429 ms, average: 2860.54 ms/ques, entity detection: 391.26 ms/ques (linking tool: 301.46 ms/ques), relation detection: 850.14 ms/ques (similarity: 681.82 ms/ques), EDG generation: 87.53 ms/ques +[INFO] Cumulative metrics, sample: 140, P: 0.648, R: 0.677, macro F1: 0.662 +[INFO] Current question consumed time: 2378 ms, average: 2723.46 ms/ques, entity detection: 376.81 ms/ques (linking tool: 292.27 ms/ques), relation detection: 833.44 ms/ques (similarity: 679.84 ms/ques), EDG generation: 86.54 ms/ques [INFO] Entity Linking P: 0.8285714285714286, R: 0.7964285714285714, micro F1: 0.8071428571428574, macro F1: 0.812182103610675 [INFO] Relation Linking P: 0.6595238095238095, R: 0.6535714285714286, micro F1: 0.651904761904762, macro F1: 0.6565341277036654 [INFO] Type Linking P: 0.4857142857142857, R: 0.4857142857142857, micro F1: 0.4857142857142857, macro F1: 0.4857142857142857 @@ -1968,8 +1968,8 @@ Question 140: What sports can be played in Kharghar's schools [INFO] Golden Answer: [http://dbpedia.org/resource/Association_football, http://dbpedia.org/resource/Basketball, http://dbpedia.org/resource/Chess, http://dbpedia.org/resource/Cricket, http://dbpedia.org/resource/Handball, http://dbpedia.org/resource/Sport_of_athletics] [INFO] Predicted Answer: [http://dbpedia.org/resource/Maharashtra, http://dbpedia.org/resource/Raigad_district] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 141, P: 0.643, R: 0.672, macro F1: 0.640, macro F1*: 0.657 -[INFO] Current time: 1304 ms, average: 2849.50 ms/ques, entity detection: 389.07 ms/ques (linking tool: 299.68 ms/ques), relation detection: 848.06 ms/ques (similarity: 681.70 ms/ques), EDG generation: 87.03 ms/ques +[INFO] Cumulative metrics, sample: 141, P: 0.643, R: 0.672, macro F1: 0.657 +[INFO] Current question consumed time: 1300 ms, average: 2713.37 ms/ques, entity detection: 374.69 ms/ques (linking tool: 290.53 ms/ques), relation detection: 831.40 ms/ques (similarity: 679.60 ms/ques), EDG generation: 86.04 ms/ques [INFO] Entity Linking P: 0.8297872340425532, R: 0.7978723404255319, micro F1: 0.8085106382978725, macro F1: 0.8135168961201502 [INFO] Relation Linking P: 0.6548463356973995, R: 0.648936170212766, micro F1: 0.6472813238770686, macro F1: 0.6518778572944195 [INFO] Type Linking P: 0.48226950354609927, R: 0.48226950354609927, micro F1: 0.48226950354609927, macro F1: 0.48226950354609927 @@ -1982,8 +1982,8 @@ Question 141: Which ethinicity of Riley Reid is the state of origin of Holmes C [INFO] Golden Answer: [http://dbpedia.org/resource/Chickasaw] [INFO] Predicted Answer: [http://dbpedia.org/resource/Chickasaw] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 142, P: 0.646, R: 0.674, macro F1: 0.643, macro F1*: 0.660 -[INFO] Current time: 1502 ms, average: 2840.01 ms/ques, entity detection: 388.88 ms/ques (linking tool: 298.97 ms/ques), relation detection: 843.42 ms/ques (similarity: 677.76 ms/ques), EDG generation: 86.63 ms/ques +[INFO] Cumulative metrics, sample: 142, P: 0.646, R: 0.674, macro F1: 0.660 +[INFO] Current question consumed time: 1246 ms, average: 2703.04 ms/ques, entity detection: 374.57 ms/ques (linking tool: 289.78 ms/ques), relation detection: 826.86 ms/ques (similarity: 675.68 ms/ques), EDG generation: 85.60 ms/ques [INFO] Entity Linking P: 0.8309859154929577, R: 0.7992957746478874, micro F1: 0.8098591549295777, macro F1: 0.8148328415416908 [INFO] Relation Linking P: 0.6572769953051643, R: 0.6514084507042254, micro F1: 0.6497652582159625, macro F1: 0.6543295648329439 [INFO] Type Linking P: 0.4859154929577465, R: 0.4859154929577465, micro F1: 0.4859154929577465, macro F1: 0.4859154929577465 @@ -1996,8 +1996,8 @@ Question 142: Was John Boehner the winner of the US House of Representatives ele [INFO] Golden Answer: [true] [INFO] Predicted Answer: [false] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 143, P: 0.641, R: 0.669, macro F1: 0.638, macro F1*: 0.655 -[INFO] Current time: 4870 ms, average: 2854.21 ms/ques, entity detection: 386.16 ms/ques (linking tool: 296.88 ms/ques), relation detection: 837.52 ms/ques (similarity: 675.94 ms/ques), EDG generation: 86.93 ms/ques +[INFO] Cumulative metrics, sample: 143, P: 0.641, R: 0.669, macro F1: 0.655 +[INFO] Current question consumed time: 4503 ms, average: 2715.62 ms/ques, entity detection: 371.95 ms/ques (linking tool: 287.76 ms/ques), relation detection: 821.08 ms/ques (similarity: 673.47 ms/ques), EDG generation: 85.87 ms/ques [INFO] Entity Linking P: 0.8251748251748252, R: 0.7937062937062938, micro F1: 0.8041958041958044, macro F1: 0.809134709782658 [INFO] Relation Linking P: 0.6526806526806527, R: 0.6468531468531469, micro F1: 0.6452214452214453, macro F1: 0.6497538336103359 [INFO] Type Linking P: 0.48951048951048953, R: 0.48951048951048953, micro F1: 0.48951048951048953, macro F1: 0.48951048951048953 @@ -2010,8 +2010,8 @@ Question 143: Which american football team has stadium as O.co Coliseum? [INFO] Golden Answer: [http://dbpedia.org/resource/Oakland_Raiders] [INFO] Predicted Answer: [11] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 144, P: 0.637, R: 0.665, macro F1: 0.634, macro F1*: 0.650 -[INFO] Current time: 3688 ms, average: 2860.00 ms/ques, entity detection: 388.69 ms/ques (linking tool: 299.60 ms/ques), relation detection: 840.60 ms/ques (similarity: 674.62 ms/ques), EDG generation: 86.46 ms/ques +[INFO] Cumulative metrics, sample: 144, P: 0.637, R: 0.665, macro F1: 0.650 +[INFO] Current question consumed time: 3577 ms, average: 2721.60 ms/ques, entity detection: 374.05 ms/ques (linking tool: 290.02 ms/ques), relation detection: 823.94 ms/ques (similarity: 672.09 ms/ques), EDG generation: 85.40 ms/ques [INFO] Entity Linking P: 0.8194444444444444, R: 0.7881944444444444, micro F1: 0.7986111111111113, macro F1: 0.8035157187425006 [INFO] Relation Linking P: 0.6481481481481481, R: 0.6423611111111112, micro F1: 0.6407407407407408, macro F1: 0.6452416542102641 [INFO] Type Linking P: 0.4861111111111111, R: 0.4861111111111111, micro F1: 0.4861111111111111, macro F1: 0.4861111111111111 @@ -2024,8 +2024,8 @@ Question 144: How many subjects are covered in the publications of Doubleday? [INFO] Golden Answer: [109] [INFO] Predicted Answer: [] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 145, P: 0.632, R: 0.660, macro F1: 0.629, macro F1*: 0.646 -[INFO] Current time: 1192 ms, average: 2848.50 ms/ques, entity detection: 387.48 ms/ques (linking tool: 298.32 ms/ques), relation detection: 837.07 ms/ques (similarity: 671.22 ms/ques), EDG generation: 86.47 ms/ques +[INFO] Cumulative metrics, sample: 145, P: 0.632, R: 0.660, macro F1: 0.646 +[INFO] Current question consumed time: 1143 ms, average: 2710.72 ms/ques, entity detection: 372.88 ms/ques (linking tool: 288.81 ms/ques), relation detection: 820.53 ms/ques (similarity: 668.74 ms/ques), EDG generation: 85.48 ms/ques [INFO] Entity Linking P: 0.8137931034482758, R: 0.7827586206896552, micro F1: 0.7931034482758622, macro F1: 0.7979742310270351 [INFO] Relation Linking P: 0.6436781609195402, R: 0.6379310344827587, micro F1: 0.6363218390804599, macro F1: 0.6407917117674348 [INFO] Type Linking P: 0.4896551724137931, R: 0.4896551724137931, micro F1: 0.4896551724137931, macro F1: 0.4896551724137931 @@ -2038,8 +2038,8 @@ Question 145: In which city of Pittsburg Steelers was the Congress of Inductrial [INFO] Golden Answer: [http://dbpedia.org/resource/Pittsburgh] [INFO] Predicted Answer: [http://dbpedia.org/resource/Heinz_Field, http://dbpedia.org/resource/Pennsylvania, http://dbpedia.org/resource/Pittsburgh] [INFO] P: 0.3333333333333333, R: 1.0, F: 0.5 -[INFO] Cumulative metrics, sample: 146, P: 0.630, R: 0.663, macro F1: 0.628, macro F1*: 0.646 -[INFO] Current time: 7242 ms, average: 2878.60 ms/ques, entity detection: 391.46 ms/ques (linking tool: 301.86 ms/ques), relation detection: 855.29 ms/ques (similarity: 680.54 ms/ques), EDG generation: 86.06 ms/ques +[INFO] Cumulative metrics, sample: 146, P: 0.630, R: 0.663, macro F1: 0.646 +[INFO] Current question consumed time: 6850 ms, average: 2739.07 ms/ques, entity detection: 376.96 ms/ques (linking tool: 292.42 ms/ques), relation detection: 837.84 ms/ques (similarity: 677.97 ms/ques), EDG generation: 85.07 ms/ques [INFO] Entity Linking P: 0.815068493150685, R: 0.7808219178082192, micro F1: 0.7922374429223746, macro F1: 0.7975777529543183 [INFO] Relation Linking P: 0.6461187214611872, R: 0.636986301369863, micro F1: 0.6365296803652969, macro F1: 0.6415200116998976 [INFO] Type Linking P: 0.4931506849315068, R: 0.4931506849315068, micro F1: 0.4931506849315068, macro F1: 0.4931506849315068 @@ -2052,8 +2052,8 @@ Question 146: What is the hometown of the residents of Martha's vineyards? [INFO] Golden Answer: [http://dbpedia.org/resource/Boston, http://dbpedia.org/resource/Massachusetts] [INFO] Predicted Answer: [http://dbpedia.org/resource/Voice_(duo), http://dbpedia.org/resource/Valanto_Trifonos, http://dbpedia.org/resource/Evdokia_Kadi, http://dbpedia.org/resource/Christina_Metaxa, http://dbpedia.org/resource/Stavros_Michalakakos, http://dbpedia.org/resource/Despina_Olympiou, http://dbpedia.org/resource/Nikolas_Metaxas, http://dbpedia.org/resource/Marios_Tokas, http://dbpedia.org/resource/One_(band), http://dbpedia.org/resource/Alkinoos_Ioannidis, http://dbpedia.org/resource/Evridiki, http://dbpedia.org/resource/Michalis_Hatzigiannis, http://dbpedia.org/resource/Winter's_Verge, http://dbpedia.org/resource/Mihalis_Violaris, http://dbpedia.org/resource/Diam's, http://dbpedia.org/resource/Loukas_Giorkas, http://dbpedia.org/resource/Monsieur_Doumani, http://dbpedia.org/resource/Stormcast, http://dbpedia.org/resource/Marlen_Angelidou, http://dbpedia.org/resource/Stavros_Konstantinou, http://dbpedia.org/resource/Okan_Ersan, http://dbpedia.org/resource/Minus_One_(band), http://dbpedia.org/resource/Yuri_Melikov, http://dbpedia.org/resource/Andreas_Paraschos] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 147, P: 0.626, R: 0.658, macro F1: 0.624, macro F1*: 0.642 -[INFO] Current time: 4875 ms, average: 2892.18 ms/ques, entity detection: 390.84 ms/ques (linking tool: 300.79 ms/ques), relation detection: 868.31 ms/ques (similarity: 686.57 ms/ques), EDG generation: 85.65 ms/ques +[INFO] Cumulative metrics, sample: 147, P: 0.626, R: 0.658, macro F1: 0.642 +[INFO] Current question consumed time: 4738 ms, average: 2752.67 ms/ques, entity detection: 376.17 ms/ques (linking tool: 291.33 ms/ques), relation detection: 850.56 ms/ques (similarity: 683.95 ms/ques), EDG generation: 84.63 ms/ques [INFO] Entity Linking P: 0.8095238095238095, R: 0.7755102040816326, micro F1: 0.7868480725623584, macro F1: 0.7921520539546291 [INFO] Relation Linking P: 0.645124716553288, R: 0.6360544217687075, micro F1: 0.6356009070294786, macro F1: 0.6405574619228221 [INFO] Type Linking P: 0.4965986394557823, R: 0.4965986394557823, micro F1: 0.4965986394557823, macro F1: 0.4965986394557823 @@ -2066,8 +2066,8 @@ Question 147: Name the sea into which Danish Straits flows and has Kaliningrad a [INFO] Golden Answer: [http://dbpedia.org/resource/Baltic_Sea] [INFO] Predicted Answer: [] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 148, P: 0.622, R: 0.654, macro F1: 0.620, macro F1*: 0.637 -[INFO] Current time: 1055 ms, average: 2879.77 ms/ques, entity detection: 388.20 ms/ques (linking tool: 298.76 ms/ques), relation detection: 863.18 ms/ques (similarity: 682.36 ms/ques), EDG generation: 85.57 ms/ques +[INFO] Cumulative metrics, sample: 148, P: 0.622, R: 0.654, macro F1: 0.637 +[INFO] Current question consumed time: 1011 ms, average: 2740.90 ms/ques, entity detection: 373.63 ms/ques (linking tool: 289.36 ms/ques), relation detection: 845.43 ms/ques (similarity: 679.73 ms/ques), EDG generation: 84.48 ms/ques [INFO] Entity Linking P: 0.8040540540540541, R: 0.7702702702702703, micro F1: 0.7815315315315318, macro F1: 0.7867996752116924 [INFO] Relation Linking P: 0.6407657657657657, R: 0.6317567567567568, micro F1: 0.6313063063063065, macro F1: 0.6362293709638842 [INFO] Type Linking P: 0.49324324324324326, R: 0.49324324324324326, micro F1: 0.49324324324324326, macro F1: 0.49324324324324326 @@ -2080,8 +2080,8 @@ Question 148: Name the team manger of Middlebrough F.C season 2006-07 ? [INFO] Golden Answer: [http://dbpedia.org/resource/Gareth_Southgate] [INFO] Predicted Answer: [http://dbpedia.org/resource/Middlesbrough_Frontline] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 149, P: 0.617, R: 0.649, macro F1: 0.616, macro F1*: 0.633 -[INFO] Current time: 2271 ms, average: 2875.68 ms/ques, entity detection: 387.83 ms/ques (linking tool: 298.81 ms/ques), relation detection: 865.62 ms/ques (similarity: 682.99 ms/ques), EDG generation: 85.17 ms/ques +[INFO] Cumulative metrics, sample: 149, P: 0.617, R: 0.649, macro F1: 0.633 +[INFO] Current question consumed time: 2311 ms, average: 2738.01 ms/ques, entity detection: 373.46 ms/ques (linking tool: 289.54 ms/ques), relation detection: 848.03 ms/ques (similarity: 680.34 ms/ques), EDG generation: 84.07 ms/ques [INFO] Entity Linking P: 0.7986577181208053, R: 0.7651006711409396, micro F1: 0.7762863534675617, macro F1: 0.781519140478728 [INFO] Relation Linking P: 0.6364653243847874, R: 0.62751677852349, micro F1: 0.6270693512304252, macro F1: 0.6319593751856031 [INFO] Type Linking P: 0.4966442953020134, R: 0.4966442953020134, micro F1: 0.4966442953020134, macro F1: 0.4966442953020134 @@ -2094,8 +2094,8 @@ Question 149: What city has the football team in which Lee Robinson debuted? [INFO] Golden Answer: [http://dbpedia.org/resource/Colorado, http://dbpedia.org/resource/Denver, http://dbpedia.org/resource/Sports_Authority_Field_at_Mile_High] [INFO] Predicted Answer: [http://dbpedia.org/resource/Kilmarnock_F.C., http://dbpedia.org/resource/Rangers_F.C., http://dbpedia.org/resource/Greenock_Morton_F.C., http://dbpedia.org/resource/Raith_Rovers_F.C., http://dbpedia.org/resource/St_Johnstone_F.C., http://dbpedia.org/resource/Queen_of_the_South_F.C., http://dbpedia.org/resource/Östersunds_FK] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 150, P: 0.613, R: 0.645, macro F1: 0.612, macro F1*: 0.629 -[INFO] Current time: 1215 ms, average: 2864.61 ms/ques, entity detection: 386.40 ms/ques (linking tool: 297.59 ms/ques), relation detection: 862.30 ms/ques (similarity: 681.03 ms/ques), EDG generation: 84.77 ms/ques +[INFO] Cumulative metrics, sample: 150, P: 0.613, R: 0.645, macro F1: 0.629 +[INFO] Current question consumed time: 1148 ms, average: 2727.41 ms/ques, entity detection: 372.11 ms/ques (linking tool: 288.37 ms/ques), relation detection: 844.61 ms/ques (similarity: 678.25 ms/ques), EDG generation: 83.65 ms/ques [INFO] Entity Linking P: 0.7933333333333333, R: 0.76, micro F1: 0.7711111111111113, macro F1: 0.7763090128755366 [INFO] Relation Linking P: 0.6322222222222222, R: 0.6233333333333333, micro F1: 0.622888888888889, macro F1: 0.6277463126843658 [INFO] Type Linking P: 0.5, R: 0.5, micro F1: 0.5, macro F1: 0.5 @@ -2108,8 +2108,8 @@ Question 150: Who is the owner of the bank which owns Bloomberg Radio? [INFO] Golden Answer: [http://dbpedia.org/resource/Michael_Bloomberg] [INFO] Predicted Answer: [http://dbpedia.org/resource/Bloomberg_News, http://dbpedia.org/resource/Bloomberg_Radio, http://dbpedia.org/resource/Bloomberg_BNA, http://dbpedia.org/resource/WBBR, http://dbpedia.org/resource/Michael_Bloomberg, http://dbpedia.org/resource/Bloomberg_TV_Indonesia, http://dbpedia.org/resource/Bloomberg_Television, http://dbpedia.org/resource/Bloomberg_TV_India, http://dbpedia.org/resource/Bloomberg_TV_Philippines] [INFO] P: 0.1111111111111111, R: 1.0, F: 0.19999999999999998 -[INFO] Cumulative metrics, sample: 151, P: 0.610, R: 0.647, macro F1: 0.609, macro F1*: 0.628 -[INFO] Current time: 1395 ms, average: 2854.88 ms/ques, entity detection: 385.29 ms/ques (linking tool: 296.62 ms/ques), relation detection: 859.58 ms/ques (similarity: 678.77 ms/ques), EDG generation: 84.35 ms/ques +[INFO] Cumulative metrics, sample: 151, P: 0.610, R: 0.647, macro F1: 0.628 +[INFO] Current question consumed time: 1297 ms, average: 2717.94 ms/ques, entity detection: 370.97 ms/ques (linking tool: 287.36 ms/ques), relation detection: 841.87 ms/ques (similarity: 675.96 ms/ques), EDG generation: 83.23 ms/ques [INFO] Entity Linking P: 0.7947019867549668, R: 0.7615894039735099, micro F1: 0.7726269315673291, macro F1: 0.7777934338452868 [INFO] Relation Linking P: 0.6346578366445916, R: 0.6258278145695364, micro F1: 0.6253863134657838, macro F1: 0.6302118973335963 [INFO] Type Linking P: 0.4966887417218543, R: 0.4966887417218543, micro F1: 0.4966887417218543, macro F1: 0.4966887417218543 @@ -2122,8 +2122,8 @@ Question 151: Which companies were behind the renovation of McKechnie Field ? [INFO] Golden Answer: [http://dbpedia.org/resource/Pittsburgh, http://dbpedia.org/resource/Louis_D._Astorino, Fawley Bryant Architects, Bradenton, Florida@en] [INFO] Predicted Answer: [http://dbpedia.org/resource/118_118_(UK), http://dbpedia.org/resource/1910_Rogers_Pass_avalanche, http://dbpedia.org/resource/1979_Mississauga_train_derailment, http://dbpedia.org/resource/2005_Tim_Hortons_Brier, http://dbpedia.org/resource/78_Tage_auf_der_Straße_des_Hasses, http://dbpedia.org/resource/8-track_tape, http://dbpedia.org/resource/A._E._Staley, http://dbpedia.org/resource/ABB_Group, http://dbpedia.org/resource/AKN_Eisenbahn, http://dbpedia.org/resource/ANEK_Lines, http://dbpedia.org/resource/ARC_Ratings, http://dbpedia.org/resource/AWG_plc, http://dbpedia.org/resource/AXA, http://dbpedia.org/resource/Acid_Jazz_Records, http://dbpedia.org/resource/Acme_Corporation, http://dbpedia.org/resource/Actors'_Theatre_of_Columbus, http://dbpedia.org/resource/Adam_Kennedy_(programmer), http://dbpedia.org/resource/Adaptiv, http://dbpedia.org/resource/Adolph_Zukor, http://dbpedia.org/resource/Adventure_Time, http://dbpedia.org/resource/Aeolian-Skinner, http://dbpedia.org/resource/Aeronautical_Development_Establishment, http://dbpedia.org/resource/Aetna, http://dbpedia.org/resource/AirScan, http://dbpedia.org/resource/Air_Force_Space_Command, http://dbpedia.org/resource/Akihiko_Matsui, http://dbpedia.org/resource/Akron_Beacon_Journal, http://dbpedia.org/resource/All_4, http://dbpedia.org/resource/All_England_Lawn_Tennis_and_Croquet_Club, http://dbpedia.org/resource/Allegiant_Air, http://dbpedia.org/resource/Allstate, http://dbpedia.org/resource/Amati, http://dbpedia.org/resource/American_City_Business_Journals, http://dbpedia.org/resource/American_Electric_Power, http://dbpedia.org/resource/American_Heritage_(magazine), http://dbpedia.org/resource/American_Woolen_Company, http://dbpedia.org/resource/Analog_television, http://dbpedia.org/resource/Andrews_McMeel_Publishing, http://dbpedia.org/resource/Andrews_McMeel_Universal, http://dbpedia.org/resource/Andy_Griffith, http://dbpedia.org/resource/Anglo_American_plc, http://dbpedia.org/resource/Aon_Center_(Chicago), http://dbpedia.org/resource/Apheresis, http://dbpedia.org/resource/Apollo_Diamond, http://dbpedia.org/resource/Applied_Industrial_Technologies, http://dbpedia.org/resource/Arab_American_Vehicles, http://dbpedia.org/resource/Arboriculture, http://dbpedia.org/resource/Archie_Comics, http://dbpedia.org/resource/Arduino, http://dbpedia.org/resource/Arena_Football_League, http://dbpedia.org/resource/Arnaldo_Carvalho_de_Melo, http://dbpedia.org/resource/Arson_in_royal_dockyards, http://dbpedia.org/resource/Asa_Griggs_Candler, http://dbpedia.org/resource/Ashford_Castle, http://dbpedia.org/resource/Asia_Society, http://dbpedia.org/resource/Association_of_International_Accountants, http://dbpedia.org/resource/Astral_Media, http://dbpedia.org/resource/At_the_Races, http://dbpedia.org/resource/Atmel, http://dbpedia.org/resource/Augmented_reality, http://dbpedia.org/resource/Automatic_firearm, http://dbpedia.org/resource/Aviation_Industry_Corporation_of_China, http://dbpedia.org/resource/BBC_Cymru_Wales, http://dbpedia.org/resource/BBC_Radio_Cornwall, http://dbpedia.org/resource/BGC_Partners, http://dbpedia.org/resource/BL_9.2-inch_howitzer, http://dbpedia.org/resource/Baker_Motor_Vehicle, http://dbpedia.org/resource/Baldwin_Locomotive_Works, http://dbpedia.org/resource/BalletMet, http://dbpedia.org/resource/Banff_Springs_Hotel, http://dbpedia.org/resource/Barbecue_in_the_United_States, http://dbpedia.org/resource/Barry_Railway_Company, http://dbpedia.org/resource/Basic_Channel, http://dbpedia.org/resource/Batchelors, http://dbpedia.org/resource/Battelle_Memorial_Institute, http://dbpedia.org/resource/Beer_in_China, http://dbpedia.org/resource/Bellator_MMA, http://dbpedia.org/resource/Belron, http://dbpedia.org/resource/Beovoz, http://dbpedia.org/resource/Beretta, http://dbpedia.org/resource/Bernie_Kosar, http://dbpedia.org/resource/Better_Farming_Train_(Saskatchewan), http://dbpedia.org/resource/Biblica, http://dbpedia.org/resource/Big_4_(sculpture), http://dbpedia.org/resource/Big_Lots, http://dbpedia.org/resource/Bilbie_family, http://dbpedia.org/resource/Bill_Tytla, http://dbpedia.org/resource/Binary_data, http://dbpedia.org/resource/Bio-energy_with_carbon_capture_and_storage, http://dbpedia.org/resource/Bioluminescence, http://dbpedia.org/resource/Bitmap_textures, http://dbpedia.org/resource/Black_Dog_Publishing, http://dbpedia.org/resource/Blast_fishing, http://dbpedia.org/resource/Blippar, http://dbpedia.org/resource/Blue_Train_(South_Africa), http://dbpedia.org/resource/Bokomo, http://dbpedia.org/resource/Brainpool_TV, http://dbpedia.org/resource/Breizh_Cola, http://dbpedia.org/resource/Brick_Lane, http://dbpedia.org/resource/Bridge_of_Independent_Lists, http://dbpedia.org/resource/Bridgewater_Associates, http://dbpedia.org/resource/Brigham_Young_University_Press, http://dbpedia.org/resource/Brinkmanship, http://dbpedia.org/resource/Bristol_Water, http://dbpedia.org/resource/British_Broadcasting_Company, http://dbpedia.org/resource/Broadway_Across_America, http://dbpedia.org/resource/Brunswick_Transit_Alternative, http://dbpedia.org/resource/Brøderbund, http://dbpedia.org/resource/Buckeye_Steel_Castings, http://dbpedia.org/resource/Buenos_Aires_and_Pacific_Railway, http://dbpedia.org/resource/Building_regulations_in_the_United_Kingdom, http://dbpedia.org/resource/Burya, http://dbpedia.org/resource/C++_classes, http://dbpedia.org/resource/CBS_Drama, http://dbpedia.org/resource/CCTV_New_Year's_Gala, http://dbpedia.org/resource/CERT_Coordination_Center, http://dbpedia.org/resource/CFA_Institute, http://dbpedia.org/resource/CFXJ-FM, http://dbpedia.org/resource/CGI_Group, http://dbpedia.org/resource/CHUM-FM, http://dbpedia.org/resource/CKFM-FM, http://dbpedia.org/resource/COSI_Columbus, http://dbpedia.org/resource/CPM_Group, http://dbpedia.org/resource/CPR_Angus_Shops, http://dbpedia.org/resource/CP_Ships, http://dbpedia.org/resource/CQ_Press, http://dbpedia.org/resource/CRT_Group, http://dbpedia.org/resource/CTfastrak, http://dbpedia.org/resource/Cabal_(software), http://dbpedia.org/resource/Cairo_Transportation_Authority, http://dbpedia.org/resource/Calgary_Herald, http://dbpedia.org/resource/California_Hotel_and_Casino, http://dbpedia.org/resource/California_Legacy_Project, http://dbpedia.org/resource/Callisto_Corporation, http://dbpedia.org/resource/Canada_Lands_Company, http://dbpedia.org/resource/Canadian_Independent_Telephone_Association, http://dbpedia.org/resource/Canadian_Locomotive_Company, http://dbpedia.org/resource/Canadian_Pacific_Hotels, http://dbpedia.org/resource/Canadian_Pacific_Limited, http://dbpedia.org/resource/Canadian_Pacific_Navigation_Company, http://dbpedia.org/resource/Canadian_Pacific_Railway_Coast_Service, http://dbpedia.org/resource/Canadian_Pacific_Railway_Lake_and_River_Service, http://dbpedia.org/resource/Canadian_Pacific_Railway_Upper_Lake_Service, http://dbpedia.org/resource/Canadian_Professional_Sales_Association, http://dbpedia.org/resource/Canadian_Radio_Broadcasting_Commission, http://dbpedia.org/resource/Canal_&_River_Trust, http://dbpedia.org/resource/CapeRay, http://dbpedia.org/resource/Cape_Times, http://dbpedia.org/resource/Capitec_Bank, http://dbpedia.org/resource/Capitol_(collection), http://dbpedia.org/resource/Capitol_Air, http://dbpedia.org/resource/Capitol_Wrestling_Corporation, http://dbpedia.org/resource/Cardiff_Bus, http://dbpedia.org/resource/Cardinal_Brewery, http://dbpedia.org/resource/Cardinal_Health, http://dbpedia.org/resource/Carfax_(company), http://dbpedia.org/resource/Carlo_Giuseppe_Testore, http://dbpedia.org/resource/Case_CE, http://dbpedia.org/resource/Case_Corporation, http://dbpedia.org/resource/Case_IH, http://dbpedia.org/resource/Cassiopeia_(train), http://dbpedia.org/resource/Cedar_Falls_Utilities, http://dbpedia.org/resource/Celestial_Digital_Entertainment, http://dbpedia.org/resource/Central_America_Volcanic_Arc, http://dbpedia.org/resource/Central_Electricity_Generating_Board, http://dbpedia.org/resource/Central_Ohio_Transit_Authority, http://dbpedia.org/resource/Ceramic_materials, http://dbpedia.org/resource/Ceres,_Celestial_Legend, http://dbpedia.org/resource/Ceres,_Inc., http://dbpedia.org/resource/Ceres_Fruit_Juices, http://dbpedia.org/resource/Ceres_Hellenic_Shipping_Enterprises, http://dbpedia.org/resource/Ceres_Liner, http://dbpedia.org/resource/Ceres_Power, http://dbpedia.org/resource/Cerrejón, http://dbpedia.org/resource/Certified_Health_Executive, http://dbpedia.org/resource/Certified_Homeland_Protection_Professional, http://dbpedia.org/resource/Chagrin_Falls_Popcorn_Shop, http://dbpedia.org/resource/Champion_Beer_of_Britain, http://dbpedia.org/resource/Champion_Beer_of_Scotland, http://dbpedia.org/resource/Chandler_Motor_Car, http://dbpedia.org/resource/Channel_4_Banned_season, http://dbpedia.org/resource/Channel_4_Sheffield_Pitch, http://dbpedia.org/resource/Channel_Four_Television_Corporation, http://dbpedia.org/resource/Channel_I/O, http://dbpedia.org/resource/Charge_card, http://dbpedia.org/resource/Charged_Productions, http://dbpedia.org/resource/Charged_Records, http://dbpedia.org/resource/Charles_Dow, http://dbpedia.org/resource/Charles_Williams_(British_writer), http://dbpedia.org/resource/Charlotte_Knights, http://dbpedia.org/resource/Chartered_Certified_Accountant, http://dbpedia.org/resource/Chateau_Lake_Louise, http://dbpedia.org/resource/Chatham_Dockyard, http://dbpedia.org/resource/Chatto_&_Windus, http://dbpedia.org/resource/Cheddar_Ales, http://dbpedia.org/resource/Cheezies, http://dbpedia.org/resource/Chemical_toilet, http://dbpedia.org/resource/Chicago_Rapid_Transit_Company, http://dbpedia.org/resource/China_Central_Television, http://dbpedia.org/resource/Chinook_Jargon, http://dbpedia.org/resource/Christian_Television_Network, http://dbpedia.org/resource/Christian_the_lion, http://dbpedia.org/resource/Christianity_Today, http://dbpedia.org/resource/Christmas_card, http://dbpedia.org/resource/Chrono_Break, http://dbpedia.org/resource/Church_discipline, http://dbpedia.org/resource/Château_Frontenac, http://dbpedia.org/resource/Cigna, http://dbpedia.org/resource/CinemaScope, http://dbpedia.org/resource/Circus_Renz, http://dbpedia.org/resource/City_(TV_network), http://dbpedia.org/resource/Cleopatra_Records, http://dbpedia.org/resource/Cleveland,_Columbus_and_Cincinnati_Railroad, http://dbpedia.org/resource/Cleveland_Play_House, http://dbpedia.org/resource/Cleveland_State_University_Poetry_Center, http://dbpedia.org/resource/Cliffs_Natural_Resources, http://dbpedia.org/resource/Coca-Cola_Amatil, http://dbpedia.org/resource/Coca-Cola_Enterprises, http://dbpedia.org/resource/Coca-Cola_FEMSA, http://dbpedia.org/resource/Coca-Cola_Freestyle, http://dbpedia.org/resource/Coca-Cola_HBC_AG, http://dbpedia.org/resource/Coca-Cola_formula, http://dbpedia.org/resource/Coca-Cola_with_Lemon, http://dbpedia.org/resource/Cockta, http://dbpedia.org/resource/Code_Lyoko, http://dbpedia.org/resource/Coeur_Mining, http://dbpedia.org/resource/Coffea_canephora, http://dbpedia.org/resource/Coke_Zero_400, http://dbpedia.org/resource/Cola_Turka, http://dbpedia.org/resource/Colalife, http://dbpedia.org/resource/Collaborative_leadership, http://dbpedia.org/resource/Collection_agency, http://dbpedia.org/resource/College_Board, http://dbpedia.org/resource/Cologne_Opera, http://dbpedia.org/resource/Colonial_Radio_Theatre, http://dbpedia.org/resource/Colt's_Manufacturing_Company, http://dbpedia.org/resource/Columbia_Aircraft, http://dbpedia.org/resource/Columbia_Center_for_New_Media_Teaching_and_Learning, http://dbpedia.org/resource/Columbia_Insurance_Group, http://dbpedia.org/resource/Columbia_School_of_Engineering_and_Applied_Science, http://dbpedia.org/resource/Columbia_and_Kootenay_Railway, http://dbpedia.org/resource/Columbia_and_Western_Railway, http://dbpedia.org/resource/Columbus_and_Xenia_Railroad, http://dbpedia.org/resource/Comcast_SportsNet_Chicago, http://dbpedia.org/resource/ComiXology, http://dbpedia.org/resource/Comico, http://dbpedia.org/resource/Comma_operator, http://dbpedia.org/resource/Commission_for_the_Management_and_Application_of_Geoscience_Information, http://dbpedia.org/resource/Communion_(music), http://dbpedia.org/resource/Compacted_graphite_iron, http://dbpedia.org/resource/Compagnie_Générale_Immobiliere, http://dbpedia.org/resource/Companhia_Paulista_de_Trens_Metropolitanos, http://dbpedia.org/resource/Comparison_of_display_technology, http://dbpedia.org/resource/Compassion_International, http://dbpedia.org/resource/Complementarianism, http://dbpedia.org/resource/Concord_Area_Transit, http://dbpedia.org/resource/Concord_Coach_Lines, http://dbpedia.org/resource/Conductor_(company), http://dbpedia.org/resource/Connecticut_Land_Company, http://dbpedia.org/resource/Connecticut_Transit, http://dbpedia.org/resource/Connecticut_Western_Reserve, http://dbpedia.org/resource/Consolidated_Edison, http://dbpedia.org/resource/Contemporary_American_Theatre_Company, http://dbpedia.org/resource/Cool_Whip, http://dbpedia.org/resource/Copper_Canyon_Press, http://dbpedia.org/resource/Corcovado_Rack_Railway, http://dbpedia.org/resource/Coronation_Fund_Managers, http://dbpedia.org/resource/Corporate_Punishment_Records, http://dbpedia.org/resource/Corrib_gas_project, http://dbpedia.org/resource/Cortex_Pharmaceuticals, http://dbpedia.org/resource/Corvus_Systems, http://dbpedia.org/resource/Crain_Communications, http://dbpedia.org/resource/Cramer_brothers, http://dbpedia.org/resource/Crank!_A_Record_Company, http://dbpedia.org/resource/Cranks_(restaurant), http://dbpedia.org/resource/Crate, http://dbpedia.org/resource/Cretan_wine, http://dbpedia.org/resource/Crosley_Broadcasting_Corporation, http://dbpedia.org/resource/Crow_Rate, http://dbpedia.org/resource/Cyan_Worlds, http://dbpedia.org/resource/Cypress_Semiconductor, http://dbpedia.org/resource/Cyrillus, http://dbpedia.org/resource/Cyrus_Pallonji_Mistry, http://dbpedia.org/resource/Cölner_Hofbräu_Früh, http://dbpedia.org/resource/DB_Regio, http://dbpedia.org/resource/DDR3_SDRAM, http://dbpedia.org/resource/Dakota,_Minnesota_and_Eastern_Railroad, http://dbpedia.org/resource/Dan_DeCarlo, http://dbpedia.org/resource/Dartmoor_Preservation_Association, http://dbpedia.org/resource/Data_General_Eclipse_MV/8000, http://dbpedia.org/resource/Datasoft, http://dbpedia.org/resource/David_C._Cook, http://dbpedia.org/resource/Daylighting, http://dbpedia.org/resource/DeMoulas_Market_Basket, http://dbpedia.org/resource/Dean_Guitars, http://dbpedia.org/resource/Decaffeination, http://dbpedia.org/resource/Del, http://dbpedia.org/resource/Delaware_and_Hudson_Railway, http://dbpedia.org/resource/Delivery.com, http://dbpedia.org/resource/Dennis_Miller, http://dbpedia.org/resource/Denver_Newspaper_Agency, http://dbpedia.org/resource/Detroit–Superior_Bridge, http://dbpedia.org/resource/Deutschlandradio, http://dbpedia.org/resource/Deutz_AG, http://dbpedia.org/resource/Devil's_Due_Publishing, http://dbpedia.org/resource/Diesel-electric_transmission, http://dbpedia.org/resource/Differential_operator, http://dbpedia.org/resource/Digital_Linear_Tape, http://dbpedia.org/resource/Digital_television_in_the_United_Kingdom, http://dbpedia.org/resource/Direct-view_bistable_storage_tube, http://dbpedia.org/resource/Disco_Demolition_Night, http://dbpedia.org/resource/Disembowelment, http://dbpedia.org/resource/Disneyland_Resort, http://dbpedia.org/resource/DivX, http://dbpedia.org/resource/Dorling_Kindersley, http://dbpedia.org/resource/Double_chronograph, http://dbpedia.org/resource/Downtown_Columbia,_Missouri, http://dbpedia.org/resource/Dr_Pepper_Snapple_Group, http://dbpedia.org/resource/Dublin_Area_Rapid_Transit, http://dbpedia.org/resource/Duffy's_Tavern, http://dbpedia.org/resource/Dungeon_Siege_III, http://dbpedia.org/resource/Dynamic_Analysis_and_Replanning_Tool, http://dbpedia.org/resource/E-card, http://dbpedia.org/resource/E4_Series_Shinkansen, http://dbpedia.org/resource/EMKA,_Ltd., http://dbpedia.org/resource/ESPN_Deportes_Radio, http://dbpedia.org/resource/East_Japan_Railway_Company, http://dbpedia.org/resource/East_Somerset_Railway, http://dbpedia.org/resource/Eastland_Mall_(Columbus,_Ohio), http://dbpedia.org/resource/Eclipse_Comics, http://dbpedia.org/resource/Eddie_Einhorn, http://dbpedia.org/resource/Eddie_Rickenbacker, http://dbpedia.org/resource/Edo_Aircraft_Corporation, http://dbpedia.org/resource/Ehsan_Bayat, http://dbpedia.org/resource/Eight-to-fourteen_modulation, http://dbpedia.org/resource/Electra_Private_Equity, http://dbpedia.org/resource/Element_Six, http://dbpedia.org/resource/Emporis, http://dbpedia.org/resource/Energetically_modified_cement, http://dbpedia.org/resource/Energy_Star, http://dbpedia.org/resource/Energy_supply, http://dbpedia.org/resource/Enix, http://dbpedia.org/resource/Eric_Davis_(clown), http://dbpedia.org/resource/Eric_Miller_(businessman), http://dbpedia.org/resource/European_Pharmacopoeia, http://dbpedia.org/resource/European_professional_qualification_directives, http://dbpedia.org/resource/Evening_Chronicle, http://dbpedia.org/resource/Exact_Change, http://dbpedia.org/resource/Exclusive_psalmody, http://dbpedia.org/resource/FM104, http://dbpedia.org/resource/Fairmont_Hotels_and_Resorts, http://dbpedia.org/resource/Fairmont_Royal_York, http://dbpedia.org/resource/Fanta, http://dbpedia.org/resource/Feargus_Urquhart, http://dbpedia.org/resource/Ferdinand_Mülhens, http://dbpedia.org/resource/Film_studio, http://dbpedia.org/resource/Final_Fantasy_VI, http://dbpedia.org/resource/FirstEnergy, http://dbpedia.org/resource/First_Canadian_Place, http://dbpedia.org/resource/Flagship_(broadcasting), http://dbpedia.org/resource/Flash_welding, http://dbpedia.org/resource/Focal_Press, http://dbpedia.org/resource/Forest_City_Enterprises, http://dbpedia.org/resource/Fourscore, http://dbpedia.org/resource/Fox_Sports_(United_States), http://dbpedia.org/resource/Fox_Sports_Networks, http://dbpedia.org/resource/Fox_Sports_Radio, http://dbpedia.org/resource/Foxwoods_Resort_Casino, http://dbpedia.org/resource/Frank_Mason_Robinson, http://dbpedia.org/resource/Funcom, http://dbpedia.org/resource/Funk_&_Wagnalls, http://dbpedia.org/resource/FxPro, http://dbpedia.org/resource/GE_Capital, http://dbpedia.org/resource/GM_Korea, http://dbpedia.org/resource/Gaeth, http://dbpedia.org/resource/Gaffel_Becker_&_Co, http://dbpedia.org/resource/Gage_Educational_Publishing_Company, http://dbpedia.org/resource/Gazetteer, http://dbpedia.org/resource/Gene_Deitch, http://dbpedia.org/resource/General_Motors_Diesel, http://dbpedia.org/resource/Generali_Deutschland, http://dbpedia.org/resource/Genesis_HealthCare, http://dbpedia.org/resource/Gentleness, http://dbpedia.org/resource/Georg_Olden_(graphic_designer), http://dbpedia.org/resource/Ghabbour_Group, http://dbpedia.org/resource/Girl_Distribution_Company, http://dbpedia.org/resource/Giuseppe_Guarneri, http://dbpedia.org/resource/Go_Go_Gophers, http://dbpedia.org/resource/Golden_Arrow_Bus_Services, http://dbpedia.org/resource/Good_Beer_Guide, http://dbpedia.org/resource/Google_China, http://dbpedia.org/resource/Government_of_Ukraine, http://dbpedia.org/resource/Grand_Rapids_and_Indiana_Railroad, http://dbpedia.org/resource/Grand_Trunk_Railway, http://dbpedia.org/resource/Gravimetry, http://dbpedia.org/resource/Great_Lakes_Central_Railroad, http://dbpedia.org/resource/Great_Northern_Railway_(U.S.), http://dbpedia.org/resource/Greater_Cleveland_Film_Commission, http://dbpedia.org/resource/Groton_Iron_Works, http://dbpedia.org/resource/Guizhentang_Pharmaceutical_company, http://dbpedia.org/resource/Guy-wire, http://dbpedia.org/resource/Guy_Branum, http://dbpedia.org/resource/HACEB, http://dbpedia.org/resource/HCJB, http://dbpedia.org/resource/Hacker-Craft, http://dbpedia.org/resource/Hamilton_Standard, http://dbpedia.org/resource/Hamish_Hamilton, http://dbpedia.org/resource/Hank_Ketcham, http://dbpedia.org/resource/Hanson_plc, http://dbpedia.org/resource/Hapag-Lloyd, http://dbpedia.org/resource/Hardee's, http://dbpedia.org/resource/Harrods, http://dbpedia.org/resource/Headway, http://dbpedia.org/resource/Heinrich_Reissdorf, http://dbpedia.org/resource/Hershy_Kay, http://dbpedia.org/resource/Higbee's, http://dbpedia.org/resource/High-level_language_computer_architecture, http://dbpedia.org/resource/Hilbert's_paradox_of_the_Grand_Hotel, http://dbpedia.org/resource/Hindoostanee_Coffee_House, http://dbpedia.org/resource/Hippocrene_Books, http://dbpedia.org/resource/History_of_ITV, http://dbpedia.org/resource/Hogarth_Press, http://dbpedia.org/resource/Hotel_Vancouver, http://dbpedia.org/resource/Houston_Ballet's_Ben_Stevenson_Academy, http://dbpedia.org/resource/Howell-North_Books, http://dbpedia.org/resource/Hudepohl_Brewing_Company, http://dbpedia.org/resource/Huntington_Bancshares, http://dbpedia.org/resource/Hyatt, http://dbpedia.org/resource/Hyperwerks, http://dbpedia.org/resource/IDEN, http://dbpedia.org/resource/IEEE-488, http://dbpedia.org/resource/ITN, http://dbpedia.org/resource/ITT_Technical_Institute, http://dbpedia.org/resource/ITV4, http://dbpedia.org/resource/ITV_Breakfast, http://dbpedia.org/resource/ITV_Wales_&_West, http://dbpedia.org/resource/ITV_plc, http://dbpedia.org/resource/Increment_and_decrement_operators, http://dbpedia.org/resource/Independent_Broadcasting_Authority, http://dbpedia.org/resource/Independent_Family_Brewers_of_Britain, http://dbpedia.org/resource/Indian_Trails, http://dbpedia.org/resource/Indumil, http://dbpedia.org/resource/Inravisión, http://dbpedia.org/resource/Institute_for_Defense_Analyses, http://dbpedia.org/resource/Institute_of_Public_Accountants, http://dbpedia.org/resource/Integrated_gasification_combined_cycle, http://dbpedia.org/resource/Intel_Core_2, http://dbpedia.org/resource/Intellectual_Reserve, http://dbpedia.org/resource/Intensive_insulinotherapy, http://dbpedia.org/resource/Intercolonial_Railway, http://dbpedia.org/resource/International_Railway_of_Maine, http://dbpedia.org/resource/International_Watch_Company, http://dbpedia.org/resource/Inverse_limit, http://dbpedia.org/resource/Ion_Television, http://dbpedia.org/resource/Iowa,_Chicago_and_Eastern_Railroad, http://dbpedia.org/resource/Isaac_Charles_Johnson, http://dbpedia.org/resource/Island_Records, http://dbpedia.org/resource/Issaries,_Inc., http://dbpedia.org/resource/Jacob_Schiff, http://dbpedia.org/resource/James_Frost_(cement_maker), http://dbpedia.org/resource/James_M._Cox, http://dbpedia.org/resource/Japan_Railways_Group, http://dbpedia.org/resource/Jaques_of_London, http://dbpedia.org/resource/Jardine_Matheson, http://dbpedia.org/resource/Java_Desktop_System, http://dbpedia.org/resource/Java_Platform,_Micro_Edition, http://dbpedia.org/resource/Jhonen_Vasquez, http://dbpedia.org/resource/Joe_Simon, http://dbpedia.org/resource/Johann_Maria_Farina_gegenüber_dem_Jülichs-Platz, http://dbpedia.org/resource/John_Ball_(author), http://dbpedia.org/resource/John_Gilkey, http://dbpedia.org/resource/John_Hodgman, http://dbpedia.org/resource/John_Kluge, http://dbpedia.org/resource/Joint_Commission, http://dbpedia.org/resource/Jones_Day, http://dbpedia.org/resource/Jordan_Motor_Car_Company, http://dbpedia.org/resource/Joseph_A._Biedenharn, http://dbpedia.org/resource/KGAN, http://dbpedia.org/resource/KTM_Komuter, http://dbpedia.org/resource/KWWF, http://dbpedia.org/resource/Kaslo_and_Slocan_Railway, http://dbpedia.org/resource/Kayak.com, http://dbpedia.org/resource/Keep_case, http://dbpedia.org/resource/Kelsey's_Neighbourhood_Bar_&_Grill, http://dbpedia.org/resource/Kelsey_Grammer, http://dbpedia.org/resource/Keppel_Corporation, http://dbpedia.org/resource/Kettle_Valley_Railway, http://dbpedia.org/resource/Kinetic_energy_penetrator, http://dbpedia.org/resource/Kirtland_Safety_Society, http://dbpedia.org/resource/Kodak, http://dbpedia.org/resource/Korail, http://dbpedia.org/resource/Kosher_foods, http://dbpedia.org/resource/LDS_Family_Services, http://dbpedia.org/resource/LDS_Philanthropies, http://dbpedia.org/resource/LSI_Corporation, http://dbpedia.org/resource/L_Brands, http://dbpedia.org/resource/Lake_Torpedo_Boat, http://dbpedia.org/resource/Laketran, http://dbpedia.org/resource/Language_Spoken_at_Home_in_the_United_States_of_America, http://dbpedia.org/resource/Larry_Dolan, http://dbpedia.org/resource/Last_Spike_(Canadian_Pacific_Railway), http://dbpedia.org/resource/Latanoprost, http://dbpedia.org/resource/Lazarus_(department_store), http://dbpedia.org/resource/Leasehold_estate, http://dbpedia.org/resource/Lee_Cooper, http://dbpedia.org/resource/Legal_&_General, http://dbpedia.org/resource/Levi_H._Greenwood, http://dbpedia.org/resource/Levi_Strauss_&_Co., http://dbpedia.org/resource/Limited_partnership, http://dbpedia.org/resource/Lincoln_National_Corporation, http://dbpedia.org/resource/Linear_Tape-Open, http://dbpedia.org/resource/List_of_companies_of_Colombia, http://dbpedia.org/resource/Liverpool_Echo, http://dbpedia.org/resource/Lorain_County_Transit, http://dbpedia.org/resource/Louisville_Bats, http://dbpedia.org/resource/Luas, http://dbpedia.org/resource/Lufthansa_CityLine, http://dbpedia.org/resource/Luis_and_Clark, http://dbpedia.org/resource/Lynne_Koplitz, http://dbpedia.org/resource/M._DuMont_Schauberg, http://dbpedia.org/resource/MFA_Incorporated, http://dbpedia.org/resource/MFA_Oil, http://dbpedia.org/resource/Macworld, http://dbpedia.org/resource/Magic_Kingdom, http://dbpedia.org/resource/Mail_order, http://dbpedia.org/resource/Major_Matt_Mason, http://dbpedia.org/resource/Major_film_studio, http://dbpedia.org/resource/Mallinckrodt, http://dbpedia.org/resource/Mandriva, http://dbpedia.org/resource/Manufacturing_Commercial_Vehicles, http://dbpedia.org/resource/Marine_steam_engine, http://dbpedia.org/resource/MarketWatch, http://dbpedia.org/resource/Marlin_Firearms, http://dbpedia.org/resource/Martin_Sargent, http://dbpedia.org/resource/Massachusetts_Mutual_Life_Insurance_Company, http://dbpedia.org/resource/Matteo_Goffriller, http://dbpedia.org/resource/Mauser, http://dbpedia.org/resource/McCann_Erickson, http://dbpedia.org/resource/McCauley_Propeller_Systems, http://dbpedia.org/resource/Measles_Initiative, http://dbpedia.org/resource/Mechanical_watch, http://dbpedia.org/resource/MediaWorks_(publisher), http://dbpedia.org/resource/Medical_Mutual_of_Ohio, http://dbpedia.org/resource/Megabús, http://dbpedia.org/resource/Melbourne_International_Comedy_Festival, http://dbpedia.org/resource/Mentec_PDP-11, http://dbpedia.org/resource/Mercator_(retail), http://dbpedia.org/resource/Mesa_Arizona_Easter_Pageant, http://dbpedia.org/resource/Metra, http://dbpedia.org/resource/Metro_Trains_Melbourne, http://dbpedia.org/resource/Metrorail_(South_Africa), http://dbpedia.org/resource/Mettler_Toledo, http://dbpedia.org/resource/Mexican_Coke, http://dbpedia.org/resource/Milestone, http://dbpedia.org/resource/Milestone_Radio, http://dbpedia.org/resource/Minister_of_Inland_Revenue_(Canada), http://dbpedia.org/resource/Minneapolis,_Northfield_and_Southern_Railway, http://dbpedia.org/resource/Minoan_Lines, http://dbpedia.org/resource/Missouri_Employers_Mutual, http://dbpedia.org/resource/Molecular_Hamiltonian, http://dbpedia.org/resource/Monster_Beverage, http://dbpedia.org/resource/Monterey_Bay_Aquarium_Research_Institute, http://dbpedia.org/resource/Montreal,_Maine_and_Atlantic_Railway, http://dbpedia.org/resource/Moon_Design_Publications, http://dbpedia.org/resource/Moses_Cleaveland, http://dbpedia.org/resource/Mr._Potato_Head, http://dbpedia.org/resource/Multimark, http://dbpedia.org/resource/Municipal_broadband, http://dbpedia.org/resource/Must_See_TV, http://dbpedia.org/resource/Mäurer_&_Wirtz, http://dbpedia.org/resource/N-tv, http://dbpedia.org/resource/NACCO_Industries, http://dbpedia.org/resource/NRK, http://dbpedia.org/resource/Naguib_Sawiris, http://dbpedia.org/resource/Naspers, http://dbpedia.org/resource/National_Fire_Protection_Association, http://dbpedia.org/resource/Nationwide_Mutual_Insurance_Company, http://dbpedia.org/resource/Nature_Publishing_Group, http://dbpedia.org/resource/Naugahyde, http://dbpedia.org/resource/Naughty_Dog, http://dbpedia.org/resource/Naval_Strike_Missile, http://dbpedia.org/resource/Necklacing, http://dbpedia.org/resource/Nederlandse_Christelijke_Radio_Vereniging, http://dbpedia.org/resource/Neocolonialism, http://dbpedia.org/resource/NetJets, http://dbpedia.org/resource/New-York_Tribune, http://dbpedia.org/resource/New_Brunswick_Railway, http://dbpedia.org/resource/New_Canaan_Branch, http://dbpedia.org/resource/New_York,_New_Haven_and_Hartford_Railroad, http://dbpedia.org/resource/New_York_City_Ballet, http://dbpedia.org/resource/New_York_City_Economic_Development_Corporation, http://dbpedia.org/resource/New_York_Life_Insurance_Company, http://dbpedia.org/resource/New_York_State_Public_Authorities_Control_Board, http://dbpedia.org/resource/Newmark_Grubb_Knight_Frank, http://dbpedia.org/resource/News_Corp, http://dbpedia.org/resource/Nicolò_Gagliano, http://dbpedia.org/resource/Night_Shade_Books, http://dbpedia.org/resource/Ninja_Tune, http://dbpedia.org/resource/Nintendo_Power, http://dbpedia.org/resource/Norfolk_Naval_Shipyard, http://dbpedia.org/resource/Norfolk_Southern_Railway, http://dbpedia.org/resource/Northern_Territory_News, http://dbpedia.org/resource/Northwestel, http://dbpedia.org/resource/Nova_Scotia_Railway, http://dbpedia.org/resource/Nuns_on_the_Run, http://dbpedia.org/resource/Náchod, http://dbpedia.org/resource/O.F._Mossberg_&_Sons, http://dbpedia.org/resource/OCLC, http://dbpedia.org/resource/ORACLE_(teletext), http://dbpedia.org/resource/Observation_deck, http://dbpedia.org/resource/Obsidian_Entertainment, http://dbpedia.org/resource/Occidental_Petroleum, http://dbpedia.org/resource/Ohio_Company, http://dbpedia.org/resource/Oil_platform, http://dbpedia.org/resource/Old_Bay_Seasoning, http://dbpedia.org/resource/Ontario_and_Quebec_Railway, http://dbpedia.org/resource/Oolong, http://dbpedia.org/resource/OpenCola_(drink), http://dbpedia.org/resource/Open_Software_Foundation, http://dbpedia.org/resource/Optare, http://dbpedia.org/resource/Orbitz, http://dbpedia.org/resource/Ordnance_Survey_Ireland, http://dbpedia.org/resource/Ottawa_Citizen, http://dbpedia.org/resource/Owens_Corning, http://dbpedia.org/resource/Pacific_Comics, http://dbpedia.org/resource/Pacific_Northwest_Ballet, http://dbpedia.org/resource/Pacific_Western_Airlines, http://dbpedia.org/resource/Paddle_wheel, http://dbpedia.org/resource/Painfully_Midwestern_Records, http://dbpedia.org/resource/Pakistan_Post, http://dbpedia.org/resource/Pantone, http://dbpedia.org/resource/Paradox_Development_Studio, http://dbpedia.org/resource/Paradox_Entertainment, http://dbpedia.org/resource/Parents_(magazine), http://dbpedia.org/resource/Partial_charge, http://dbpedia.org/resource/Passenger_car_(rail), http://dbpedia.org/resource/Pemberton's_French_Wine_Coca, http://dbpedia.org/resource/Pershing_Square_Capital_Management, http://dbpedia.org/resource/Petronius_(oil_platform), http://dbpedia.org/resource/Philadelphia_76ers, http://dbpedia.org/resource/Philippe_Dauman, http://dbpedia.org/resource/Philippine_National_Railways, http://dbpedia.org/resource/Philips_CD-i, http://dbpedia.org/resource/Photon_energy, http://dbpedia.org/resource/Pick_n_Pay_Stores, http://dbpedia.org/resource/Pietra_Brewery, http://dbpedia.org/resource/Piper_Aircraft, http://dbpedia.org/resource/Pipex, http://dbpedia.org/resource/Pirate_FM, http://dbpedia.org/resource/Pitney_Bowes, http://dbpedia.org/resource/Pittsburgh,_Cincinnati,_Chicago_and_St._Louis_Railroad, http://dbpedia.org/resource/Pixar_Image_Computer, http://dbpedia.org/resource/Plasma_processing, http://dbpedia.org/resource/Polaris_Fashion_Place, http://dbpedia.org/resource/Popeye_(missile), http://dbpedia.org/resource/Populous_(company), http://dbpedia.org/resource/Porch, http://dbpedia.org/resource/Port_of_Cleveland, http://dbpedia.org/resource/Portage, http://dbpedia.org/resource/Porter_(beer), http://dbpedia.org/resource/Postal_worker, http://dbpedia.org/resource/Postmodern_philosophy, http://dbpedia.org/resource/Postobón, http://dbpedia.org/resource/Prague_Stock_Exchange, http://dbpedia.org/resource/Present_value, http://dbpedia.org/resource/Pressman_Toy_Corporation, http://dbpedia.org/resource/Pressure_swing_adsorption, http://dbpedia.org/resource/Primary_dealer, http://dbpedia.org/resource/Production_I.G, http://dbpedia.org/resource/Project_Management_Institute, http://dbpedia.org/resource/Project_engineering, http://dbpedia.org/resource/Public_Radio_International, http://dbpedia.org/resource/QR_code, http://dbpedia.org/resource/QUBE, http://dbpedia.org/resource/Qibla_Cola, http://dbpedia.org/resource/Quantum_information, http://dbpedia.org/resource/Quebec_Central_Railway, http://dbpedia.org/resource/Queensland_Rail, http://dbpedia.org/resource/Queensland_Rail_City_network, http://dbpedia.org/resource/QuickRing, http://dbpedia.org/resource/REWE_Group, http://dbpedia.org/resource/RNC_Media, http://dbpedia.org/resource/RPGamer, http://dbpedia.org/resource/Read-write_memory, http://dbpedia.org/resource/Reality_Simulations, http://dbpedia.org/resource/Realogy, http://dbpedia.org/resource/Rear-projection_television, http://dbpedia.org/resource/Recension, http://dbpedia.org/resource/Reconfigurable_computing, http://dbpedia.org/resource/Red_Hat_Enterprise_Linux, http://dbpedia.org/resource/Red_Hat_Linux, http://dbpedia.org/resource/Regenerative_agriculture, http://dbpedia.org/resource/Regnery_Publishing, http://dbpedia.org/resource/Reims_Aviation, http://dbpedia.org/resource/Removal_Units, http://dbpedia.org/resource/Renowned_Films, http://dbpedia.org/resource/Residue_field, http://dbpedia.org/resource/Rexair, http://dbpedia.org/resource/Reyes_rendering, http://dbpedia.org/resource/Richard_P._Gabriel, http://dbpedia.org/resource/Riddell, http://dbpedia.org/resource/Ripley's_Believe_It_or_Not!, http://dbpedia.org/resource/Road_case, http://dbpedia.org/resource/Roald_Hoffmann, http://dbpedia.org/resource/Robert_Dollar, http://dbpedia.org/resource/Rogers_Pass_(British_Columbia), http://dbpedia.org/resource/Roman_technology, http://dbpedia.org/resource/Rory_McGrath, http://dbpedia.org/resource/Rotring, http://dbpedia.org/resource/Rovos_Rail, http://dbpedia.org/resource/Royal_Gorge_Cross_Country_Ski_Resort, http://dbpedia.org/resource/Royal_Mail_Steam_Packet_Company, http://dbpedia.org/resource/Rubber_duck, http://dbpedia.org/resource/Rubing, http://dbpedia.org/resource/S4C_Authority, http://dbpedia.org/resource/SACO_(Colombia), http://dbpedia.org/resource/SCADTA, http://dbpedia.org/resource/SWALEC, http://dbpedia.org/resource/Sadler's_Wells_Theatre, http://dbpedia.org/resource/Sainsbury's, http://dbpedia.org/resource/Sanlam, http://dbpedia.org/resource/Sarah_Silverman, http://dbpedia.org/resource/Sasol, http://dbpedia.org/resource/Schwarze_Pumpe_power_station, http://dbpedia.org/resource/Science_Applications_International_Corporation, http://dbpedia.org/resource/Season_ticket, http://dbpedia.org/resource/Seiko, http://dbpedia.org/resource/Seoudi_Group, http://dbpedia.org/resource/Seoul_Metropolitan_Subway, http://dbpedia.org/resource/Sergio_Bonelli_Editore, http://dbpedia.org/resource/Shakespeare's_Pizza, http://dbpedia.org/resource/Sheerness, http://dbpedia.org/resource/Shelter_Insurance, http://dbpedia.org/resource/Shenyang_Aircraft_Corporation, http://dbpedia.org/resource/Shipping_container, http://dbpedia.org/resource/Shoprite_(South_Africa), http://dbpedia.org/resource/Shosholoza_Meyl, http://dbpedia.org/resource/Sicamous_(sternwheeler), http://dbpedia.org/resource/Sikorsky_Aircraft, http://dbpedia.org/resource/Silver_Key_Press, http://dbpedia.org/resource/Simultaneous_substitution, http://dbpedia.org/resource/Ski_boot, http://dbpedia.org/resource/Sky+_HD, http://dbpedia.org/resource/Sky_Go, http://dbpedia.org/resource/Slackers_CDs_and_Games, http://dbpedia.org/resource/Snake_case, http://dbpedia.org/resource/Society_for_the_Preservation_of_Beers_from_the_Wood, http://dbpedia.org/resource/Society_of_Independent_Brewers, http://dbpedia.org/resource/Société_de_transport_de_Montréal, http://dbpedia.org/resource/Socket_(telecommunications), http://dbpedia.org/resource/Soo_Line_Railroad, http://dbpedia.org/resource/Sorel_cement, http://dbpedia.org/resource/South_Wales_Echo, http://dbpedia.org/resource/South_Wales_Police, http://dbpedia.org/resource/Spanish_River_derailment, http://dbpedia.org/resource/Special_Administrative_Unit_of_Civil_Aeronautics, http://dbpedia.org/resource/Specific_energy, http://dbpedia.org/resource/Spectral_sensitivity, http://dbpedia.org/resource/Speranza_Motors, http://dbpedia.org/resource/Spillers_Records, http://dbpedia.org/resource/Spire_Healthcare, http://dbpedia.org/resource/Sportsvision, http://dbpedia.org/resource/Square_(company), http://dbpedia.org/resource/Square_Enix, http://dbpedia.org/resource/St._John_Publications, http://dbpedia.org/resource/St._Lawrence_and_Hudson_Railway, http://dbpedia.org/resource/Stadtwerke_Köln, http://dbpedia.org/resource/Stag_Arms, http://dbpedia.org/resource/Star_Reach, http://dbpedia.org/resource/State_Farm_Insurance, http://dbpedia.org/resource/Steam_car, http://dbpedia.org/resource/Steeplegate_Mall, http://dbpedia.org/resource/Stefano_Scarampella, http://dbpedia.org/resource/Steinway_Hall, http://dbpedia.org/resource/Stepan_Company, http://dbpedia.org/resource/Stereo-Pak, http://dbpedia.org/resource/Steven_Sykes_(artist), http://dbpedia.org/resource/Stouffer's, http://dbpedia.org/resource/Structural_coloration, http://dbpedia.org/resource/Sturm,_Ruger_&_Co., http://dbpedia.org/resource/Sun_Newspapers, http://dbpedia.org/resource/Supercritical_steam_generator, http://dbpedia.org/resource/Supersessionism, http://dbpedia.org/resource/Surface-conduction_electron-emitter_display, http://dbpedia.org/resource/Sustainable_forest_management, http://dbpedia.org/resource/Sydney_Trains, http://dbpedia.org/resource/System_3_(software_company), http://dbpedia.org/resource/T&T_Clark, http://dbpedia.org/resource/T._V._Soong, http://dbpedia.org/resource/T4_(Channel_4), http://dbpedia.org/resource/TDK_Mediactive, http://dbpedia.org/resource/TD_Ameritrade, http://dbpedia.org/resource/THQ, http://dbpedia.org/resource/TSR_(company), http://dbpedia.org/resource/TVCatchup, http://dbpedia.org/resource/Tabasco_sauce, http://dbpedia.org/resource/Takashi_Tokita, http://dbpedia.org/resource/Tall_Man_lettering, http://dbpedia.org/resource/Tandem_Verlag, http://dbpedia.org/resource/Taschen, http://dbpedia.org/resource/Teleconference, http://dbpedia.org/resource/Telefilm_Canada, http://dbpedia.org/resource/Telefunken, http://dbpedia.org/resource/Telelatino, http://dbpedia.org/resource/Television_South_West, http://dbpedia.org/resource/Television_in_the_United_States, http://dbpedia.org/resource/Temple_University_Press, http://dbpedia.org/resource/Ten_Cent_Beer_Night, http://dbpedia.org/resource/Terminal_Press, http://dbpedia.org/resource/Terminal_Tower, http://dbpedia.org/resource/Terra_Nova_Tel, http://dbpedia.org/resource/Textron_Marine_&_Land_Systems, http://dbpedia.org/resource/The_China_Navigation_Company, http://dbpedia.org/resource/The_Cubby_Bear, http://dbpedia.org/resource/The_Earth_Institute, http://dbpedia.org/resource/The_Folklore_Society, http://dbpedia.org/resource/The_Gaslight_Cafe, http://dbpedia.org/resource/The_Hahn_Company, http://dbpedia.org/resource/The_Learning_Company, http://dbpedia.org/resource/The_Mary_Tyler_Moore_Show, http://dbpedia.org/resource/The_Movie_Channel, http://dbpedia.org/resource/The_O2, http://dbpedia.org/resource/The_River_Café_(London), http://dbpedia.org/resource/The_Rodale_Institute, http://dbpedia.org/resource/The_Root_Glass_Company, http://dbpedia.org/resource/The_Salt_Lake_Tribune, http://dbpedia.org/resource/The_San_Francisco_Examiner, http://dbpedia.org/resource/The_Travelers_Companies, http://dbpedia.org/resource/This_TV, http://dbpedia.org/resource/Thomas_Vitale, http://dbpedia.org/resource/Thums_Up, http://dbpedia.org/resource/Tim_Hortons, http://dbpedia.org/resource/Times_Books, http://dbpedia.org/resource/Tiocem, http://dbpedia.org/resource/TipRanks, http://dbpedia.org/resource/Tommy_Boy_Records, http://dbpedia.org/resource/Tomorrow's_World, http://dbpedia.org/resource/Topiary, http://dbpedia.org/resource/Toronto,_Hamilton_and_Buffalo_Railway, http://dbpedia.org/resource/Toronto_Islands, http://dbpedia.org/resource/Toronto_Parks,_Forestry_and_Recreation_Division, http://dbpedia.org/resource/Tose_(company), http://dbpedia.org/resource/Tower_City_Center, http://dbpedia.org/resource/Townsville_Bulletin, http://dbpedia.org/resource/Toyota_Motorsport_GmbH, http://dbpedia.org/resource/Toyota_Racing, http://dbpedia.org/resource/Trade_name, http://dbpedia.org/resource/Trailways_Transportation_System, http://dbpedia.org/resource/Trans-Caribbean_pipeline, http://dbpedia.org/resource/Trans_World_Corporation, http://dbpedia.org/resource/Transdev_Auckland, http://dbpedia.org/resource/Tranz_Metro, http://dbpedia.org/resource/Tribune_Media, http://dbpedia.org/resource/Trifecta_Entertainment_&_Media, http://dbpedia.org/resource/Trinity_Broadcasting_Network, http://dbpedia.org/resource/Tyrone_Guthrie, http://dbpedia.org/resource/TÜV_Rheinland, http://dbpedia.org/resource/UBM_Canon, http://dbpedia.org/resource/Uclick, http://dbpedia.org/resource/Union_Station_(Toronto), http://dbpedia.org/resource/Union_of_Christendom, http://dbpedia.org/resource/UnitedHealth_Group, http://dbpedia.org/resource/Universal_Television, http://dbpedia.org/resource/University_Press_of_America, http://dbpedia.org/resource/University_Press_of_Kansas, http://dbpedia.org/resource/University_of_Missouri_Health_Care, http://dbpedia.org/resource/University_of_Utah_Press, http://dbpedia.org/resource/Univision, http://dbpedia.org/resource/Unix_System_Laboratories, http://dbpedia.org/resource/Upright_Citizens_Brigade, http://dbpedia.org/resource/Uraga,_Kanagawa, http://dbpedia.org/resource/Utah-Idaho_Sugar_Company, http://dbpedia.org/resource/VOX_(TV_channel), http://dbpedia.org/resource/Vaisala, http://dbpedia.org/resource/Varian_Data_Machines, http://dbpedia.org/resource/Veeraswamy, http://dbpedia.org/resource/Vehicular_cycling, http://dbpedia.org/resource/Veil_Nebula, http://dbpedia.org/resource/Velotype, http://dbpedia.org/resource/Vi-Co, http://dbpedia.org/resource/Viacom_(original), http://dbpedia.org/resource/Vida_e_Caffè, http://dbpedia.org/resource/Vince_Vaughn, http://dbpedia.org/resource/Vintage_Books, http://dbpedia.org/resource/Virgin_Interactive, http://dbpedia.org/resource/Virgin_Mobile, http://dbpedia.org/resource/VistaVision, http://dbpedia.org/resource/Vought, http://dbpedia.org/resource/W._R._Case_&_Sons_Cutlery_Co., http://dbpedia.org/resource/WBTV, http://dbpedia.org/resource/WCFL_(AM), http://dbpedia.org/resource/WCIU-TV, http://dbpedia.org/resource/WCNC-TV, http://dbpedia.org/resource/WFLD, http://dbpedia.org/resource/WGN_(AM), http://dbpedia.org/resource/WGN_Sports, http://dbpedia.org/resource/WHK_(AM), http://dbpedia.org/resource/WIFR, http://dbpedia.org/resource/WJZY, http://dbpedia.org/resource/WKRC-TV, http://dbpedia.org/resource/WLS_(AM), http://dbpedia.org/resource/WMAQ_(AM), http://dbpedia.org/resource/WMBD-TV, http://dbpedia.org/resource/WMMS, http://dbpedia.org/resource/WOIO, http://dbpedia.org/resource/WPWR-TV, http://dbpedia.org/resource/WRFD, http://dbpedia.org/resource/WSTR-TV, http://dbpedia.org/resource/WSYX, http://dbpedia.org/resource/WTTE, http://dbpedia.org/resource/WUAB, http://dbpedia.org/resource/WWHO, http://dbpedia.org/resource/WZMH_Architects, http://dbpedia.org/resource/Waltham_Watch_Company, http://dbpedia.org/resource/Warren_Ellis, http://dbpedia.org/resource/Welcome_to_Convenience_Store, http://dbpedia.org/resource/Welsh_National_Opera, http://dbpedia.org/resource/Wendy's, http://dbpedia.org/resource/West_Cornwall_Railway, http://dbpedia.org/resource/Western_Mail_(Wales), http://dbpedia.org/resource/Western_Morning_News, http://dbpedia.org/resource/Western_People, http://dbpedia.org/resource/Westfield_London, http://dbpedia.org/resource/Westminster_John_Knox, http://dbpedia.org/resource/Westward_Television, http://dbpedia.org/resource/Weta_Workshop, http://dbpedia.org/resource/Whale_watching, http://dbpedia.org/resource/White_Castle_(restaurant), http://dbpedia.org/resource/White_Coke, http://dbpedia.org/resource/White_Motor_Company, http://dbpedia.org/resource/White_Rock_Beverages, http://dbpedia.org/resource/Wildlife_conservation, http://dbpedia.org/resource/Wilkinson_Sword, http://dbpedia.org/resource/William_Aspdin, http://dbpedia.org/resource/Winchester_Repeating_Arms_Company, http://dbpedia.org/resource/Windrow_composting, http://dbpedia.org/resource/Winery, http://dbpedia.org/resource/Winton_Motor_Carriage_Company, http://dbpedia.org/resource/Wirtualna_Polska, http://dbpedia.org/resource/Wisconsin_Central_Ltd., http://dbpedia.org/resource/Woolworths_(South_Africa), http://dbpedia.org/resource/Worcestershire_sauce, http://dbpedia.org/resource/WorldWideWeb, http://dbpedia.org/resource/World_Fuel_Services, http://dbpedia.org/resource/XM_Satellite_Radio, http://dbpedia.org/resource/Xeon, http://dbpedia.org/resource/Xi_Graphics, http://dbpedia.org/resource/Yissum_Research_Development_Company_of_the_Hebrew_University, http://dbpedia.org/resource/Yoshimoto_Cube, http://dbpedia.org/resource/Yum_cha, http://dbpedia.org/resource/Zariski_tangent_space, http://dbpedia.org/resource/Zatarain's, http://dbpedia.org/resource/Zondervan, http://dbpedia.org/resource/Éditions_du_Seuil, http://dbpedia.org/resource/1998–99_Ecuador_banking_crisis, http://dbpedia.org/resource/19_Entertainment, http://dbpedia.org/resource/19_Kids_and_Counting, http://dbpedia.org/resource/2nd_Avenue_(television_channel), http://dbpedia.org/resource/4J_Studios, http://dbpedia.org/resource/801_Grand, http://dbpedia.org/resource/A._G._Becker_&_Co., http://dbpedia.org/resource/ACE_Limited, http://dbpedia.org/resource/AMC_(TV_channel), http://dbpedia.org/resource/ASB_Bank, http://dbpedia.org/resource/AXE_telephone_exchange, http://dbpedia.org/resource/AXN, http://dbpedia.org/resource/Aastra_Technologies, http://dbpedia.org/resource/Acclaim_Entertainment, http://dbpedia.org/resource/Ace_Books, http://dbpedia.org/resource/Adaptationism, http://dbpedia.org/resource/Adaptive_Vehicle_Make, http://dbpedia.org/resource/Addictive_Games, http://dbpedia.org/resource/Adolfo_Orsi, http://dbpedia.org/resource/Aero_Research_Limited, http://dbpedia.org/resource/Agriculture_in_the_United_States, http://dbpedia.org/resource/Aksumite_currency, http://dbpedia.org/resource/Alexander_Nubia_International, http://dbpedia.org/resource/Algonquin_Hotel, http://dbpedia.org/resource/All-American_Publications, http://dbpedia.org/resource/Alliance_Trust, http://dbpedia.org/resource/Allied_Insurance, http://dbpedia.org/resource/Ally_Financial, http://dbpedia.org/resource/Alpari_Group, http://dbpedia.org/resource/AlphaSmart, http://dbpedia.org/resource/Alpha_Software, http://dbpedia.org/resource/Alternating_Gradient_Synchrotron, http://dbpedia.org/resource/Amalgam_Comics, http://dbpedia.org/resource/Amaranth_Advisors, http://dbpedia.org/resource/Ambassador_Theatre_Group, http://dbpedia.org/resource/American_Public_Television, http://dbpedia.org/resource/American_Sound_Studio, http://dbpedia.org/resource/American_Water_(company), http://dbpedia.org/resource/Ampal-American_Israel_Corporation, http://dbpedia.org/resource/Amylin_Pharmaceuticals, http://dbpedia.org/resource/Analytics, http://dbpedia.org/resource/Anchor_Bay_Entertainment, http://dbpedia.org/resource/Andamiro, http://dbpedia.org/resource/Andersen_Corporation, http://dbpedia.org/resource/Andrew_Fastow, http://dbpedia.org/resource/AngloGold_Ashanti]... [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 152, P: 0.606, R: 0.643, macro F1: 0.605, macro F1*: 0.624 -[INFO] Current time: 8509 ms, average: 2892.08 ms/ques, entity detection: 384.70 ms/ques (linking tool: 295.52 ms/ques), relation detection: 856.05 ms/ques (similarity: 675.70 ms/ques), EDG generation: 84.36 ms/ques +[INFO] Cumulative metrics, sample: 152, P: 0.606, R: 0.643, macro F1: 0.624 +[INFO] Current question consumed time: 8013 ms, average: 2752.78 ms/ques, entity detection: 370.20 ms/ques (linking tool: 286.27 ms/ques), relation detection: 838.52 ms/ques (similarity: 672.96 ms/ques), EDG generation: 83.24 ms/ques [INFO] Entity Linking P: 0.7894736842105263, R: 0.756578947368421, micro F1: 0.767543859649123, macro F1: 0.7726763717805151 [INFO] Relation Linking P: 0.6304824561403508, R: 0.6217105263157895, micro F1: 0.6212719298245615, macro F1: 0.6260657664300856 [INFO] Type Linking P: 0.4934210526315789, R: 0.4934210526315789, micro F1: 0.4934210526315789, macro F1: 0.4934210526315789 @@ -2136,8 +2136,8 @@ Question 152: Who was the coach of marquette Golden Eagles men's basketball team [INFO] Golden Answer: [http://dbpedia.org/resource/Buzz_Williams] [INFO] Predicted Answer: [http://dbpedia.org/resource/Steve_Wojciechowski] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 153, P: 0.602, R: 0.639, macro F1: 0.601, macro F1*: 0.620 -[INFO] Current time: 1193 ms, average: 2880.97 ms/ques, entity detection: 383.40 ms/ques (linking tool: 294.44 ms/ques), relation detection: 853.48 ms/ques (similarity: 673.13 ms/ques), EDG generation: 83.97 ms/ques +[INFO] Cumulative metrics, sample: 153, P: 0.602, R: 0.639, macro F1: 0.620 +[INFO] Current question consumed time: 1160 ms, average: 2742.37 ms/ques, entity detection: 368.99 ms/ques (linking tool: 285.22 ms/ques), relation detection: 836.12 ms/ques (similarity: 670.53 ms/ques), EDG generation: 82.84 ms/ques [INFO] Entity Linking P: 0.7843137254901961, R: 0.7516339869281046, micro F1: 0.7625272331154687, macro F1: 0.7676261994159366 [INFO] Relation Linking P: 0.6328976034858388, R: 0.6241830065359477, micro F1: 0.6237472766884533, macro F1: 0.628510098435677 [INFO] Type Linking P: 0.49673202614379086, R: 0.49673202614379086, micro F1: 0.49673202614379086, macro F1: 0.49673202614379086 @@ -2150,8 +2150,8 @@ Question 153: Miluk and Cayuse are still spoken in which part of the world? [INFO] Golden Answer: [http://dbpedia.org/resource/Oregon] [INFO] Predicted Answer: [http://dbpedia.org/resource/Oregon] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 154, P: 0.605, R: 0.641, macro F1: 0.604, macro F1*: 0.622 -[INFO] Current time: 2699 ms, average: 2879.79 ms/ques, entity detection: 381.70 ms/ques (linking tool: 292.94 ms/ques), relation detection: 854.48 ms/ques (similarity: 676.51 ms/ques), EDG generation: 83.81 ms/ques +[INFO] Cumulative metrics, sample: 154, P: 0.605, R: 0.641, macro F1: 0.622 +[INFO] Current question consumed time: 2742 ms, average: 2742.36 ms/ques, entity detection: 367.45 ms/ques (linking tool: 283.85 ms/ques), relation detection: 837.24 ms/ques (similarity: 673.94 ms/ques), EDG generation: 82.67 ms/ques [INFO] Entity Linking P: 0.7824675324675324, R: 0.75, micro F1: 0.7608225108225111, macro F1: 0.7658898305084747 [INFO] Relation Linking P: 0.6287878787878788, R: 0.6201298701298701, micro F1: 0.6196969696969699, macro F1: 0.6244288640302506 [INFO] Type Linking P: 0.5, R: 0.5, micro F1: 0.5, macro F1: 0.5 @@ -2164,8 +2164,8 @@ Question 154: Which governer of Winston bryant is also the president of Carl Sto [INFO] Golden Answer: [http://dbpedia.org/resource/Bill_Clinton] [INFO] Predicted Answer: [http://dbpedia.org/resource/Bill_Clinton] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 155, P: 0.607, R: 0.643, macro F1: 0.606, macro F1*: 0.625 -[INFO] Current time: 2457 ms, average: 2877.06 ms/ques, entity detection: 382.35 ms/ques (linking tool: 292.68 ms/ques), relation detection: 853.85 ms/ques (similarity: 675.30 ms/ques), EDG generation: 83.89 ms/ques +[INFO] Cumulative metrics, sample: 155, P: 0.607, R: 0.643, macro F1: 0.625 +[INFO] Current question consumed time: 2337 ms, average: 2739.75 ms/ques, entity detection: 367.90 ms/ques (linking tool: 283.43 ms/ques), relation detection: 836.93 ms/ques (similarity: 672.96 ms/ques), EDG generation: 82.75 ms/ques [INFO] Entity Linking P: 0.7838709677419354, R: 0.7516129032258064, micro F1: 0.7623655913978497, macro F1: 0.7674030902683653 [INFO] Relation Linking P: 0.6311827956989247, R: 0.6225806451612903, micro F1: 0.6221505376344088, macro F1: 0.6268522104797211 [INFO] Type Linking P: 0.5032258064516129, R: 0.5032258064516129, micro F1: 0.5032258064516129, macro F1: 0.5032258064516129 @@ -2178,8 +2178,8 @@ Question 155: List the sports of PATTS College of Aeronautics which are of inter [INFO] Golden Answer: [http://dbpedia.org/resource/Chess] [INFO] Predicted Answer: [] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 156, P: 0.603, R: 0.639, macro F1: 0.602, macro F1*: 0.621 -[INFO] Current time: 1940 ms, average: 2871.06 ms/ques, entity detection: 381.99 ms/ques (linking tool: 292.21 ms/ques), relation detection: 851.04 ms/ques (similarity: 672.53 ms/ques), EDG generation: 83.54 ms/ques +[INFO] Cumulative metrics, sample: 156, P: 0.603, R: 0.639, macro F1: 0.621 +[INFO] Current question consumed time: 1814 ms, average: 2733.81 ms/ques, entity detection: 367.50 ms/ques (linking tool: 282.93 ms/ques), relation detection: 834.00 ms/ques (similarity: 670.18 ms/ques), EDG generation: 82.40 ms/ques [INFO] Entity Linking P: 0.7788461538461539, R: 0.7467948717948718, micro F1: 0.7574786324786327, macro F1: 0.7624838396897221 [INFO] Relation Linking P: 0.6271367521367521, R: 0.6185897435897436, micro F1: 0.6181623931623933, macro F1: 0.6228339270792101 [INFO] Type Linking P: 0.5064102564102564, R: 0.5064102564102564, micro F1: 0.5064102564102564, macro F1: 0.5064102564102564 @@ -2192,8 +2192,8 @@ Question 156: What is the home stadium of the club coached by Justine Omojong? [INFO] Golden Answer: [http://dbpedia.org/resource/Kenya, http://dbpedia.org/resource/Nairobi, http://dbpedia.org/resource/Moi_International_Sports_Centre] [INFO] Predicted Answer: [] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 157, P: 0.599, R: 0.635, macro F1: 0.598, macro F1*: 0.617 -[INFO] Current time: 1414 ms, average: 2861.78 ms/ques, entity detection: 380.59 ms/ques (linking tool: 290.88 ms/ques), relation detection: 848.41 ms/ques (similarity: 671.38 ms/ques), EDG generation: 83.69 ms/ques +[INFO] Cumulative metrics, sample: 157, P: 0.599, R: 0.635, macro F1: 0.617 +[INFO] Current question consumed time: 1417 ms, average: 2725.43 ms/ques, entity detection: 366.10 ms/ques (linking tool: 281.64 ms/ques), relation detection: 831.50 ms/ques (similarity: 669.06 ms/ques), EDG generation: 82.60 ms/ques [INFO] Entity Linking P: 0.7738853503184714, R: 0.7420382165605095, micro F1: 0.7526539278131636, macro F1: 0.7576272547235454 [INFO] Relation Linking P: 0.6231422505307855, R: 0.6146496815286624, micro F1: 0.6142250530785563, macro F1: 0.6188668320022724 [INFO] Type Linking P: 0.5031847133757962, R: 0.5031847133757962, micro F1: 0.5031847133757962, macro F1: 0.5031847133757962 @@ -2206,8 +2206,8 @@ Question 157: Which are the predecessors of the cars which are similar to the Ol [INFO] Golden Answer: [http://dbpedia.org/resource/Pontiac_Star_Chief, http://dbpedia.org/resource/Pontiac_Torpedo, http://dbpedia.org/resource/Pontiac_Bonneville, http://dbpedia.org/resource/Pontiac_Catalina, http://dbpedia.org/resource/Chevrolet_Lumina, http://dbpedia.org/resource/Mexico, http://dbpedia.org/resource/North_America, http://dbpedia.org/resource/Pontiac_G8, http://dbpedia.org/resource/Chevrolet_Impala, http://dbpedia.org/resource/Buick_Series_40, http://dbpedia.org/resource/Buick_Century, http://dbpedia.org/resource/Buick_Special, http://dbpedia.org/resource/Buick_Invicta, http://dbpedia.org/resource/Cadillac_Series_62, http://dbpedia.org/resource/Chevrolet_Deluxe, http://dbpedia.org/resource/Pontiac_Chieftain, http://dbpedia.org/resource/Oldsmobile_88] [INFO] Predicted Answer: [http://dbpedia.org/resource/Pontiac_G8, http://dbpedia.org/resource/Pontiac_GTO, http://dbpedia.org/resource/Pontiac_Grand_Safari, http://dbpedia.org/resource/Pontiac_Grand_Ville, http://dbpedia.org/resource/Pontiac_Parisienne, http://dbpedia.org/resource/Pontiac_Catalina, http://dbpedia.org/resource/Pontiac_Star_Chief, http://dbpedia.org/resource/Pontiac_Phoenix, http://dbpedia.org/resource/Oldsmobile_88, http://dbpedia.org/resource/Chevrolet_Parkwood, http://dbpedia.org/resource/Chevrolet_Lumina, http://dbpedia.org/resource/Buick_Apollo, http://dbpedia.org/resource/Buick_LeSabre, http://dbpedia.org/resource/Buick_Park_Avenue, http://dbpedia.org/resource/Buick_Wildcat, http://dbpedia.org/resource/Oldsmobile_Aurora, http://dbpedia.org/resource/Chevrolet_Impala_(fourth_generation), http://dbpedia.org/resource/Chevrolet_Camaro, http://dbpedia.org/resource/Chevrolet_Caprice, http://dbpedia.org/resource/Chevrolet_Caprice__Sixth_generation__1, http://dbpedia.org/resource/Buick_Centurion, http://dbpedia.org/resource/Cadillac_DTS, http://dbpedia.org/resource/Pontiac_Torpedo, http://dbpedia.org/resource/Pontiac_Bonneville, http://dbpedia.org/resource/Holden_Caprice, http://dbpedia.org/resource/Buick_Electra, http://dbpedia.org/resource/Mexico, http://dbpedia.org/resource/North_America, http://dbpedia.org/resource/Chevrolet_Impala, http://dbpedia.org/resource/Buick_Series_40, http://dbpedia.org/resource/Buick_Roadmaster, http://dbpedia.org/resource/Buick_Super, http://dbpedia.org/resource/Buick_Century, http://dbpedia.org/resource/Buick_Special, http://dbpedia.org/resource/Cadillac_Series_60, http://dbpedia.org/resource/Pontiac_Chieftain, http://dbpedia.org/resource/Buick_Invicta, http://dbpedia.org/resource/Cadillac_Series_62, http://dbpedia.org/resource/Chevrolet_Deluxe, http://dbpedia.org/resource/Pontiac_6000, http://dbpedia.org/resource/Pontiac_Tempest, 1957, 1958, 1968, Oldsmobile Custom Cruiser 90@en, Ninth-generation Impala@en, Oldsmobile 88 Fiesta@en] [INFO] P: 0.3617021276595745, R: 1.0, F: 0.53125 -[INFO] Cumulative metrics, sample: 158, P: 0.598, R: 0.638, macro F1: 0.598, macro F1*: 0.617 -[INFO] Current time: 1944 ms, average: 2855.97 ms/ques, entity detection: 379.50 ms/ques (linking tool: 289.92 ms/ques), relation detection: 848.14 ms/ques (similarity: 670.63 ms/ques), EDG generation: 83.37 ms/ques +[INFO] Cumulative metrics, sample: 158, P: 0.598, R: 0.638, macro F1: 0.617 +[INFO] Current question consumed time: 1906 ms, average: 2720.24 ms/ques, entity detection: 365.08 ms/ques (linking tool: 280.68 ms/ques), relation detection: 831.25 ms/ques (similarity: 668.29 ms/ques), EDG generation: 82.28 ms/ques [INFO] Entity Linking P: 0.7753164556962026, R: 0.7436708860759493, micro F1: 0.7542194092827006, macro F1: 0.7591640295358648 [INFO] Relation Linking P: 0.6255274261603375, R: 0.6170886075949367, micro F1: 0.6166666666666668, macro F1: 0.6212793621456664 [INFO] Type Linking P: 0.5063291139240507, R: 0.5063291139240507, micro F1: 0.5063291139240507, macro F1: 0.5063291139240507 @@ -2220,8 +2220,8 @@ Question 158: Name a queen whose parent is Ptolemy XII Auletes and consort is Pt [INFO] Golden Answer: [http://dbpedia.org/resource/Cleopatra] [INFO] Predicted Answer: [Arsinoe IV@en, Cleopatra VII Philopator@en, q:rw-W:p-d:r:t*H8-nTr-t:H8-R7:t-z:N36@en, q:rw-i-wA-p-A-d:r-t-A-H8@en, wr*t:r-t:W:t-A53-n:X2*t:z@en, wr:r-nb-nfr-nfr-nfr-H2:x-O22@en] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 159, P: 0.594, R: 0.634, macro F1: 0.594, macro F1*: 0.613 -[INFO] Current time: 3396 ms, average: 2859.36 ms/ques, entity detection: 380.27 ms/ques (linking tool: 289.79 ms/ques), relation detection: 847.94 ms/ques (similarity: 670.96 ms/ques), EDG generation: 89.45 ms/ques +[INFO] Cumulative metrics, sample: 159, P: 0.594, R: 0.634, macro F1: 0.613 +[INFO] Current question consumed time: 3368 ms, average: 2724.31 ms/ques, entity detection: 365.94 ms/ques (linking tool: 280.62 ms/ques), relation detection: 831.28 ms/ques (similarity: 668.62 ms/ques), EDG generation: 88.35 ms/ques [INFO] Entity Linking P: 0.7767295597484277, R: 0.7421383647798742, micro F1: 0.7536687631027256, macro F1: 0.7590400666692708 [INFO] Relation Linking P: 0.6247379454926625, R: 0.6163522012578616, micro F1: 0.6159329140461217, macro F1: 0.6205167431582527 [INFO] Type Linking P: 0.5031446540880503, R: 0.5031446540880503, micro F1: 0.5031446540880503, macro F1: 0.5031446540880503 @@ -2234,8 +2234,8 @@ Question 159: Which show's theme music was composed by Primus and voiced by Isaa [INFO] Golden Answer: [http://dbpedia.org/resource/South_Park] [INFO] Predicted Answer: [http://dbpedia.org/resource/South_Park] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 160, P: 0.597, R: 0.636, macro F1: 0.597, macro F1*: 0.616 -[INFO] Current time: 4843 ms, average: 2871.76 ms/ques, entity detection: 380.26 ms/ques (linking tool: 289.43 ms/ques), relation detection: 853.58 ms/ques (similarity: 680.16 ms/ques), EDG generation: 89.28 ms/ques +[INFO] Cumulative metrics, sample: 160, P: 0.597, R: 0.636, macro F1: 0.616 +[INFO] Current question consumed time: 4679 ms, average: 2736.53 ms/ques, entity detection: 366.07 ms/ques (linking tool: 280.35 ms/ques), relation detection: 837.36 ms/ques (similarity: 677.84 ms/ques), EDG generation: 88.18 ms/ques [INFO] Entity Linking P: 0.778125, R: 0.74375, micro F1: 0.7552083333333336, macro F1: 0.7605492813141683 [INFO] Relation Linking P: 0.6208333333333333, R: 0.6125, micro F1: 0.6120833333333334, macro F1: 0.6166385135135135 [INFO] Type Linking P: 0.5, R: 0.5, micro F1: 0.5, macro F1: 0.5 @@ -2248,8 +2248,8 @@ Question 160: Who is the chancellor of the university whose campus is in Woodlan [INFO] Golden Answer: [http://dbpedia.org/resource/Dawn_French] [INFO] Predicted Answer: [] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 161, P: 0.593, R: 0.632, macro F1: 0.593, macro F1*: 0.612 -[INFO] Current time: 5752 ms, average: 2889.65 ms/ques, entity detection: 379.28 ms/ques (linking tool: 288.70 ms/ques), relation detection: 863.52 ms/ques (similarity: 692.18 ms/ques), EDG generation: 88.98 ms/ques +[INFO] Cumulative metrics, sample: 161, P: 0.593, R: 0.632, macro F1: 0.612 +[INFO] Current question consumed time: 5748 ms, average: 2755.24 ms/ques, entity detection: 365.21 ms/ques (linking tool: 279.68 ms/ques), relation detection: 847.48 ms/ques (similarity: 689.93 ms/ques), EDG generation: 87.88 ms/ques [INFO] Entity Linking P: 0.7732919254658385, R: 0.7391304347826086, micro F1: 0.7505175983436856, macro F1: 0.7558253727345774 [INFO] Relation Linking P: 0.6169772256728778, R: 0.6086956521739131, micro F1: 0.608281573498965, macro F1: 0.6128084606345476 [INFO] Type Linking P: 0.4968944099378882, R: 0.4968944099378882, micro F1: 0.4968944099378882, macro F1: 0.4968944099378882 @@ -2262,8 +2262,8 @@ Question 161: Name the mountain whose parent peak is located in Mexico. [INFO] Golden Answer: [http://dbpedia.org/resource/Mount_Whitney, http://dbpedia.org/resource/Nevado_de_Toluca] [INFO] Predicted Answer: [http://dbpedia.org/resource/Popocatépetl, http://dbpedia.org/resource/Mount_Tlaloc, http://dbpedia.org/resource/Sierra_Negra, http://dbpedia.org/resource/Peña_de_Bernal, http://dbpedia.org/resource/Tequila_Volcano, http://dbpedia.org/resource/Boot_Heel_volcanic_field, http://dbpedia.org/resource/Parícutin, http://dbpedia.org/resource/Iztaccihuatl, http://dbpedia.org/resource/Volcán_Tacaná, http://dbpedia.org/resource/San_Benedicto_Island, http://dbpedia.org/resource/El_Jorullo, http://dbpedia.org/resource/Samalayuca_Dune_Fields, http://dbpedia.org/resource/Cofre_de_Perote, http://dbpedia.org/resource/Picacho_del_Diablo, http://dbpedia.org/resource/Nevado_de_Toluca, http://dbpedia.org/resource/Aguajito, http://dbpedia.org/resource/Cerro_Potosí, http://dbpedia.org/resource/Cerro_Mohinora, http://dbpedia.org/resource/Michoacán–Guanajuato_volcanic_field, http://dbpedia.org/resource/San_Martin_Tuxtla, http://dbpedia.org/resource/Cerro_de_la_Loma_Larga, http://dbpedia.org/resource/Cerro_de_la_Silla, http://dbpedia.org/resource/Cerro_de_las_Mitras, http://dbpedia.org/resource/Cerro_del_Topo_Chico, http://dbpedia.org/resource/Potrillo_volcanic_field, http://dbpedia.org/resource/Volcán_Bárcena, http://dbpedia.org/resource/La_Malintzin] [INFO] P: 0.037037037037037035, R: 0.5, F: 0.06896551724137931 -[INFO] Cumulative metrics, sample: 162, P: 0.590, R: 0.631, macro F1: 0.590, macro F1*: 0.610 -[INFO] Current time: 9753 ms, average: 2932.02 ms/ques, entity detection: 390.93 ms/ques (linking tool: 300.30 ms/ques), relation detection: 877.84 ms/ques (similarity: 705.96 ms/ques), EDG generation: 88.56 ms/ques +[INFO] Cumulative metrics, sample: 162, P: 0.590, R: 0.631, macro F1: 0.610 +[INFO] Current question consumed time: 9741 ms, average: 2798.36 ms/ques, entity detection: 376.74 ms/ques (linking tool: 291.19 ms/ques), relation detection: 862.31 ms/ques (similarity: 703.59 ms/ques), EDG generation: 87.44 ms/ques [INFO] Entity Linking P: 0.7685185185185185, R: 0.7345679012345679, micro F1: 0.7458847736625517, macro F1: 0.7511597840139934 [INFO] Relation Linking P: 0.6193415637860082, R: 0.6080246913580247, micro F1: 0.6086419753086422, macro F1: 0.6136309542288867 [INFO] Type Linking P: 0.49382716049382713, R: 0.49382716049382713, micro F1: 0.49382716049382713, macro F1: 0.49382716049382713 @@ -2276,8 +2276,8 @@ Question 162: List the ingredient of Blodpalt ? [INFO] Golden Answer: [http://dbpedia.org/resource/Flour, http://dbpedia.org/resource/Potato, http://dbpedia.org/resource/Blood] [INFO] Predicted Answer: [http://dbpedia.org/resource/Flour, http://dbpedia.org/resource/Potato, http://dbpedia.org/resource/Blood] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 163, P: 0.592, R: 0.633, macro F1: 0.592, macro F1*: 0.612 -[INFO] Current time: 339 ms, average: 2916.11 ms/ques, entity detection: 388.74 ms/ques (linking tool: 298.61 ms/ques), relation detection: 872.87 ms/ques (similarity: 701.90 ms/ques), EDG generation: 89.02 ms/ques +[INFO] Cumulative metrics, sample: 163, P: 0.592, R: 0.633, macro F1: 0.612 +[INFO] Current question consumed time: 191 ms, average: 2782.36 ms/ques, entity detection: 374.64 ms/ques (linking tool: 289.55 ms/ques), relation detection: 857.48 ms/ques (similarity: 699.60 ms/ques), EDG generation: 86.95 ms/ques [INFO] Entity Linking P: 0.7699386503067485, R: 0.7361963190184049, micro F1: 0.7474437627811863, macro F1: 0.7526895155748253 [INFO] Relation Linking P: 0.6216768916155418, R: 0.6104294478527608, micro F1: 0.6110429447852762, macro F1: 0.6160018328539062 [INFO] Type Linking P: 0.49693251533742333, R: 0.49693251533742333, micro F1: 0.49693251533742333, macro F1: 0.49693251533742333 @@ -2290,8 +2290,8 @@ Question 163: List the developers who has contributed to both GNOWSYS and GNU Te [INFO] Golden Answer: [http://dbpedia.org/resource/GNU] [INFO] Predicted Answer: [http://dbpedia.org/resource/GNU] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 164, P: 0.595, R: 0.636, macro F1: 0.595, macro F1*: 0.614 -[INFO] Current time: 1407 ms, average: 2906.91 ms/ques, entity detection: 387.93 ms/ques (linking tool: 297.96 ms/ques), relation detection: 869.15 ms/ques (similarity: 699.24 ms/ques), EDG generation: 88.88 ms/ques +[INFO] Cumulative metrics, sample: 164, P: 0.595, R: 0.636, macro F1: 0.614 +[INFO] Current question consumed time: 1351 ms, average: 2773.63 ms/ques, entity detection: 373.73 ms/ques (linking tool: 288.81 ms/ques), relation detection: 853.90 ms/ques (similarity: 697.03 ms/ques), EDG generation: 86.80 ms/ques [INFO] Entity Linking P: 0.7713414634146342, R: 0.7378048780487805, micro F1: 0.7489837398373986, macro F1: 0.7542005420054201 [INFO] Relation Linking P: 0.6239837398373983, R: 0.6128048780487805, micro F1: 0.6134146341463416, macro F1: 0.6183437882036995 [INFO] Type Linking P: 0.5, R: 0.5, micro F1: 0.5, macro F1: 0.5 @@ -2304,8 +2304,8 @@ Question 164: List the served region of the organisations of Trade association. [INFO] Golden Answer: [http://dbpedia.org/resource/New_England, http://dbpedia.org/resource/Kingdom_of_Portugal, http://dbpedia.org/resource/Trade_route, http://dbpedia.org/resource/Canada, http://dbpedia.org/resource/United_States, http://dbpedia.org/resource/North_America, http://dbpedia.org/resource/São_Paulo_(state), http://dbpedia.org/resource/West_Africa, http://dbpedia.org/resource/United_Kingdom, http://dbpedia.org/resource/Northern_Ireland, http://dbpedia.org/resource/Europe, http://dbpedia.org/resource/Tennessee, http://dbpedia.org/resource/Newfoundland_and_Labrador, http://dbpedia.org/resource/European_Union, http://dbpedia.org/resource/England, http://dbpedia.org/resource/Wales] [INFO] Predicted Answer: [] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 165, P: 0.591, R: 0.632, macro F1: 0.591, macro F1*: 0.611 -[INFO] Current time: 1866 ms, average: 2900.60 ms/ques, entity detection: 386.45 ms/ques (linking tool: 296.68 ms/ques), relation detection: 869.29 ms/ques (similarity: 698.14 ms/ques), EDG generation: 88.46 ms/ques +[INFO] Cumulative metrics, sample: 165, P: 0.591, R: 0.632, macro F1: 0.611 +[INFO] Current question consumed time: 1832 ms, average: 2767.93 ms/ques, entity detection: 372.35 ms/ques (linking tool: 287.59 ms/ques), relation detection: 853.81 ms/ques (similarity: 695.92 ms/ques), EDG generation: 86.38 ms/ques [INFO] Entity Linking P: 0.7666666666666667, R: 0.7333333333333333, micro F1: 0.7444444444444447, macro F1: 0.7496296296296295 [INFO] Relation Linking P: 0.6202020202020202, R: 0.6090909090909091, micro F1: 0.6096969696969698, macro F1: 0.6145962500933742 [INFO] Type Linking P: 0.49696969696969695, R: 0.49696969696969695, micro F1: 0.49696969696969695, macro F1: 0.49696969696969695 @@ -2318,8 +2318,8 @@ Question 165: Is Hyundai Lavita assembled in Malaysia? [INFO] Golden Answer: [true] [INFO] Predicted Answer: [true] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 166, P: 0.593, R: 0.634, macro F1: 0.594, macro F1*: 0.613 -[INFO] Current time: 6350 ms, average: 2921.38 ms/ques, entity detection: 393.38 ms/ques (linking tool: 301.89 ms/ques), relation detection: 885.84 ms/ques (similarity: 706.31 ms/ques), EDG generation: 87.98 ms/ques +[INFO] Cumulative metrics, sample: 166, P: 0.593, R: 0.634, macro F1: 0.613 +[INFO] Current question consumed time: 6293 ms, average: 2789.17 ms/ques, entity detection: 378.94 ms/ques (linking tool: 293.20 ms/ques), relation detection: 870.70 ms/ques (similarity: 704.05 ms/ques), EDG generation: 85.92 ms/ques [INFO] Entity Linking P: 0.7650602409638554, R: 0.7319277108433735, micro F1: 0.742971887550201, macro F1: 0.7481273181256212 [INFO] Relation Linking P: 0.6164658634538153, R: 0.6054216867469879, micro F1: 0.6060240963855423, macro F1: 0.6108938630446189 [INFO] Type Linking P: 0.5, R: 0.5, micro F1: 0.5, macro F1: 0.5 @@ -2332,8 +2332,8 @@ Question 166: Who composed the music for Tonight's the Night ? [INFO] Golden Answer: [http://dbpedia.org/resource/Rod_Stewart] [INFO] Predicted Answer: [Online@en] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 167, P: 0.590, R: 0.630, macro F1: 0.590, macro F1*: 0.609 -[INFO] Current time: 1130 ms, average: 2910.65 ms/ques, entity detection: 393.23 ms/ques (linking tool: 301.43 ms/ques), relation detection: 881.16 ms/ques (similarity: 702.49 ms/ques), EDG generation: 87.56 ms/ques +[INFO] Cumulative metrics, sample: 167, P: 0.590, R: 0.630, macro F1: 0.609 +[INFO] Current question consumed time: 1013 ms, average: 2778.53 ms/ques, entity detection: 378.58 ms/ques (linking tool: 292.58 ms/ques), relation detection: 866.13 ms/ques (similarity: 700.28 ms/ques), EDG generation: 85.58 ms/ques [INFO] Entity Linking P: 0.7604790419161677, R: 0.7275449101796407, micro F1: 0.7385229540918166, macro F1: 0.7436475138254677 [INFO] Relation Linking P: 0.6127744510978044, R: 0.6017964071856288, micro F1: 0.6023952095808385, macro F1: 0.6072358159605193 [INFO] Type Linking P: 0.5029940119760479, R: 0.5029940119760479, micro F1: 0.5029940119760479, macro F1: 0.5029940119760479 @@ -2346,8 +2346,8 @@ Question 167: How many things are made by companies who produce mass media artic [INFO] Golden Answer: [41] [INFO] Predicted Answer: [41, 43] [INFO] P: 0.5, R: 1.0, F: 0.6666666666666666 -[INFO] Cumulative metrics, sample: 168, P: 0.589, R: 0.632, macro F1: 0.591, macro F1*: 0.610 -[INFO] Current time: 4693 ms, average: 2921.26 ms/ques, entity detection: 392.29 ms/ques (linking tool: 300.81 ms/ques), relation detection: 886.50 ms/ques (similarity: 708.49 ms/ques), EDG generation: 87.15 ms/ques +[INFO] Cumulative metrics, sample: 168, P: 0.589, R: 0.632, macro F1: 0.610 +[INFO] Current question consumed time: 4294 ms, average: 2787.55 ms/ques, entity detection: 377.42 ms/ques (linking tool: 291.72 ms/ques), relation detection: 870.52 ms/ques (similarity: 706.17 ms/ques), EDG generation: 85.17 ms/ques [INFO] Entity Linking P: 0.7619047619047619, R: 0.7291666666666666, micro F1: 0.7400793650793653, macro F1: 0.7451763140385894 [INFO] Relation Linking P: 0.6121031746031745, R: 0.6011904761904762, micro F1: 0.6017857142857144, macro F1: 0.6065977494840945 [INFO] Type Linking P: 0.5, R: 0.5, micro F1: 0.5, macro F1: 0.5 @@ -2360,8 +2360,8 @@ Question 168: What awards have been given to race horses bred by Bert Michell? [INFO] Golden Answer: [http://dbpedia.org/resource/Kentucky, http://dbpedia.org/resource/Florence,_Kentucky, http://dbpedia.org/resource/National_Museum_of_Racing_and_Hall_of_Fame, http://dbpedia.org/resource/Lincoln_Heritage_Handicap] [INFO] Predicted Answer: [http://dbpedia.org/resource/American_Champion_Two-Year-Old_Filly] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 169, P: 0.586, R: 0.629, macro F1: 0.587, macro F1*: 0.606 -[INFO] Current time: 2322 ms, average: 2917.72 ms/ques, entity detection: 390.74 ms/ques (linking tool: 299.49 ms/ques), relation detection: 886.31 ms/ques (similarity: 709.71 ms/ques), EDG generation: 86.78 ms/ques +[INFO] Cumulative metrics, sample: 169, P: 0.586, R: 0.629, macro F1: 0.606 +[INFO] Current question consumed time: 2021 ms, average: 2783.02 ms/ques, entity detection: 375.87 ms/ques (linking tool: 290.41 ms/ques), relation detection: 870.10 ms/ques (similarity: 707.37 ms/ques), EDG generation: 84.82 ms/ques [INFO] Entity Linking P: 0.7633136094674556, R: 0.7307692307692307, micro F1: 0.7416173570019726, macro F1: 0.7466869763899467 [INFO] Relation Linking P: 0.6084812623274162, R: 0.5976331360946746, micro F1: 0.5982248520710061, macro F1: 0.6030084136883307 [INFO] Type Linking P: 0.4970414201183432, R: 0.4970414201183432, micro F1: 0.4970414201183432, macro F1: 0.4970414201183432 @@ -2374,8 +2374,8 @@ Question 169: Whose mayors are affiliated with Christen-Democratisch en Vlaams? [INFO] Golden Answer: [http://dbpedia.org/resource/Brussels, http://dbpedia.org/resource/Wallonia, http://dbpedia.org/resource/French_Community_of_Belgium] [INFO] Predicted Answer: [http://dbpedia.org/resource/Centre_démocrate_humaniste, http://dbpedia.org/resource/Christlich_Soziale_Partei_(Belgium)] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 170, P: 0.582, R: 0.625, macro F1: 0.584, macro F1*: 0.603 -[INFO] Current time: 1094 ms, average: 2906.99 ms/ques, entity detection: 389.58 ms/ques (linking tool: 298.72 ms/ques), relation detection: 883.15 ms/ques (similarity: 706.82 ms/ques), EDG generation: 86.39 ms/ques +[INFO] Cumulative metrics, sample: 170, P: 0.582, R: 0.625, macro F1: 0.603 +[INFO] Current question consumed time: 970 ms, average: 2772.35 ms/ques, entity detection: 374.65 ms/ques (linking tool: 289.57 ms/ques), relation detection: 866.86 ms/ques (similarity: 704.41 ms/ques), EDG generation: 84.45 ms/ques [INFO] Entity Linking P: 0.7647058823529411, R: 0.7323529411764705, micro F1: 0.743137254901961, macro F1: 0.7481798220270426 [INFO] Relation Linking P: 0.6107843137254901, R: 0.5970588235294118, micro F1: 0.5986274509803923, macro F1: 0.6038435828877005 [INFO] Type Linking P: 0.5, R: 0.5, micro F1: 0.5, macro F1: 0.5 @@ -2388,8 +2388,8 @@ Question 170: What is the common nationality of Viveka Babajee anad Ronny Vencat [INFO] Golden Answer: [http://dbpedia.org/resource/Mauritius] [INFO] Predicted Answer: [http://dbpedia.org/resource/Mauritius] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 171, P: 0.585, R: 0.627, macro F1: 0.586, macro F1*: 0.605 -[INFO] Current time: 873 ms, average: 2895.09 ms/ques, entity detection: 389.33 ms/ques (linking tool: 298.25 ms/ques), relation detection: 878.57 ms/ques (similarity: 703.06 ms/ques), EDG generation: 86.10 ms/ques +[INFO] Cumulative metrics, sample: 171, P: 0.585, R: 0.627, macro F1: 0.605 +[INFO] Current question consumed time: 795 ms, average: 2760.79 ms/ques, entity detection: 374.29 ms/ques (linking tool: 289.06 ms/ques), relation detection: 862.44 ms/ques (similarity: 700.65 ms/ques), EDG generation: 84.22 ms/ques [INFO] Entity Linking P: 0.7660818713450293, R: 0.7309941520467836, micro F1: 0.7426900584795324, macro F1: 0.7481268274853802 [INFO] Relation Linking P: 0.6130604288499025, R: 0.5964912280701754, micro F1: 0.5990253411306045, macro F1: 0.6046623407834655 [INFO] Type Linking P: 0.5029239766081871, R: 0.5029239766081871, micro F1: 0.5029239766081871, macro F1: 0.5029239766081871 @@ -2402,8 +2402,8 @@ Question 171: Is the Hyundai Lavita assembled in Ulsan? [INFO] Golden Answer: [true] [INFO] Predicted Answer: [true] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 172, P: 0.587, R: 0.629, macro F1: 0.589, macro F1*: 0.608 -[INFO] Current time: 5014 ms, average: 2907.41 ms/ques, entity detection: 390.07 ms/ques (linking tool: 298.23 ms/ques), relation detection: 893.63 ms/ques (similarity: 711.29 ms/ques), EDG generation: 85.67 ms/ques +[INFO] Cumulative metrics, sample: 172, P: 0.587, R: 0.629, macro F1: 0.608 +[INFO] Current question consumed time: 4611 ms, average: 2771.55 ms/ques, entity detection: 374.70 ms/ques (linking tool: 288.80 ms/ques), relation detection: 877.12 ms/ques (similarity: 708.89 ms/ques), EDG generation: 83.73 ms/ques [INFO] Entity Linking P: 0.7645348837209303, R: 0.7296511627906976, micro F1: 0.7412790697674421, macro F1: 0.7466858202877568 [INFO] Relation Linking P: 0.6094961240310077, R: 0.5930232558139535, micro F1: 0.595542635658915, macro F1: 0.6011468620579802 [INFO] Type Linking P: 0.5058139534883721, R: 0.5058139534883721, micro F1: 0.5058139534883721, macro F1: 0.5058139534883721 @@ -2416,8 +2416,8 @@ Question 172: Which party won the elections when Y.S. Rajasekhara Reddy was in p [INFO] Golden Answer: [http://dbpedia.org/resource/Indian_National_Congress, http://dbpedia.org/resource/Telugu_Desam_Party] [INFO] Predicted Answer: [Indian National Congress@en, Telugu Desam Party@en, Praja Rajyam Party@en, Bharatiya Janata Party@en, http://dbpedia.org/resource/Telangana_Rashtra_Samithi, http://dbpedia.org/resource/Telugu_Desam_Party, Independent@en, Samajwadi Party@en, Rashtriya Janata Dal@en, Janata Party@en, Communist Party of India@en, Bahujan Samaj Party@en, Telangana Rashtra Samithi@en, Republican Party of India@en, All India Majlis-E-Ittehadul Muslimeen@en, Peoples Republican Party@en, Marxist Communist Party of India@en, Praja Party@en, Akhil Bharatiya Jan Sangh@en, Ambedkar National Congress@en, Majlis Bachao Tahreek@en, Mudiraj Rashtriya Samithi@en, Pyramid Party of India@en, Muslim League Kerala State Committee@en, Bharatiya Republican Paksha@en, Communist Party of India Liberation@en, Unity Centre of Communist Revolutionaries of India@en, NTR Telugu Desam Party@en] [INFO] P: 0.07692307692307693, R: 1.0, F: 0.14285714285714288 -[INFO] Cumulative metrics, sample: 173, P: 0.584, R: 0.631, macro F1: 0.586, macro F1*: 0.607 -[INFO] Current time: 2077 ms, average: 2902.61 ms/ques, entity detection: 388.64 ms/ques (linking tool: 297.06 ms/ques), relation detection: 893.16 ms/ques (similarity: 712.11 ms/ques), EDG generation: 85.32 ms/ques +[INFO] Cumulative metrics, sample: 173, P: 0.584, R: 0.631, macro F1: 0.607 +[INFO] Current question consumed time: 1999 ms, average: 2767.08 ms/ques, entity detection: 373.34 ms/ques (linking tool: 287.66 ms/ques), relation detection: 876.36 ms/ques (similarity: 709.68 ms/ques), EDG generation: 83.79 ms/ques [INFO] Entity Linking P: 0.7658959537572254, R: 0.7312138728323699, micro F1: 0.7427745664739888, macro F1: 0.748153190349722 [INFO] Relation Linking P: 0.6088631984585742, R: 0.5924855491329479, micro F1: 0.5949903660886322, macro F1: 0.6005627378540868 [INFO] Type Linking P: 0.5028901734104047, R: 0.5028901734104047, micro F1: 0.5028901734104047, macro F1: 0.5028901734104047 @@ -2430,8 +2430,8 @@ Question 173: Where does the river begin whose left tributary is Krapanski Potok [INFO] Golden Answer: [http://dbpedia.org/resource/Pićan, http://dbpedia.org/resource/Croatia] [INFO] Predicted Answer: [http://dbpedia.org/resource/Pićan, http://dbpedia.org/resource/Croatia, Near Pićan, Croatia@en] [INFO] P: 0.6666666666666666, R: 1.0, F: 0.8 -[INFO] Cumulative metrics, sample: 174, P: 0.585, R: 0.634, macro F1: 0.587, macro F1*: 0.608 -[INFO] Current time: 844 ms, average: 2890.78 ms/ques, entity detection: 387.13 ms/ques (linking tool: 295.76 ms/ques), relation detection: 889.24 ms/ques (similarity: 708.82 ms/ques), EDG generation: 85.05 ms/ques +[INFO] Cumulative metrics, sample: 174, P: 0.585, R: 0.634, macro F1: 0.608 +[INFO] Current question consumed time: 780 ms, average: 2755.66 ms/ques, entity detection: 371.91 ms/ques (linking tool: 286.42 ms/ques), relation detection: 872.45 ms/ques (similarity: 706.37 ms/ques), EDG generation: 83.48 ms/ques [INFO] Entity Linking P: 0.7672413793103449, R: 0.7327586206896551, micro F1: 0.7442528735632187, macro F1: 0.7496036464526358 [INFO] Relation Linking P: 0.611111111111111, R: 0.5948275862068966, micro F1: 0.5973180076628354, macro F1: 0.6028594122319301 [INFO] Type Linking P: 0.5057471264367817, R: 0.5057471264367817, micro F1: 0.5057471264367817, macro F1: 0.5057471264367817 @@ -2444,8 +2444,8 @@ Question 174: What are the notable commanders of the army whose one of the nota [INFO] Golden Answer: [http://dbpedia.org/resource/Felix_Steiner, http://dbpedia.org/resource/Kurt_Meyer, http://dbpedia.org/resource/Sepp_Dietrich, http://dbpedia.org/resource/Theodor_Eicke, http://dbpedia.org/resource/Joachim_Peiper, http://dbpedia.org/resource/Paul_Hausser, http://dbpedia.org/resource/Heinrich_Himmler, http://dbpedia.org/resource/Brigadeführer, http://dbpedia.org/resource/Oberführer, http://dbpedia.org/resource/Obergruppenführer] [INFO] Predicted Answer: [http://dbpedia.org/resource/Erich_Brandenberger, http://dbpedia.org/resource/Heinrich_Eberbach, http://dbpedia.org/resource/Paul_Hausser, http://dbpedia.org/resource/Friedrich_Dollmann, http://dbpedia.org/resource/Hans_Felber] [INFO] P: 0.2, R: 0.1, F: 0.13333333333333333 -[INFO] Cumulative metrics, sample: 175, P: 0.583, R: 0.630, macro F1: 0.585, macro F1*: 0.606 -[INFO] Current time: 2596 ms, average: 2889.10 ms/ques, entity detection: 386.63 ms/ques (linking tool: 295.16 ms/ques), relation detection: 889.62 ms/ques (similarity: 708.01 ms/ques), EDG generation: 84.89 ms/ques +[INFO] Cumulative metrics, sample: 175, P: 0.583, R: 0.630, macro F1: 0.606 +[INFO] Current question consumed time: 2362 ms, average: 2753.41 ms/ques, entity detection: 371.26 ms/ques (linking tool: 285.63 ms/ques), relation detection: 872.73 ms/ques (similarity: 705.45 ms/ques), EDG generation: 83.36 ms/ques [INFO] Entity Linking P: 0.7657142857142857, R: 0.7342857142857143, micro F1: 0.743809523809524, macro F1: 0.7496707482993198 [INFO] Relation Linking P: 0.6076190476190476, R: 0.5914285714285714, micro F1: 0.593904761904762, macro F1: 0.5994145013048905 [INFO] Type Linking P: 0.5085714285714286, R: 0.5085714285714286, micro F1: 0.5085714285714286, macro F1: 0.5085714285714286 @@ -2458,8 +2458,8 @@ Question 175: Name the source of Raa ? [INFO] Golden Answer: [http://dbpedia.org/resource/Pićan, http://dbpedia.org/resource/Croatia] [INFO] Predicted Answer: [Keerthika Udhaya@en] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 176, P: 0.579, R: 0.627, macro F1: 0.581, macro F1*: 0.602 -[INFO] Current time: 682 ms, average: 2876.56 ms/ques, entity detection: 385.26 ms/ques (linking tool: 293.85 ms/ques), relation detection: 885.48 ms/ques (similarity: 704.53 ms/ques), EDG generation: 84.51 ms/ques +[INFO] Cumulative metrics, sample: 176, P: 0.579, R: 0.627, macro F1: 0.602 +[INFO] Current question consumed time: 579 ms, average: 2741.06 ms/ques, entity detection: 369.86 ms/ques (linking tool: 284.35 ms/ques), relation detection: 868.59 ms/ques (similarity: 701.99 ms/ques), EDG generation: 82.89 ms/ques [INFO] Entity Linking P: 0.7613636363636364, R: 0.7301136363636364, micro F1: 0.7395833333333336, macro F1: 0.7454112554112553 [INFO] Relation Linking P: 0.6041666666666666, R: 0.5880681818181818, micro F1: 0.5905303030303032, macro F1: 0.5960087370929308 [INFO] Type Linking P: 0.5113636363636364, R: 0.5113636363636364, micro F1: 0.5113636363636364, macro F1: 0.5113636363636364 @@ -2472,8 +2472,8 @@ Question 176: Does Fa'a' International Airport serve tahiti too? [INFO] Golden Answer: [true] [INFO] Predicted Answer: [true] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 177, P: 0.582, R: 0.629, macro F1: 0.584, macro F1*: 0.604 -[INFO] Current time: 1542 ms, average: 2869.02 ms/ques, entity detection: 385.11 ms/ques (linking tool: 293.58 ms/ques), relation detection: 882.69 ms/ques (similarity: 701.98 ms/ques), EDG generation: 84.09 ms/ques +[INFO] Cumulative metrics, sample: 177, P: 0.582, R: 0.629, macro F1: 0.604 +[INFO] Current question consumed time: 1311 ms, average: 2732.98 ms/ques, entity detection: 369.73 ms/ques (linking tool: 284.06 ms/ques), relation detection: 865.94 ms/ques (similarity: 699.46 ms/ques), EDG generation: 82.49 ms/ques [INFO] Entity Linking P: 0.7570621468926554, R: 0.7259887005649718, micro F1: 0.7354048964218458, macro F1: 0.7411998923863331 [INFO] Relation Linking P: 0.6064030131826742, R: 0.5903954802259888, micro F1: 0.5928436911487761, macro F1: 0.5982921939662261 [INFO] Type Linking P: 0.5141242937853108, R: 0.5141242937853108, micro F1: 0.5141242937853108, macro F1: 0.5141242937853108 @@ -2486,8 +2486,8 @@ Question 177: Who is the producer of Frampton's Camel ? [INFO] Golden Answer: [http://dbpedia.org/resource/Peter_Frampton] [INFO] Predicted Answer: [http://dbpedia.org/resource/Peter_Frampton] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 178, P: 0.584, R: 0.631, macro F1: 0.586, macro F1*: 0.607 -[INFO] Current time: 1315 ms, average: 2860.29 ms/ques, entity detection: 384.72 ms/ques (linking tool: 293.02 ms/ques), relation detection: 879.84 ms/ques (similarity: 699.35 ms/ques), EDG generation: 83.69 ms/ques +[INFO] Cumulative metrics, sample: 178, P: 0.584, R: 0.631, macro F1: 0.607 +[INFO] Current question consumed time: 1249 ms, average: 2724.64 ms/ques, entity detection: 369.31 ms/ques (linking tool: 283.51 ms/ques), relation detection: 863.19 ms/ques (similarity: 696.90 ms/ques), EDG generation: 82.10 ms/ques [INFO] Entity Linking P: 0.7528089887640449, R: 0.7219101123595506, micro F1: 0.7312734082397007, macro F1: 0.7370358480470839 [INFO] Relation Linking P: 0.6086142322097378, R: 0.5926966292134831, micro F1: 0.5951310861423222, macro F1: 0.6005499750409415 [INFO] Type Linking P: 0.5168539325842697, R: 0.5168539325842697, micro F1: 0.5168539325842697, macro F1: 0.5168539325842697 @@ -2500,8 +2500,8 @@ Question 178: Name the directors of Lennon Legend: The Very Best of John Lennon [INFO] Golden Answer: [http://dbpedia.org/resource/John_Lennon, http://dbpedia.org/resource/Yoko_Ono, http://dbpedia.org/resource/Simon_Hilton] [INFO] Predicted Answer: [Simon Hilton, John Lennon, Yoko Ono, Joe Pytka, Bruce Westcott@en, http://dbpedia.org/resource/John_Lennon, http://dbpedia.org/resource/Yoko_Ono, http://dbpedia.org/resource/Simon_Hilton] [INFO] P: 0.75, R: 1.0, F: 0.8571428571428571 -[INFO] Cumulative metrics, sample: 179, P: 0.585, R: 0.633, macro F1: 0.587, macro F1*: 0.608 -[INFO] Current time: 1459 ms, average: 2852.46 ms/ques, entity detection: 384.97 ms/ques (linking tool: 293.54 ms/ques), relation detection: 875.96 ms/ques (similarity: 696.06 ms/ques), EDG generation: 83.46 ms/ques +[INFO] Cumulative metrics, sample: 179, P: 0.585, R: 0.633, macro F1: 0.608 +[INFO] Current question consumed time: 1283 ms, average: 2716.59 ms/ques, entity detection: 369.29 ms/ques (linking tool: 283.78 ms/ques), relation detection: 859.32 ms/ques (similarity: 693.53 ms/ques), EDG generation: 81.93 ms/ques [INFO] Entity Linking P: 0.7541899441340782, R: 0.723463687150838, micro F1: 0.7327746741154565, macro F1: 0.738507355503691 [INFO] Relation Linking P: 0.6108007448789572, R: 0.5949720670391061, micro F1: 0.597392923649907, macro F1: 0.6027825111623994 [INFO] Type Linking P: 0.5195530726256983, R: 0.5195530726256983, micro F1: 0.5195530726256983, macro F1: 0.5195530726256983 @@ -2514,8 +2514,8 @@ Question 179: What is the political party of Arun Chandra Guha and Indira Gandhi [INFO] Golden Answer: [http://dbpedia.org/resource/Indian_National_Congress] [INFO] Predicted Answer: [http://dbpedia.org/resource/Neyyattinkara__Christ_Nagar_Residents'_Association_CNRA_,_Christ__1, http://dbpedia.org/resource/Sonia_Gandhi] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 180, P: 0.582, R: 0.630, macro F1: 0.584, macro F1*: 0.605 -[INFO] Current time: 1856 ms, average: 2846.93 ms/ques, entity detection: 383.77 ms/ques (linking tool: 292.42 ms/ques), relation detection: 875.25 ms/ques (similarity: 694.26 ms/ques), EDG generation: 83.15 ms/ques +[INFO] Cumulative metrics, sample: 180, P: 0.582, R: 0.630, macro F1: 0.605 +[INFO] Current question consumed time: 1670 ms, average: 2710.77 ms/ques, entity detection: 368.02 ms/ques (linking tool: 282.73 ms/ques), relation detection: 858.28 ms/ques (similarity: 691.65 ms/ques), EDG generation: 81.64 ms/ques [INFO] Entity Linking P: 0.7555555555555555, R: 0.7222222222222222, micro F1: 0.7324074074074076, macro F1: 0.7385129490392647 [INFO] Relation Linking P: 0.6101851851851852, R: 0.5972222222222222, micro F1: 0.597777777777778, macro F1: 0.6036341172460805 [INFO] Type Linking P: 0.5222222222222223, R: 0.5222222222222223, micro F1: 0.5222222222222223, macro F1: 0.5222222222222223 @@ -2528,8 +2528,8 @@ Question 180: To which country is Svetlana Navasardyan from? [INFO] Golden Answer: [http://dbpedia.org/resource/Armenia] [INFO] Predicted Answer: [http://dbpedia.org/resource/Armenia] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 181, P: 0.584, R: 0.632, macro F1: 0.587, macro F1*: 0.607 -[INFO] Current time: 719 ms, average: 2835.17 ms/ques, entity detection: 382.38 ms/ques (linking tool: 291.23 ms/ques), relation detection: 871.43 ms/ques (similarity: 691.45 ms/ques), EDG generation: 82.75 ms/ques +[INFO] Cumulative metrics, sample: 181, P: 0.584, R: 0.632, macro F1: 0.607 +[INFO] Current question consumed time: 680 ms, average: 2699.55 ms/ques, entity detection: 366.71 ms/ques (linking tool: 281.57 ms/ques), relation detection: 854.53 ms/ques (similarity: 688.83 ms/ques), EDG generation: 81.19 ms/ques [INFO] Entity Linking P: 0.7569060773480663, R: 0.7237569060773481, micro F1: 0.7338858195211788, macro F1: 0.7399604188999753 [INFO] Relation Linking P: 0.6068139963167587, R: 0.5939226519337016, micro F1: 0.5944751381215472, macro F1: 0.6002991221231738 [INFO] Type Linking P: 0.5193370165745856, R: 0.5193370165745856, micro F1: 0.5193370165745856, macro F1: 0.5193370165745856 @@ -2542,8 +2542,8 @@ Question 181: What is the alma mater of the scientist who is known for Rational [INFO] Golden Answer: [http://dbpedia.org/resource/Stanford_University, http://dbpedia.org/resource/University_of_British_Columbia] [INFO] Predicted Answer: [http://dbpedia.org/resource/Stanford_University, http://dbpedia.org/resource/University_of_British_Columbia, Stanford University@en, University of British Columbia@en] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 182, P: 0.586, R: 0.634, macro F1: 0.589, macro F1*: 0.609 -[INFO] Current time: 947 ms, average: 2824.80 ms/ques, entity detection: 381.20 ms/ques (linking tool: 290.10 ms/ques), relation detection: 868.11 ms/ques (similarity: 688.79 ms/ques), EDG generation: 82.51 ms/ques +[INFO] Cumulative metrics, sample: 182, P: 0.586, R: 0.634, macro F1: 0.609 +[INFO] Current question consumed time: 869 ms, average: 2689.49 ms/ques, entity detection: 365.45 ms/ques (linking tool: 280.46 ms/ques), relation detection: 851.15 ms/ques (similarity: 686.14 ms/ques), EDG generation: 80.95 ms/ques [INFO] Entity Linking P: 0.7582417582417582, R: 0.7252747252747253, micro F1: 0.7353479853479855, macro F1: 0.7413919413919412 [INFO] Relation Linking P: 0.6089743589743589, R: 0.5961538461538461, micro F1: 0.5967032967032969, macro F1: 0.6024959083469721 [INFO] Type Linking P: 0.5164835164835165, R: 0.5164835164835165, micro F1: 0.5164835164835165, macro F1: 0.5164835164835165 @@ -2556,8 +2556,8 @@ Question 182: Who is the owner of Chelsea F.C. ? [INFO] Golden Answer: [http://dbpedia.org/resource/Roman_Abramovich] [INFO] Predicted Answer: [http://dbpedia.org/resource/Cobham_Training_Centre, http://dbpedia.org/resource/Roman_Abramovich, http://dbpedia.org/resource/Chelsea_TV] [INFO] P: 0.3333333333333333, R: 1.0, F: 0.5 -[INFO] Cumulative metrics, sample: 183, P: 0.585, R: 0.636, macro F1: 0.588, macro F1*: 0.609 -[INFO] Current time: 2030 ms, average: 2820.45 ms/ques, entity detection: 381.12 ms/ques (linking tool: 290.23 ms/ques), relation detection: 867.34 ms/ques (similarity: 687.09 ms/ques), EDG generation: 82.11 ms/ques +[INFO] Cumulative metrics, sample: 183, P: 0.585, R: 0.636, macro F1: 0.609 +[INFO] Current question consumed time: 1717 ms, average: 2684.18 ms/ques, entity detection: 365.22 ms/ques (linking tool: 280.46 ms/ques), relation detection: 850.11 ms/ques (similarity: 684.47 ms/ques), EDG generation: 80.56 ms/ques [INFO] Entity Linking P: 0.7595628415300546, R: 0.726775956284153, micro F1: 0.7367941712204009, macro F1: 0.7428077788492446 [INFO] Relation Linking P: 0.611111111111111, R: 0.5983606557377049, micro F1: 0.5989071038251368, macro F1: 0.6046686746987951 [INFO] Type Linking P: 0.5191256830601093, R: 0.5191256830601093, micro F1: 0.5191256830601093, macro F1: 0.5191256830601093 @@ -2570,8 +2570,8 @@ Question 183: In how many places has Don R. Berlin worked? [INFO] Golden Answer: [8] [INFO] Predicted Answer: [1] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 184, P: 0.582, R: 0.632, macro F1: 0.585, macro F1*: 0.606 -[INFO] Current time: 1982 ms, average: 2815.90 ms/ques, entity detection: 382.29 ms/ques (linking tool: 291.58 ms/ques), relation detection: 864.39 ms/ques (similarity: 685.15 ms/ques), EDG generation: 81.72 ms/ques +[INFO] Cumulative metrics, sample: 184, P: 0.582, R: 0.632, macro F1: 0.606 +[INFO] Current question consumed time: 1719 ms, average: 2678.93 ms/ques, entity detection: 366.09 ms/ques (linking tool: 281.54 ms/ques), relation detection: 847.11 ms/ques (similarity: 682.51 ms/ques), EDG generation: 80.21 ms/ques [INFO] Entity Linking P: 0.7608695652173914, R: 0.7282608695652174, micro F1: 0.7382246376811596, macro F1: 0.7442081878768645 [INFO] Relation Linking P: 0.6077898550724637, R: 0.595108695652174, micro F1: 0.5956521739130437, macro F1: 0.6013824319015191 [INFO] Type Linking P: 0.5163043478260869, R: 0.5163043478260869, micro F1: 0.5163043478260869, macro F1: 0.5163043478260869 @@ -2584,8 +2584,8 @@ Question 184: For how many movies are there whose musicians home town is in Volo [INFO] Golden Answer: [18] [INFO] Predicted Answer: [2] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 185, P: 0.579, R: 0.629, macro F1: 0.582, macro F1*: 0.603 -[INFO] Current time: 3600 ms, average: 2820.14 ms/ques, entity detection: 381.36 ms/ques (linking tool: 290.68 ms/ques), relation detection: 868.03 ms/ques (similarity: 691.41 ms/ques), EDG generation: 81.42 ms/ques +[INFO] Cumulative metrics, sample: 185, P: 0.579, R: 0.629, macro F1: 0.603 +[INFO] Current question consumed time: 3345 ms, average: 2682.54 ms/ques, entity detection: 365.09 ms/ques (linking tool: 280.60 ms/ques), relation detection: 850.39 ms/ques (similarity: 688.69 ms/ques), EDG generation: 79.95 ms/ques [INFO] Entity Linking P: 0.7621621621621621, R: 0.7297297297297297, micro F1: 0.7396396396396399, macro F1: 0.745593419506463 [INFO] Relation Linking P: 0.6099099099099099, R: 0.5945945945945946, micro F1: 0.5960360360360363, macro F1: 0.602154884877398 [INFO] Type Linking P: 0.5135135135135135, R: 0.5135135135135135, micro F1: 0.5135135135135135, macro F1: 0.5135135135135135 @@ -2598,8 +2598,8 @@ Question 185: In all the places with a stagecoach, where is a Peetz Table Wind E [INFO] Golden Answer: [http://dbpedia.org/resource/Colorado] [INFO] Predicted Answer: [http://dbpedia.org/resource/Ute_Wars, http://dbpedia.org/resource/Black_Hawk_War_(1865–72), http://dbpedia.org/resource/1990_NBA_Playoffs, http://dbpedia.org/resource/1994_NBA_Playoffs, http://dbpedia.org/resource/1995_NBA_Playoffs, http://dbpedia.org/resource/Colorado_Coalfield_War, http://dbpedia.org/resource/White_Ash_Mine_Disaster, http://dbpedia.org/resource/Bluff_War, http://dbpedia.org/resource/2004_NBA_Playoffs, http://dbpedia.org/resource/2008_NBA_Playoffs, http://dbpedia.org/resource/2010_NBA_Playoffs, http://dbpedia.org/resource/2011_NBA_Playoffs, http://dbpedia.org/resource/2013_NBA_Playoffs, http://dbpedia.org/resource/2004–05_San_Antonio_Spurs_season, http://dbpedia.org/resource/2005_NBA_Playoffs, http://dbpedia.org/resource/2005–06_Los_Angeles_Clippers_season, http://dbpedia.org/resource/2006_NBA_Playoffs, http://dbpedia.org/resource/2007_NBA_Playoffs, http://dbpedia.org/resource/2008–09_New_Orleans_Hornets_season, http://dbpedia.org/resource/2009_NBA_Playoffs, http://dbpedia.org/resource/2012_NBA_Playoffs, http://dbpedia.org/resource/White_River_War, http://dbpedia.org/resource/Battle_of_Milk_Creek, http://dbpedia.org/resource/NBA_All-Star_Celebrity_Game, http://dbpedia.org/resource/1982_NBA_Playoffs, http://dbpedia.org/resource/1983_NBA_Playoffs, http://dbpedia.org/resource/1984_NBA_Playoffs, http://dbpedia.org/resource/1985_NBA_Playoffs, http://dbpedia.org/resource/1986_NBA_Playoffs, http://dbpedia.org/resource/1987_NBA_Playoffs, http://dbpedia.org/resource/1988_NBA_Playoffs, http://dbpedia.org/resource/1989_NBA_Playoffs, http://dbpedia.org/resource/Battle_of_Summit_Springs, http://dbpedia.org/resource/Colorado_War, http://dbpedia.org/resource/2004–05_Denver_Nuggets_season, http://dbpedia.org/resource/2005–06_Denver_Nuggets_season, http://dbpedia.org/resource/2008–09_Dallas_Mavericks_season, http://dbpedia.org/resource/Battle_of_Beecher_Island, http://dbpedia.org/resource/American_Ranch_massacre] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 186, P: 0.575, R: 0.625, macro F1: 0.579, macro F1*: 0.599 -[INFO] Current time: 4778 ms, average: 2830.66 ms/ques, entity detection: 381.34 ms/ques (linking tool: 290.36 ms/ques), relation detection: 876.43 ms/ques (similarity: 695.17 ms/ques), EDG generation: 81.21 ms/ques +[INFO] Cumulative metrics, sample: 186, P: 0.575, R: 0.625, macro F1: 0.599 +[INFO] Current question consumed time: 4189 ms, average: 2690.63 ms/ques, entity detection: 364.83 ms/ques (linking tool: 280.04 ms/ques), relation detection: 857.94 ms/ques (similarity: 692.39 ms/ques), EDG generation: 79.69 ms/ques [INFO] Entity Linking P: 0.7634408602150538, R: 0.728494623655914, micro F1: 0.7392473118279571, macro F1: 0.745558461687494 [INFO] Relation Linking P: 0.6093189964157706, R: 0.5967741935483871, micro F1: 0.5964157706093193, macro F1: 0.6029813545511191 [INFO] Type Linking P: 0.5161290322580645, R: 0.5161290322580645, micro F1: 0.5161290322580645, macro F1: 0.5161290322580645 @@ -2612,8 +2612,8 @@ Question 186: Name the fictional character whose relatives are Huey, Dewey, and [INFO] Golden Answer: [http://dbpedia.org/resource/Scrooge_McDuck, http://dbpedia.org/resource/Donald_Duck] [INFO] Predicted Answer: [http://dbpedia.org/resource/Scrooge_McDuck, http://dbpedia.org/resource/Huey,_Dewey,_and_Louie, http://dbpedia.org/resource/Donald_Duck] [INFO] P: 0.6666666666666666, R: 1.0, F: 0.8 -[INFO] Cumulative metrics, sample: 187, P: 0.576, R: 0.627, macro F1: 0.580, macro F1*: 0.601 -[INFO] Current time: 2869 ms, average: 2830.87 ms/ques, entity detection: 382.65 ms/ques (linking tool: 290.67 ms/ques), relation detection: 875.06 ms/ques (similarity: 693.49 ms/ques), EDG generation: 81.20 ms/ques +[INFO] Cumulative metrics, sample: 187, P: 0.576, R: 0.627, macro F1: 0.601 +[INFO] Current question consumed time: 2576 ms, average: 2690.02 ms/ques, entity detection: 365.65 ms/ques (linking tool: 280.05 ms/ques), relation detection: 856.44 ms/ques (similarity: 690.59 ms/ques), EDG generation: 80.16 ms/ques [INFO] Entity Linking P: 0.7647058823529411, R: 0.7272727272727273, micro F1: 0.7388591800356508, macro F1: 0.7455197132616488 [INFO] Relation Linking P: 0.6114081996434937, R: 0.5962566844919787, micro F1: 0.5967914438502676, macro F1: 0.6037373956627267 [INFO] Type Linking P: 0.5133689839572193, R: 0.5133689839572193, micro F1: 0.5133689839572193, macro F1: 0.5133689839572193 @@ -2626,8 +2626,8 @@ Question 187: Who is the owner of the website which has the label Seed Eight ? [INFO] Golden Answer: [http://dbpedia.org/resource/Apple_Inc.] [INFO] Predicted Answer: [http://dbpedia.org/resource/Apple_Inc.] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 188, P: 0.578, R: 0.629, macro F1: 0.582, macro F1*: 0.603 -[INFO] Current time: 1057 ms, average: 2821.43 ms/ques, entity detection: 381.39 ms/ques (linking tool: 289.85 ms/ques), relation detection: 872.72 ms/ques (similarity: 691.92 ms/ques), EDG generation: 80.93 ms/ques +[INFO] Cumulative metrics, sample: 188, P: 0.578, R: 0.629, macro F1: 0.603 +[INFO] Current question consumed time: 921 ms, average: 2680.61 ms/ques, entity detection: 364.37 ms/ques (linking tool: 279.23 ms/ques), relation detection: 854.03 ms/ques (similarity: 689.13 ms/ques), EDG generation: 79.90 ms/ques [INFO] Entity Linking P: 0.7659574468085106, R: 0.7287234042553191, micro F1: 0.7402482269503547, macro F1: 0.7468766563186188 [INFO] Relation Linking P: 0.6134751773049645, R: 0.598404255319149, micro F1: 0.5989361702127662, macro F1: 0.6058460053852978 [INFO] Type Linking P: 0.5106382978723404, R: 0.5106382978723404, micro F1: 0.5106382978723404, macro F1: 0.5106382978723404 @@ -2640,8 +2640,8 @@ Question 188: In which country is Sierra del Merendin? [INFO] Golden Answer: [http://dbpedia.org/resource/Guatemala, http://dbpedia.org/resource/Honduras] [INFO] Predicted Answer: [http://dbpedia.org/resource/Guatemala, http://dbpedia.org/resource/Honduras] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 189, P: 0.580, R: 0.631, macro F1: 0.584, macro F1*: 0.605 -[INFO] Current time: 2123 ms, average: 2817.74 ms/ques, entity detection: 380.00 ms/ques (linking tool: 288.64 ms/ques), relation detection: 872.60 ms/ques (similarity: 693.85 ms/ques), EDG generation: 80.53 ms/ques +[INFO] Cumulative metrics, sample: 189, P: 0.580, R: 0.631, macro F1: 0.605 +[INFO] Current question consumed time: 1957 ms, average: 2676.78 ms/ques, entity detection: 362.93 ms/ques (linking tool: 278.08 ms/ques), relation detection: 853.83 ms/ques (similarity: 691.03 ms/ques), EDG generation: 79.48 ms/ques [INFO] Entity Linking P: 0.7671957671957672, R: 0.7301587301587301, micro F1: 0.7416225749559084, macro F1: 0.7482191934488754 [INFO] Relation Linking P: 0.6155202821869489, R: 0.6005291005291006, micro F1: 0.6010582010582013, macro F1: 0.6079322874101699 [INFO] Type Linking P: 0.5079365079365079, R: 0.5079365079365079, micro F1: 0.5079365079365079, macro F1: 0.5079365079365079 @@ -2654,8 +2654,8 @@ Question 189: What is on the border of the places which fall under crook county [INFO] Golden Answer: [http://dbpedia.org/resource/Idaho, http://dbpedia.org/resource/Nevada, http://dbpedia.org/resource/Cascade_Range, http://dbpedia.org/resource/Blue_Mountains_(Pacific_Northwest)] [INFO] Predicted Answer: [] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 190, P: 0.577, R: 0.628, macro F1: 0.581, macro F1*: 0.602 -[INFO] Current time: 6504 ms, average: 2837.14 ms/ques, entity detection: 380.57 ms/ques (linking tool: 289.43 ms/ques), relation detection: 881.68 ms/ques (similarity: 703.67 ms/ques), EDG generation: 80.35 ms/ques +[INFO] Cumulative metrics, sample: 190, P: 0.577, R: 0.628, macro F1: 0.602 +[INFO] Current question consumed time: 5771 ms, average: 2693.07 ms/ques, entity detection: 363.08 ms/ques (linking tool: 278.44 ms/ques), relation detection: 861.68 ms/ques (similarity: 700.84 ms/ques), EDG generation: 79.32 ms/ques [INFO] Entity Linking P: 0.7631578947368421, R: 0.7263157894736842, micro F1: 0.7377192982456141, macro F1: 0.7442811976938815 [INFO] Relation Linking P: 0.612280701754386, R: 0.5973684210526315, micro F1: 0.5978947368421055, macro F1: 0.6047326437922217 [INFO] Type Linking P: 0.5105263157894737, R: 0.5105263157894737, micro F1: 0.5105263157894737, macro F1: 0.5105263157894737 @@ -2668,8 +2668,8 @@ Question 190: Which is the university of Vanderbilt Commodores ? [INFO] Golden Answer: [http://dbpedia.org/resource/Vanderbilt_University] [INFO] Predicted Answer: [http://dbpedia.org/resource/Vanderbilt_University] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 191, P: 0.580, R: 0.630, macro F1: 0.584, macro F1*: 0.604 -[INFO] Current time: 1141 ms, average: 2828.26 ms/ques, entity detection: 379.61 ms/ques (linking tool: 288.68 ms/ques), relation detection: 879.42 ms/ques (similarity: 701.50 ms/ques), EDG generation: 79.98 ms/ques +[INFO] Cumulative metrics, sample: 191, P: 0.580, R: 0.630, macro F1: 0.604 +[INFO] Current question consumed time: 1058 ms, average: 2684.51 ms/ques, entity detection: 362.08 ms/ques (linking tool: 277.64 ms/ques), relation detection: 859.48 ms/ques (similarity: 698.71 ms/ques), EDG generation: 78.96 ms/ques [INFO] Entity Linking P: 0.7643979057591623, R: 0.7277486910994765, micro F1: 0.7390924956369983, macro F1: 0.7456232203545513 [INFO] Relation Linking P: 0.6143106457242583, R: 0.599476439790576, micro F1: 0.6000000000000003, macro F1: 0.6068028952013881 [INFO] Type Linking P: 0.5130890052356021, R: 0.5130890052356021, micro F1: 0.5130890052356021, macro F1: 0.5130890052356021 @@ -2682,8 +2682,8 @@ Question 191: Whose relatives are Clan McDuck and Huey, Dewey, and Louie? [INFO] Golden Answer: [http://dbpedia.org/resource/Donald_Duck] [INFO] Predicted Answer: [http://dbpedia.org/resource/Donald_Duck] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 192, P: 0.582, R: 0.632, macro F1: 0.586, macro F1*: 0.606 -[INFO] Current time: 3997 ms, average: 2834.34 ms/ques, entity detection: 380.92 ms/ques (linking tool: 289.07 ms/ques), relation detection: 880.14 ms/ques (similarity: 703.83 ms/ques), EDG generation: 79.80 ms/ques +[INFO] Cumulative metrics, sample: 192, P: 0.582, R: 0.632, macro F1: 0.606 +[INFO] Current question consumed time: 3614 ms, average: 2689.35 ms/ques, entity detection: 363.19 ms/ques (linking tool: 277.92 ms/ques), relation detection: 859.97 ms/ques (similarity: 701.03 ms/ques), EDG generation: 78.76 ms/ques [INFO] Entity Linking P: 0.765625, R: 0.7265625, micro F1: 0.7387152777777778, macro F1: 0.7455824607329843 [INFO] Relation Linking P: 0.6163194444444444, R: 0.6015625, micro F1: 0.6020833333333336, macro F1: 0.6088515680684248 [INFO] Type Linking P: 0.515625, R: 0.515625, micro F1: 0.515625, macro F1: 0.515625 @@ -2696,8 +2696,8 @@ Question 192: What musicians are signed up with colpix records? [INFO] Golden Answer: [http://dbpedia.org/resource/Davy_Jones_(musician), http://dbpedia.org/resource/Stu_Phillips_(composer), http://dbpedia.org/resource/Jane_Morgan, http://dbpedia.org/resource/Nina_Simone, http://dbpedia.org/resource/Ronnie_Spector, http://dbpedia.org/resource/Johnny_Loves_Me] [INFO] Predicted Answer: [http://dbpedia.org/resource/Blue_Moon_(1934_song), http://dbpedia.org/resource/Davy_Jones_(musician), http://dbpedia.org/resource/The_Jazz_Messengers, http://dbpedia.org/resource/Stu_Phillips_(composer), http://dbpedia.org/resource/Jane_Morgan, http://dbpedia.org/resource/Broadway_Soul, http://dbpedia.org/resource/Gone_with_the_Wave, http://dbpedia.org/resource/My_Coloring_Book, http://dbpedia.org/resource/Johnny_Angel_(song), http://dbpedia.org/resource/Nina_at_the_Village_Gate, http://dbpedia.org/resource/Nina_Simone, http://dbpedia.org/resource/Ronnie_Spector, http://dbpedia.org/resource/The_Ronettes, http://dbpedia.org/resource/Chad_Mitchell_Trio, http://dbpedia.org/resource/Goodbye_Cruel_World_(James_Darren_song), http://dbpedia.org/resource/The_Things_We_Did_Last_Summer, http://dbpedia.org/resource/Highlife_(album), http://dbpedia.org/resource/Johnny_Loves_Me, http://dbpedia.org/resource/Lollipops_and_Roses_(album), http://dbpedia.org/resource/Nina_Simone_Sings_Ellington, http://dbpedia.org/resource/Golden_Boy_(Art_Blakey_album), http://dbpedia.org/resource/Blue_Moon_(The_Marcels_album), http://dbpedia.org/resource/Folksy_Nina, http://dbpedia.org/resource/Forbidden_Fruit_(Nina_Simone_album), http://dbpedia.org/resource/Nina's_Choice, http://dbpedia.org/resource/Nina_Simone_at_Carnegie_Hall, http://dbpedia.org/resource/Nina_Simone_at_Newport, http://dbpedia.org/resource/Nina_Simone_at_Town_Hall, http://dbpedia.org/resource/The_Amazing_Nina_Simone, http://dbpedia.org/resource/Pepe_(song), http://dbpedia.org/resource/Teenage_Triangle, http://dbpedia.org/resource/The_Things_We_Did_Last_Summer_(album), http://dbpedia.org/resource/Shelley!] [INFO] P: 0.18181818181818182, R: 1.0, F: 0.3076923076923077 -[INFO] Cumulative metrics, sample: 193, P: 0.580, R: 0.634, macro F1: 0.584, macro F1*: 0.606 -[INFO] Current time: 1014 ms, average: 2824.91 ms/ques, entity detection: 379.55 ms/ques (linking tool: 287.87 ms/ques), relation detection: 876.39 ms/ques (similarity: 700.79 ms/ques), EDG generation: 79.46 ms/ques +[INFO] Cumulative metrics, sample: 193, P: 0.580, R: 0.634, macro F1: 0.606 +[INFO] Current question consumed time: 913 ms, average: 2680.15 ms/ques, entity detection: 361.79 ms/ques (linking tool: 276.72 ms/ques), relation detection: 856.26 ms/ques (similarity: 697.99 ms/ques), EDG generation: 78.43 ms/ques [INFO] Entity Linking P: 0.7668393782383419, R: 0.727979274611399, micro F1: 0.7400690846286702, macro F1: 0.7469042124262534 [INFO] Relation Linking P: 0.618307426597582, R: 0.6036269430051814, micro F1: 0.6041450777202075, macro F1: 0.6108789981508267 [INFO] Type Linking P: 0.5129533678756477, R: 0.5129533678756477, micro F1: 0.5129533678756477, macro F1: 0.5129533678756477 @@ -2710,8 +2710,8 @@ Question 193: Who is the person that first ascented the Cathedral Peak and Mount [INFO] Golden Answer: [http://dbpedia.org/resource/John_Muir] [INFO] Predicted Answer: [1917] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 194, P: 0.577, R: 0.631, macro F1: 0.581, macro F1*: 0.602 -[INFO] Current time: 2337 ms, average: 2822.40 ms/ques, entity detection: 380.32 ms/ques (linking tool: 287.84 ms/ques), relation detection: 874.81 ms/ques (similarity: 700.58 ms/ques), EDG generation: 79.29 ms/ques +[INFO] Cumulative metrics, sample: 194, P: 0.577, R: 0.631, macro F1: 0.602 +[INFO] Current question consumed time: 2053 ms, average: 2676.91 ms/ques, entity detection: 362.06 ms/ques (linking tool: 276.43 ms/ques), relation detection: 854.62 ms/ques (similarity: 697.80 ms/ques), EDG generation: 78.26 ms/ques [INFO] Entity Linking P: 0.7628865979381443, R: 0.7242268041237113, micro F1: 0.7362542955326461, macro F1: 0.743054190712716 [INFO] Relation Linking P: 0.6151202749140893, R: 0.6005154639175257, micro F1: 0.6010309278350519, macro F1: 0.6077301373356162 [INFO] Type Linking P: 0.5154639175257731, R: 0.5154639175257731, micro F1: 0.5154639175257731, macro F1: 0.5154639175257731 @@ -2724,8 +2724,8 @@ Question 194: What currencies are used in the country whose capital is Podgorica [INFO] Golden Answer: [Euro b@en, http://dbpedia.org/resource/Euro, http://dbpedia.org/resource/Yugoslav_dinar, http://dbpedia.org/resource/Deutsche_Mark] [INFO] Predicted Answer: [http://dbpedia.org/resource/Serbia_and_Montenegro, http://dbpedia.org/resource/Euro, http://dbpedia.org/resource/Yugoslav_dinar, http://dbpedia.org/resource/Deutsche_Mark, Euro b@en] [INFO] P: 0.8, R: 1.0, F: 0.888888888888889 -[INFO] Cumulative metrics, sample: 195, P: 0.578, R: 0.632, macro F1: 0.583, macro F1*: 0.604 -[INFO] Current time: 6159 ms, average: 2839.51 ms/ques, entity detection: 379.59 ms/ques (linking tool: 287.29 ms/ques), relation detection: 886.35 ms/ques (similarity: 711.78 ms/ques), EDG generation: 79.02 ms/ques +[INFO] Cumulative metrics, sample: 195, P: 0.578, R: 0.632, macro F1: 0.604 +[INFO] Current question consumed time: 5776 ms, average: 2692.81 ms/ques, entity detection: 361.34 ms/ques (linking tool: 275.88 ms/ques), relation detection: 865.71 ms/ques (similarity: 709.01 ms/ques), EDG generation: 78.00 ms/ques [INFO] Entity Linking P: 0.764102564102564, R: 0.7256410256410256, micro F1: 0.7376068376068377, macro F1: 0.7443753034114479 [INFO] Relation Linking P: 0.6170940170940171, R: 0.6025641025641025, micro F1: 0.6030769230769233, macro F1: 0.6097425116345929 [INFO] Type Linking P: 0.5128205128205128, R: 0.5128205128205128, micro F1: 0.5128205128205128, macro F1: 0.5128205128205128 @@ -2738,8 +2738,8 @@ Question 195: How many other home stadium are there of the soccer club whose hom [INFO] Golden Answer: [3] [INFO] Predicted Answer: [3, 6] [INFO] P: 0.5, R: 1.0, F: 0.6666666666666666 -[INFO] Cumulative metrics, sample: 196, P: 0.577, R: 0.634, macro F1: 0.583, macro F1*: 0.605 -[INFO] Current time: 4980 ms, average: 2850.43 ms/ques, entity detection: 378.78 ms/ques (linking tool: 286.49 ms/ques), relation detection: 888.41 ms/ques (similarity: 715.69 ms/ques), EDG generation: 78.90 ms/ques +[INFO] Cumulative metrics, sample: 196, P: 0.577, R: 0.634, macro F1: 0.605 +[INFO] Current question consumed time: 4767 ms, average: 2703.39 ms/ques, entity detection: 360.53 ms/ques (linking tool: 275.11 ms/ques), relation detection: 867.89 ms/ques (similarity: 713.81 ms/ques), EDG generation: 77.90 ms/ques [INFO] Entity Linking P: 0.7653061224489796, R: 0.7270408163265306, micro F1: 0.7389455782312926, macro F1: 0.7456828885400314 [INFO] Relation Linking P: 0.6190476190476191, R: 0.6045918367346939, micro F1: 0.6051020408163268, macro F1: 0.6117343393229425 [INFO] Type Linking P: 0.5102040816326531, R: 0.5102040816326531, micro F1: 0.5102040816326531, macro F1: 0.5102040816326531 @@ -2752,8 +2752,8 @@ Question 196: Where was William Anthony trained ? [INFO] Golden Answer: [http://dbpedia.org/resource/Yale_University, http://dbpedia.org/resource/Art_Students_League_of_New_York, http://dbpedia.org/resource/San_Francisco_Art_Institute] [INFO] Predicted Answer: [http://dbpedia.org/resource/Yale_University, http://dbpedia.org/resource/Art_Students_League_of_New_York, http://dbpedia.org/resource/San_Francisco_Art_Institute, Art Students League of New York@en, San Francisco Art Institute@en] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 197, P: 0.580, R: 0.636, macro F1: 0.585, macro F1*: 0.607 -[INFO] Current time: 789 ms, average: 2839.96 ms/ques, entity detection: 377.64 ms/ques (linking tool: 285.47 ms/ques), relation detection: 885.12 ms/ques (similarity: 712.84 ms/ques), EDG generation: 78.54 ms/ques +[INFO] Cumulative metrics, sample: 197, P: 0.580, R: 0.636, macro F1: 0.607 +[INFO] Current question consumed time: 745 ms, average: 2693.45 ms/ques, entity detection: 359.36 ms/ques (linking tool: 274.08 ms/ques), relation detection: 864.74 ms/ques (similarity: 710.98 ms/ques), EDG generation: 77.53 ms/ques [INFO] Entity Linking P: 0.766497461928934, R: 0.7284263959390863, micro F1: 0.7402707275803723, macro F1: 0.7469771530512871 [INFO] Relation Linking P: 0.6209813874788493, R: 0.6065989847715736, micro F1: 0.6071065989847718, macro F1: 0.6137059335938455 [INFO] Type Linking P: 0.5126903553299492, R: 0.5126903553299492, micro F1: 0.5126903553299492, macro F1: 0.5126903553299492 @@ -2766,8 +2766,8 @@ Question 197: Which composer of Motorpsycho Nitemare has also composed The time [INFO] Golden Answer: [http://dbpedia.org/resource/Bob_Dylan] [INFO] Predicted Answer: [http://dbpedia.org/resource/Bob_Dylan] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 198, P: 0.582, R: 0.638, macro F1: 0.587, macro F1*: 0.609 -[INFO] Current time: 1318 ms, average: 2832.28 ms/ques, entity detection: 376.94 ms/ques (linking tool: 284.83 ms/ques), relation detection: 883.33 ms/ques (similarity: 710.96 ms/ques), EDG generation: 78.29 ms/ques +[INFO] Cumulative metrics, sample: 198, P: 0.582, R: 0.638, macro F1: 0.609 +[INFO] Current question consumed time: 1271 ms, average: 2686.26 ms/ques, entity detection: 358.70 ms/ques (linking tool: 273.46 ms/ques), relation detection: 863.01 ms/ques (similarity: 709.12 ms/ques), EDG generation: 77.29 ms/ques [INFO] Entity Linking P: 0.7676767676767676, R: 0.7272727272727273, micro F1: 0.73989898989899, macro F1: 0.7469287469287469 [INFO] Relation Linking P: 0.6228956228956228, R: 0.6060606060606061, micro F1: 0.6074074074074077, macro F1: 0.614362806143628 [INFO] Type Linking P: 0.5151515151515151, R: 0.5151515151515151, micro F1: 0.5151515151515151, macro F1: 0.5151515151515151 @@ -2780,8 +2780,8 @@ Question 198: What river is it whose source is Dowra? [INFO] Golden Answer: [http://dbpedia.org/resource/River_Shannon] [INFO] Predicted Answer: [http://dbpedia.org/resource/River_Shannon] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 199, P: 0.584, R: 0.640, macro F1: 0.590, macro F1*: 0.611 -[INFO] Current time: 808 ms, average: 2822.11 ms/ques, entity detection: 375.43 ms/ques (linking tool: 283.66 ms/ques), relation detection: 880.56 ms/ques (similarity: 709.34 ms/ques), EDG generation: 77.96 ms/ques +[INFO] Cumulative metrics, sample: 199, P: 0.584, R: 0.640, macro F1: 0.611 +[INFO] Current question consumed time: 789 ms, average: 2676.73 ms/ques, entity detection: 357.30 ms/ques (linking tool: 272.37 ms/ques), relation detection: 860.34 ms/ques (similarity: 707.54 ms/ques), EDG generation: 76.96 ms/ques [INFO] Entity Linking P: 0.7688442211055276, R: 0.7286432160804021, micro F1: 0.7412060301507538, macro F1: 0.7482041077872584 [INFO] Relation Linking P: 0.6197654941373534, R: 0.6030150753768844, micro F1: 0.6043551088777223, macro F1: 0.6112755558614992 [INFO] Type Linking P: 0.5125628140703518, R: 0.5125628140703518, micro F1: 0.5125628140703518, macro F1: 0.5125628140703518 @@ -2794,8 +2794,8 @@ Question 199: What is the religion of the president who won the Mongolian legisl [INFO] Golden Answer: [http://dbpedia.org/resource/Buddhism] [INFO] Predicted Answer: [http://dbpedia.org/resource/Buddhism, http://dbpedia.org/resource/Islam, http://dbpedia.org/resource/Shamanism] [INFO] P: 0.3333333333333333, R: 1.0, F: 0.5 -[INFO] Cumulative metrics, sample: 200, P: 0.583, R: 0.642, macro F1: 0.589, macro F1*: 0.611 -[INFO] Current time: 2875 ms, average: 2822.38 ms/ques, entity detection: 374.04 ms/ques (linking tool: 282.47 ms/ques), relation detection: 883.49 ms/ques (similarity: 710.86 ms/ques), EDG generation: 77.72 ms/ques +[INFO] Cumulative metrics, sample: 200, P: 0.583, R: 0.642, macro F1: 0.611 +[INFO] Current question consumed time: 2791 ms, average: 2677.30 ms/ques, entity detection: 355.94 ms/ques (linking tool: 271.19 ms/ques), relation detection: 863.22 ms/ques (similarity: 709.09 ms/ques), EDG generation: 76.73 ms/ques [INFO] Entity Linking P: 0.765, R: 0.725, micro F1: 0.7375, macro F1: 0.7444630872483222 [INFO] Relation Linking P: 0.6191666666666666, R: 0.6025, micro F1: 0.6038333333333337, macro F1: 0.6107196452933152 [INFO] Type Linking P: 0.515, R: 0.515, micro F1: 0.515, macro F1: 0.515 @@ -2808,8 +2808,8 @@ Question 200: Give me everything owned by networks which is lead by Steve Burke? [INFO] Golden Answer: [1393] [INFO] Predicted Answer: [http://dbpedia.org/resource/Big_Huge_Games] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 201, P: 0.580, R: 0.638, macro F1: 0.586, macro F1*: 0.608 -[INFO] Current time: 3235 ms, average: 2824.43 ms/ques, entity detection: 373.09 ms/ques (linking tool: 281.63 ms/ques), relation detection: 884.07 ms/ques (similarity: 711.48 ms/ques), EDG generation: 77.47 ms/ques +[INFO] Cumulative metrics, sample: 201, P: 0.580, R: 0.638, macro F1: 0.608 +[INFO] Current question consumed time: 2757 ms, average: 2677.71 ms/ques, entity detection: 355.04 ms/ques (linking tool: 270.39 ms/ques), relation detection: 863.92 ms/ques (similarity: 709.71 ms/ques), EDG generation: 76.52 ms/ques [INFO] Entity Linking P: 0.7611940298507462, R: 0.7213930348258707, micro F1: 0.7338308457711443, macro F1: 0.7407592907943503 [INFO] Relation Linking P: 0.6185737976782753, R: 0.6019900497512438, micro F1: 0.6033167495854066, macro F1: 0.6101692623837335 [INFO] Type Linking P: 0.5174129353233831, R: 0.5174129353233831, micro F1: 0.5174129353233831, macro F1: 0.5174129353233831 @@ -2822,8 +2822,8 @@ Question 201: What is the municipality of Homestead Grays Bridge ? [INFO] Golden Answer: [http://dbpedia.org/resource/Pittsburgh] [INFO] Predicted Answer: [http://dbpedia.org/resource/Pittsburgh, http://dbpedia.org/resource/Homestead,_Pennsylvania] [INFO] P: 0.5, R: 1.0, F: 0.6666666666666666 -[INFO] Cumulative metrics, sample: 202, P: 0.579, R: 0.640, macro F1: 0.587, macro F1*: 0.608 -[INFO] Current time: 1096 ms, average: 2815.87 ms/ques, entity detection: 372.26 ms/ques (linking tool: 280.98 ms/ques), relation detection: 882.05 ms/ques (similarity: 709.49 ms/ques), EDG generation: 77.14 ms/ques +[INFO] Cumulative metrics, sample: 202, P: 0.579, R: 0.640, macro F1: 0.608 +[INFO] Current question consumed time: 1074 ms, average: 2669.77 ms/ques, entity detection: 354.33 ms/ques (linking tool: 269.81 ms/ques), relation detection: 861.95 ms/ques (similarity: 707.71 ms/ques), EDG generation: 76.19 ms/ques [INFO] Entity Linking P: 0.7623762376237624, R: 0.7227722772277227, micro F1: 0.7351485148514851, macro F1: 0.742046204620462 [INFO] Relation Linking P: 0.6204620462046204, R: 0.6039603960396039, micro F1: 0.6052805280528055, macro F1: 0.6121000240185743 [INFO] Type Linking P: 0.5198019801980198, R: 0.5198019801980198, micro F1: 0.5198019801980198, macro F1: 0.5198019801980198 @@ -2836,8 +2836,8 @@ Question 202: What religion do the politicians follow, who graduated from the Yo [INFO] Golden Answer: [http://dbpedia.org/resource/Buddhism] [INFO] Predicted Answer: [http://dbpedia.org/resource/Willem, http://dbpedia.org/resource/Hannes, http://dbpedia.org/resource/Mitraillette, http://dbpedia.org/resource/Cliniques_Universitaires_Saint-Luc, http://dbpedia.org/resource/Omloop_Het_Nieuwsblad, http://dbpedia.org/resource/Six_Days_of_Ghent, http://dbpedia.org/resource/Northern_Romani_dialects, http://dbpedia.org/resource/Qvarforth, http://dbpedia.org/resource/Stadsprijs_Geraardsbergen, http://dbpedia.org/resource/Heistse_Pijl, http://dbpedia.org/resource/Lotto–Belisol_Belgium_Tour, http://dbpedia.org/resource/Riffian_people, http://dbpedia.org/resource/Prince_of_Ligne, http://dbpedia.org/resource/Vlaamse_Aardbeiencross, http://dbpedia.org/resource/Vlaamse_Druivenveldrit_Overijse, http://dbpedia.org/resource/House_of_Ligne, http://dbpedia.org/resource/Schaal_Sels-Merksem, http://dbpedia.org/resource/Omloop_van_het_Waasland, http://dbpedia.org/resource/Dwars_door_de_Antwerpse_Kempen, http://dbpedia.org/resource/Halle-Buizingen, http://dbpedia.org/resource/Steenbergcross, http://dbpedia.org/resource/Cyclocross_Leuven, http://dbpedia.org/resource/Azencross, http://dbpedia.org/resource/Duinencross_Koksijde, http://dbpedia.org/resource/GP_Mario_De_Clercq, http://dbpedia.org/resource/Grote_Prijs_Stad_Eeklo, http://dbpedia.org/resource/Noordzeecross, http://dbpedia.org/resource/Scheldeprijs, http://dbpedia.org/resource/Three_Days_of_De_Panne, http://dbpedia.org/resource/Tour_of_Flanders_for_Women, http://dbpedia.org/resource/GP_Impanis-Van_Petegem, http://dbpedia.org/resource/Tour_de_Wallonie, http://dbpedia.org/resource/Bandidos_Motorcycle_Club, http://dbpedia.org/resource/Outlaws_Motorcycle_Club, http://dbpedia.org/resource/Grote_1-MeiPrijs, http://dbpedia.org/resource/Brussels_Cycling_Classic, http://dbpedia.org/resource/Canadians, http://dbpedia.org/resource/Mongols_Motorcycle_Club, http://dbpedia.org/resource/Halle–Ingooigem, http://dbpedia.org/resource/Omloop_van_het_Houtland, http://dbpedia.org/resource/Bollekescross, http://dbpedia.org/resource/Grand_Prix_De_Ster_Sint-Niklaas, http://dbpedia.org/resource/Grand_Prix_Neerpelt, http://dbpedia.org/resource/Grand_Prix_Sven_Nys, http://dbpedia.org/resource/Superprestige_Diegem, http://dbpedia.org/resource/Grand_Prix_de_Dottignies, http://dbpedia.org/resource/International_Belgian_Open, http://dbpedia.org/resource/German_Standard_German, http://dbpedia.org/resource/Tour_of_Belgium, http://dbpedia.org/resource/Kampioenschap_van_Vlaanderen, http://dbpedia.org/resource/Ronde_van_Limburg_(Belgium), http://dbpedia.org/resource/Handzame_Classic, http://dbpedia.org/resource/Memorial_Van_Coningsloo, http://dbpedia.org/resource/Zellik–Galmaarden, http://dbpedia.org/resource/Cyclocross_Vossenhol-Maldegem, http://dbpedia.org/resource/E3_Harelbeke, http://dbpedia.org/resource/Kuurne–Brussels–Kuurne, http://dbpedia.org/resource/Arrazola_de_Oñate, http://dbpedia.org/resource/Cyclo-cross_Zonhoven, http://dbpedia.org/resource/Prince_de_Chimay, http://dbpedia.org/resource/GP_Stad_Roeselare, http://dbpedia.org/resource/Omloop_van_de_Vlaamse_Scheldeboorden, http://dbpedia.org/resource/Sluitingsprijs_Oostmalle, http://dbpedia.org/resource/Cyclo-cross_Namur, http://dbpedia.org/resource/Gooik–Geraardsbergen–Gooik, http://dbpedia.org/resource/Kattekoers, http://dbpedia.org/resource/Eurométropole, http://dbpedia.org/resource/Diamond_Tour, http://dbpedia.org/resource/Antwerpse_Havenpijl, http://dbpedia.org/resource/Gooikse_Pijl, http://dbpedia.org/resource/Grand_Prix_Criquielion, http://dbpedia.org/resource/Omloop_van_het_Hageland, http://dbpedia.org/resource/Erondegemse_Pijl] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 203, P: 0.576, R: 0.637, macro F1: 0.584, macro F1*: 0.605 -[INFO] Current time: 8824 ms, average: 2845.47 ms/ques, entity detection: 371.66 ms/ques (linking tool: 280.33 ms/ques), relation detection: 902.33 ms/ques (similarity: 720.19 ms/ques), EDG generation: 76.89 ms/ques +[INFO] Cumulative metrics, sample: 203, P: 0.576, R: 0.637, macro F1: 0.605 +[INFO] Current question consumed time: 8343 ms, average: 2697.71 ms/ques, entity detection: 353.79 ms/ques (linking tool: 269.22 ms/ques), relation detection: 881.45 ms/ques (similarity: 718.54 ms/ques), EDG generation: 75.95 ms/ques [INFO] Entity Linking P: 0.7586206896551724, R: 0.7192118226600985, micro F1: 0.7315270935960592, macro F1: 0.7383908045977011 [INFO] Relation Linking P: 0.6174055829228243, R: 0.6009852216748769, micro F1: 0.6022988505747129, macro F1: 0.6090847529642958 [INFO] Type Linking P: 0.5172413793103449, R: 0.5172413793103449, micro F1: 0.5172413793103449, macro F1: 0.5172413793103449 @@ -2850,8 +2850,8 @@ Question 203: Who is the narrator of The Incredible Hulk , 1982 TV series ? [INFO] Golden Answer: [http://dbpedia.org/resource/Stan_Lee] [INFO] Predicted Answer: [http://dbpedia.org/resource/Comic_Book_Confidential, http://dbpedia.org/resource/Who_Wants_to_Be_a_Superhero%3F_(UK_TV_series), http://dbpedia.org/resource/Stan_Lee's_Mutants,_Monsters_&_Marvels, http://dbpedia.org/resource/The_Comic_Book_Greats, http://dbpedia.org/resource/Who_Wants_to_Be_a_Superhero%3F] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 204, P: 0.574, R: 0.634, macro F1: 0.581, macro F1*: 0.602 -[INFO] Current time: 2706 ms, average: 2844.78 ms/ques, entity detection: 371.12 ms/ques (linking tool: 279.82 ms/ques), relation detection: 903.07 ms/ques (similarity: 719.82 ms/ques), EDG generation: 76.63 ms/ques +[INFO] Cumulative metrics, sample: 204, P: 0.574, R: 0.634, macro F1: 0.602 +[INFO] Current question consumed time: 2481 ms, average: 2696.65 ms/ques, entity detection: 353.21 ms/ques (linking tool: 268.63 ms/ques), relation detection: 882.07 ms/ques (similarity: 718.04 ms/ques), EDG generation: 75.66 ms/ques [INFO] Entity Linking P: 0.7549019607843137, R: 0.7156862745098039, micro F1: 0.7279411764705882, macro F1: 0.7347712418300653 [INFO] Relation Linking P: 0.6143790849673202, R: 0.5980392156862745, micro F1: 0.5993464052287585, macro F1: 0.6060990433909412 [INFO] Type Linking P: 0.5196078431372549, R: 0.5196078431372549, micro F1: 0.5196078431372549, macro F1: 0.5196078431372549 @@ -2864,8 +2864,8 @@ Question 204: What sports are played in schools where Swahili is the official la [INFO] Golden Answer: [http://dbpedia.org/resource/Basketball, http://dbpedia.org/resource/Tennis, http://dbpedia.org/resource/Association_football, http://dbpedia.org/resource/Cricket, http://dbpedia.org/resource/Rugby_union, http://dbpedia.org/resource/Swimming_(sport), http://dbpedia.org/resource/Squash_(sport)] [INFO] Predicted Answer: [http://dbpedia.org/resource/Burundi, http://dbpedia.org/resource/Comoros, http://dbpedia.org/resource/Democratic_Republic_of_the_Congo, http://dbpedia.org/resource/Kenya, http://dbpedia.org/resource/Madagascar, http://dbpedia.org/resource/Malawi, http://dbpedia.org/resource/Mozambique, http://dbpedia.org/resource/Rwanda, http://dbpedia.org/resource/Somalia, http://dbpedia.org/resource/South_Sudan, http://dbpedia.org/resource/Tanzania, http://dbpedia.org/resource/Uganda, http://dbpedia.org/resource/Zambia, http://dbpedia.org/resource/Mayotte] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 205, P: 0.571, R: 0.631, macro F1: 0.578, macro F1*: 0.599 -[INFO] Current time: 1674 ms, average: 2839.08 ms/ques, entity detection: 370.07 ms/ques (linking tool: 278.98 ms/ques), relation detection: 901.78 ms/ques (similarity: 719.40 ms/ques), EDG generation: 76.41 ms/ques +[INFO] Cumulative metrics, sample: 205, P: 0.571, R: 0.631, macro F1: 0.599 +[INFO] Current question consumed time: 1487 ms, average: 2690.75 ms/ques, entity detection: 352.11 ms/ques (linking tool: 267.75 ms/ques), relation detection: 880.64 ms/ques (similarity: 717.58 ms/ques), EDG generation: 75.40 ms/ques [INFO] Entity Linking P: 0.751219512195122, R: 0.7121951219512195, micro F1: 0.724390243902439, macro F1: 0.7311869918699186 [INFO] Relation Linking P: 0.6138211382113821, R: 0.5975609756097561, micro F1: 0.5988617886178864, macro F1: 0.6055819283025045 [INFO] Type Linking P: 0.5219512195121951, R: 0.5219512195121951, micro F1: 0.5219512195121951, macro F1: 0.5219512195121951 @@ -2878,8 +2878,8 @@ Question 205: Which interest of Philip Novak is the relegion of Vesak ? [INFO] Golden Answer: [http://dbpedia.org/resource/Buddhism] [INFO] Predicted Answer: [http://dbpedia.org/resource/Buddhism] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 206, P: 0.573, R: 0.633, macro F1: 0.580, macro F1*: 0.601 -[INFO] Current time: 1154 ms, average: 2830.90 ms/ques, entity detection: 369.82 ms/ques (linking tool: 278.48 ms/ques), relation detection: 898.72 ms/ques (similarity: 716.71 ms/ques), EDG generation: 76.19 ms/ques +[INFO] Cumulative metrics, sample: 206, P: 0.573, R: 0.633, macro F1: 0.601 +[INFO] Current question consumed time: 961 ms, average: 2682.35 ms/ques, entity detection: 351.60 ms/ques (linking tool: 267.13 ms/ques), relation detection: 877.55 ms/ques (similarity: 714.89 ms/ques), EDG generation: 75.15 ms/ques [INFO] Entity Linking P: 0.7524271844660194, R: 0.7135922330097088, micro F1: 0.7257281553398058, macro F1: 0.7324953385198998 [INFO] Relation Linking P: 0.6132686084142395, R: 0.5970873786407767, micro F1: 0.598381877022654, macro F1: 0.6050698302268832 [INFO] Type Linking P: 0.5242718446601942, R: 0.5242718446601942, micro F1: 0.5242718446601942, macro F1: 0.5242718446601942 @@ -2892,8 +2892,8 @@ Question 206: Count all the ethinic group who speaks language which belongs to A [INFO] Golden Answer: [165] [INFO] Predicted Answer: [6] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 207, P: 0.570, R: 0.630, macro F1: 0.577, macro F1*: 0.598 -[INFO] Current time: 4902 ms, average: 2840.90 ms/ques, entity detection: 369.04 ms/ques (linking tool: 277.72 ms/ques), relation detection: 901.38 ms/ques (similarity: 718.29 ms/ques), EDG generation: 76.08 ms/ques +[INFO] Cumulative metrics, sample: 207, P: 0.570, R: 0.630, macro F1: 0.598 +[INFO] Current question consumed time: 3961 ms, average: 2688.53 ms/ques, entity detection: 350.72 ms/ques (linking tool: 266.33 ms/ques), relation detection: 879.70 ms/ques (similarity: 716.45 ms/ques), EDG generation: 75.00 ms/ques [INFO] Entity Linking P: 0.748792270531401, R: 0.7101449275362319, micro F1: 0.7222222222222222, macro F1: 0.728956713696132 [INFO] Relation Linking P: 0.610305958132045, R: 0.5942028985507246, micro F1: 0.5954911433172305, macro F1: 0.6021467875687824 [INFO] Type Linking P: 0.5217391304347826, R: 0.5217391304347826, micro F1: 0.5217391304347826, macro F1: 0.5217391304347826 @@ -2906,8 +2906,8 @@ Question 207: What is the area of the Tar Heel Sports Network which is official [INFO] Golden Answer: [http://dbpedia.org/resource/Virginia] [INFO] Predicted Answer: [http://dbpedia.org/resource/Poor_Mountain, http://dbpedia.org/resource/Elliott_Knob, http://dbpedia.org/resource/Old_Rag_Mountain, http://dbpedia.org/resource/Cedar_Mountain,_Virginia, http://dbpedia.org/resource/Twelve_O'clock_Knob_(Roanoke_County,_Virginia), http://dbpedia.org/resource/Memorial_Bridge_(Roanoke,_Virginia), http://dbpedia.org/resource/Furnace_Mountain_(Virginia), http://dbpedia.org/resource/Timber_Ridge, http://dbpedia.org/resource/Clinch_Mountain, http://dbpedia.org/resource/Paris_Mountain, http://dbpedia.org/resource/Charles_Hardaway_Marks_Bridges, http://dbpedia.org/resource/Humpback_Rock, http://dbpedia.org/resource/Signal_Knob_(Virginia), http://dbpedia.org/resource/High_Knob_(Blue_Ridge,_Virginia), http://dbpedia.org/resource/Salt_Pond_Mountain, http://dbpedia.org/resource/Chesapeake_Bay_Bridge–Tunnel, http://dbpedia.org/resource/Apple_Orchard_Mountain, http://dbpedia.org/resource/Hawksbill_Mountain, http://dbpedia.org/resource/Whitetop_Mountain, http://dbpedia.org/resource/Great_North_Mountain, http://dbpedia.org/resource/Devils_Backbone_(Highland_County,_Virginia), http://dbpedia.org/resource/Willis_Mountain, http://dbpedia.org/resource/Massanutten_Mountain, http://dbpedia.org/resource/Mount_Rogers, http://dbpedia.org/resource/Beartown_Mountain, http://dbpedia.org/resource/Blue_Ridge_Mountain, http://dbpedia.org/resource/Cacapon_Mountain, http://dbpedia.org/resource/Short_Hill_Mountain, http://dbpedia.org/resource/The_Priest_(mountain), http://dbpedia.org/resource/Fort_Lewis_Mountain, http://dbpedia.org/resource/Loudoun_Heights_(Mountain), http://dbpedia.org/resource/Peaks_of_Otter, http://dbpedia.org/resource/Holston_Mountain, http://dbpedia.org/resource/Short_Mountain_(Virginia), http://dbpedia.org/resource/Maintop_Mountain, http://dbpedia.org/resource/Hogback_Mountain_(Loudoun_County,_Virginia), http://dbpedia.org/resource/Elk_Pond_Mountain, http://dbpedia.org/resource/Rocky_Mountain_(Rockbridge_County,_Virginia)] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 208, P: 0.567, R: 0.627, macro F1: 0.574, macro F1*: 0.596 -[INFO] Current time: 5198 ms, average: 2852.24 ms/ques, entity detection: 368.96 ms/ques (linking tool: 277.44 ms/ques), relation detection: 909.11 ms/ques (similarity: 720.49 ms/ques), EDG generation: 76.00 ms/ques +[INFO] Cumulative metrics, sample: 208, P: 0.567, R: 0.627, macro F1: 0.596 +[INFO] Current question consumed time: 4257 ms, average: 2696.07 ms/ques, entity detection: 350.37 ms/ques (linking tool: 265.88 ms/ques), relation detection: 885.66 ms/ques (similarity: 718.68 ms/ques), EDG generation: 74.90 ms/ques [INFO] Entity Linking P: 0.75, R: 0.7115384615384616, micro F1: 0.7235576923076923, macro F1: 0.7302631578947367 [INFO] Relation Linking P: 0.6105769230769231, R: 0.5961538461538461, micro F1: 0.5964743589743592, macro F1: 0.6032791909285934 [INFO] Type Linking P: 0.5240384615384616, R: 0.5240384615384616, micro F1: 0.5240384615384616, macro F1: 0.5240384615384616 @@ -2920,8 +2920,8 @@ Question 208: Cologny and Lancy are the neighboring municipalities of which plac [INFO] Golden Answer: [http://dbpedia.org/resource/Geneva] [INFO] Predicted Answer: [http://dbpedia.org/resource/Geneva] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 209, P: 0.569, R: 0.628, macro F1: 0.577, macro F1*: 0.597 -[INFO] Current time: 2054 ms, average: 2848.42 ms/ques, entity detection: 367.80 ms/ques (linking tool: 276.43 ms/ques), relation detection: 908.42 ms/ques (similarity: 721.43 ms/ques), EDG generation: 76.01 ms/ques +[INFO] Cumulative metrics, sample: 209, P: 0.569, R: 0.628, macro F1: 0.597 +[INFO] Current question consumed time: 1823 ms, average: 2691.89 ms/ques, entity detection: 349.21 ms/ques (linking tool: 264.88 ms/ques), relation detection: 884.95 ms/ques (similarity: 719.61 ms/ques), EDG generation: 74.83 ms/ques [INFO] Entity Linking P: 0.7511961722488039, R: 0.7129186602870813, micro F1: 0.7248803827751196, macro F1: 0.731557056634456 [INFO] Relation Linking P: 0.6076555023923444, R: 0.5933014354066986, micro F1: 0.5936204146730465, macro F1: 0.6003926876227149 [INFO] Type Linking P: 0.5215311004784688, R: 0.5215311004784688, micro F1: 0.5215311004784688, macro F1: 0.5215311004784688 @@ -2934,8 +2934,8 @@ Question 209: How many artists have their works in the Sao Paolo Museum of Art? [INFO] Golden Answer: [9] [INFO] Predicted Answer: [6, 3] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 210, P: 0.567, R: 0.625, macro F1: 0.574, macro F1*: 0.595 -[INFO] Current time: 2626 ms, average: 2847.36 ms/ques, entity detection: 367.70 ms/ques (linking tool: 276.18 ms/ques), relation detection: 907.83 ms/ques (similarity: 721.79 ms/ques), EDG generation: 76.09 ms/ques +[INFO] Cumulative metrics, sample: 210, P: 0.567, R: 0.625, macro F1: 0.595 +[INFO] Current question consumed time: 2220 ms, average: 2689.65 ms/ques, entity detection: 348.74 ms/ques (linking tool: 264.46 ms/ques), relation detection: 884.03 ms/ques (similarity: 719.94 ms/ques), EDG generation: 74.91 ms/ques [INFO] Entity Linking P: 0.7476190476190476, R: 0.7095238095238096, micro F1: 0.7214285714285714, macro F1: 0.7280734516028634 [INFO] Relation Linking P: 0.6047619047619047, R: 0.5904761904761905, micro F1: 0.5907936507936511, macro F1: 0.5975336748245115 [INFO] Type Linking P: 0.5190476190476191, R: 0.5190476190476191, micro F1: 0.5190476190476191, macro F1: 0.5190476190476191 @@ -2948,8 +2948,8 @@ Question 210: VP-8 and VP-10 shares which command structure ? [INFO] Golden Answer: [http://dbpedia.org/resource/United_States_Navy] [INFO] Predicted Answer: [http://dbpedia.org/resource/United_States_Navy] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 211, P: 0.569, R: 0.627, macro F1: 0.576, macro F1*: 0.597 -[INFO] Current time: 1971 ms, average: 2843.20 ms/ques, entity detection: 368.41 ms/ques (linking tool: 276.67 ms/ques), relation detection: 905.60 ms/ques (similarity: 720.56 ms/ques), EDG generation: 76.02 ms/ques +[INFO] Cumulative metrics, sample: 211, P: 0.569, R: 0.627, macro F1: 0.597 +[INFO] Current question consumed time: 1682 ms, average: 2684.87 ms/ques, entity detection: 349.06 ms/ques (linking tool: 264.66 ms/ques), relation detection: 881.75 ms/ques (similarity: 718.71 ms/ques), EDG generation: 74.82 ms/ques [INFO] Entity Linking P: 0.7488151658767772, R: 0.7085308056872038, micro F1: 0.721169036334913, macro F1: 0.7281162100720534 [INFO] Relation Linking P: 0.6066350710900474, R: 0.5924170616113744, micro F1: 0.5927330173775673, macro F1: 0.5994417698518255 [INFO] Type Linking P: 0.5213270142180095, R: 0.5213270142180095, micro F1: 0.5213270142180095, macro F1: 0.5213270142180095 @@ -2962,8 +2962,8 @@ Question 211: What is used as money for French Southern and Antarctic Lands is a [INFO] Golden Answer: [http://dbpedia.org/resource/Euro] [INFO] Predicted Answer: [http://dbpedia.org/resource/Euro] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 212, P: 0.571, R: 0.629, macro F1: 0.578, macro F1*: 0.598 -[INFO] Current time: 2068 ms, average: 2839.55 ms/ques, entity detection: 368.85 ms/ques (linking tool: 277.10 ms/ques), relation detection: 903.49 ms/ques (similarity: 718.47 ms/ques), EDG generation: 76.18 ms/ques +[INFO] Cumulative metrics, sample: 212, P: 0.571, R: 0.629, macro F1: 0.598 +[INFO] Current question consumed time: 1706 ms, average: 2680.25 ms/ques, entity detection: 349.06 ms/ques (linking tool: 264.74 ms/ques), relation detection: 879.50 ms/ques (similarity: 716.60 ms/ques), EDG generation: 75.10 ms/ques [INFO] Entity Linking P: 0.75, R: 0.7099056603773585, micro F1: 0.7224842767295597, macro F1: 0.7294022617124393 [INFO] Relation Linking P: 0.6084905660377359, R: 0.5943396226415094, micro F1: 0.5946540880503147, macro F1: 0.6013318534961155 [INFO] Type Linking P: 0.5235849056603774, R: 0.5235849056603774, micro F1: 0.5235849056603774, macro F1: 0.5235849056603774 @@ -2976,8 +2976,8 @@ Question 212: Give me the total number of membership of the organisations which [INFO] Golden Answer: [37] [INFO] Predicted Answer: [2] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 213, P: 0.568, R: 0.626, macro F1: 0.575, macro F1*: 0.596 -[INFO] Current time: 5574 ms, average: 2852.38 ms/ques, entity detection: 371.43 ms/ques (linking tool: 279.59 ms/ques), relation detection: 907.95 ms/ques (similarity: 723.46 ms/ques), EDG generation: 76.04 ms/ques +[INFO] Cumulative metrics, sample: 213, P: 0.568, R: 0.626, macro F1: 0.596 +[INFO] Current question consumed time: 4920 ms, average: 2690.77 ms/ques, entity detection: 351.63 ms/ques (linking tool: 267.23 ms/ques), relation detection: 883.02 ms/ques (similarity: 721.53 ms/ques), EDG generation: 75.00 ms/ques [INFO] Entity Linking P: 0.7511737089201878, R: 0.7112676056338029, micro F1: 0.7237871674491393, macro F1: 0.7306761919833609 [INFO] Relation Linking P: 0.6056338028169014, R: 0.5915492957746479, micro F1: 0.5918622848200316, macro F1: 0.5985086992543497 [INFO] Type Linking P: 0.5211267605633803, R: 0.5211267605633803, micro F1: 0.5211267605633803, macro F1: 0.5211267605633803 @@ -2990,8 +2990,8 @@ Question 213: What is the content licence of MSX BASIC ? [INFO] Golden Answer: [http://dbpedia.org/resource/End-user_license_agreement, http://dbpedia.org/resource/Microsoft] [INFO] Predicted Answer: [http://dbpedia.org/resource/Microsoft, http://dbpedia.org/resource/End-user_license_agreement] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 214, P: 0.570, R: 0.628, macro F1: 0.577, macro F1*: 0.598 -[INFO] Current time: 732 ms, average: 2842.48 ms/ques, entity detection: 371.07 ms/ques (linking tool: 279.30 ms/ques), relation detection: 904.23 ms/ques (similarity: 720.39 ms/ques), EDG generation: 75.77 ms/ques +[INFO] Cumulative metrics, sample: 214, P: 0.570, R: 0.628, macro F1: 0.598 +[INFO] Current question consumed time: 646 ms, average: 2681.21 ms/ques, entity detection: 351.13 ms/ques (linking tool: 266.91 ms/ques), relation detection: 879.41 ms/ques (similarity: 718.47 ms/ques), EDG generation: 74.71 ms/ques [INFO] Entity Linking P: 0.7523364485981309, R: 0.7126168224299065, micro F1: 0.7250778816199377, macro F1: 0.7319381716823921 [INFO] Relation Linking P: 0.6074766355140186, R: 0.5934579439252337, micro F1: 0.5937694704049846, macro F1: 0.6003854685624932 [INFO] Type Linking P: 0.5233644859813084, R: 0.5233644859813084, micro F1: 0.5233644859813084, macro F1: 0.5233644859813084 @@ -3004,8 +3004,8 @@ Question 214: What are the central characters in spy kids series? [INFO] Golden Answer: [http://dbpedia.org/resource/Carmen_Cortez, http://dbpedia.org/resource/Juni_Cortez] [INFO] Predicted Answer: [http://dbpedia.org/resource/Joel_McHale, http://dbpedia.org/resource/Daryl_Sabara, http://dbpedia.org/resource/Alexa_Vega, http://dbpedia.org/resource/Mason_Cook, http://dbpedia.org/resource/Carla_Gugino, http://dbpedia.org/resource/Jessica_Alba, http://dbpedia.org/resource/Rowan_Blanchard, http://dbpedia.org/resource/Antonio_Banderas] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 215, P: 0.568, R: 0.625, macro F1: 0.574, macro F1*: 0.595 -[INFO] Current time: 946 ms, average: 2833.66 ms/ques, entity detection: 370.11 ms/ques (linking tool: 278.51 ms/ques), relation detection: 901.37 ms/ques (similarity: 717.93 ms/ques), EDG generation: 75.47 ms/ques +[INFO] Cumulative metrics, sample: 215, P: 0.568, R: 0.625, macro F1: 0.595 +[INFO] Current question consumed time: 904 ms, average: 2672.95 ms/ques, entity detection: 350.18 ms/ques (linking tool: 266.13 ms/ques), relation detection: 876.67 ms/ques (similarity: 716.00 ms/ques), EDG generation: 74.45 ms/ques [INFO] Entity Linking P: 0.7534883720930232, R: 0.713953488372093, micro F1: 0.7263565891472867, macro F1: 0.7331883684074743 [INFO] Relation Linking P: 0.6046511627906976, R: 0.5906976744186047, micro F1: 0.5910077519379847, macro F1: 0.5975929780110396 [INFO] Type Linking P: 0.5255813953488372, R: 0.5255813953488372, micro F1: 0.5255813953488372, macro F1: 0.5255813953488372 @@ -3018,8 +3018,8 @@ Question 215: Which comic characters were created by Paul Dini? [INFO] Golden Answer: [http://dbpedia.org/resource/Ventriloquist_(comics), http://dbpedia.org/resource/Roxy_Rocket, http://dbpedia.org/resource/Forerunner_(comics), http://dbpedia.org/resource/Mercy_Graves, http://dbpedia.org/resource/Derek_Powers, http://dbpedia.org/resource/Andrea_Beaumont, http://dbpedia.org/resource/Batman_(Terry_McGinnis), http://dbpedia.org/resource/Harley_Quinn, http://dbpedia.org/resource/Nora_Fries, http://dbpedia.org/resource/Condiment_King] [INFO] Predicted Answer: [http://dbpedia.org/resource/Ventriloquist_(comics), http://dbpedia.org/resource/Roxy_Rocket, http://dbpedia.org/resource/Forerunner_(comics), http://dbpedia.org/resource/Mercy_Graves, http://dbpedia.org/resource/Derek_Powers, http://dbpedia.org/resource/Andrea_Beaumont, http://dbpedia.org/resource/Batman_(Terry_McGinnis), http://dbpedia.org/resource/Harley_Quinn, http://dbpedia.org/resource/Nora_Fries, http://dbpedia.org/resource/Condiment_King] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 216, P: 0.570, R: 0.627, macro F1: 0.576, macro F1*: 0.597 -[INFO] Current time: 1092 ms, average: 2825.59 ms/ques, entity detection: 369.21 ms/ques (linking tool: 277.74 ms/ques), relation detection: 898.73 ms/ques (similarity: 716.41 ms/ques), EDG generation: 75.19 ms/ques +[INFO] Cumulative metrics, sample: 216, P: 0.570, R: 0.627, macro F1: 0.597 +[INFO] Current question consumed time: 1038 ms, average: 2665.38 ms/ques, entity detection: 349.28 ms/ques (linking tool: 265.31 ms/ques), relation detection: 874.12 ms/ques (similarity: 714.42 ms/ques), EDG generation: 74.16 ms/ques [INFO] Entity Linking P: 0.7546296296296297, R: 0.7152777777777778, micro F1: 0.7276234567901234, macro F1: 0.734426946631671 [INFO] Relation Linking P: 0.6064814814814815, R: 0.5925925925925926, micro F1: 0.5929012345679014, macro F1: 0.5994565994565995 [INFO] Type Linking P: 0.5231481481481481, R: 0.5231481481481481, micro F1: 0.5231481481481481, macro F1: 0.5231481481481481 @@ -3032,8 +3032,8 @@ Question 216: who owns the ship Victoria? [INFO] Golden Answer: [http://dbpedia.org/resource/Ferdinand_Magellan] [INFO] Predicted Answer: [] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 217, P: 0.567, R: 0.624, macro F1: 0.574, macro F1*: 0.594 -[INFO] Current time: 605 ms, average: 2815.36 ms/ques, entity detection: 368.12 ms/ques (linking tool: 276.67 ms/ques), relation detection: 895.09 ms/ques (similarity: 713.40 ms/ques), EDG generation: 74.88 ms/ques +[INFO] Cumulative metrics, sample: 217, P: 0.567, R: 0.624, macro F1: 0.594 +[INFO] Current question consumed time: 558 ms, average: 2655.67 ms/ques, entity detection: 348.27 ms/ques (linking tool: 264.28 ms/ques), relation detection: 870.49 ms/ques (similarity: 711.41 ms/ques), EDG generation: 73.85 ms/ques [INFO] Entity Linking P: 0.7511520737327189, R: 0.7119815668202765, micro F1: 0.7242703533026114, macro F1: 0.7310424906564098 [INFO] Relation Linking P: 0.6036866359447005, R: 0.5898617511520737, micro F1: 0.5901689708141323, macro F1: 0.5966941266480438 [INFO] Type Linking P: 0.5253456221198156, R: 0.5253456221198156, micro F1: 0.5253456221198156, macro F1: 0.5253456221198156 @@ -3046,8 +3046,8 @@ Question 217: Where was the predecessor of Sevan Ross born? [INFO] Golden Answer: [http://dbpedia.org/resource/Michigan] [INFO] Predicted Answer: [http://dbpedia.org/resource/United_States, http://dbpedia.org/resource/New_Kensington,_Pennsylvania] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 218, P: 0.564, R: 0.621, macro F1: 0.571, macro F1*: 0.591 -[INFO] Current time: 633 ms, average: 2805.35 ms/ques, entity detection: 367.10 ms/ques (linking tool: 275.73 ms/ques), relation detection: 891.58 ms/ques (similarity: 710.50 ms/ques), EDG generation: 74.60 ms/ques +[INFO] Cumulative metrics, sample: 218, P: 0.564, R: 0.621, macro F1: 0.591 +[INFO] Current question consumed time: 635 ms, average: 2646.40 ms/ques, entity detection: 347.29 ms/ques (linking tool: 263.39 ms/ques), relation detection: 867.06 ms/ques (similarity: 708.50 ms/ques), EDG generation: 73.57 ms/ques [INFO] Entity Linking P: 0.7522935779816514, R: 0.713302752293578, micro F1: 0.7255351681957186, macro F1: 0.7322795078319048 [INFO] Relation Linking P: 0.6009174311926605, R: 0.5871559633027523, micro F1: 0.5874617737003061, macro F1: 0.5939569976267223 [INFO] Type Linking P: 0.5275229357798165, R: 0.5275229357798165, micro F1: 0.5275229357798165, macro F1: 0.5275229357798165 @@ -3060,8 +3060,8 @@ Question 218: Which city council owns the Ted Williams tunnel and Leonard Memori [INFO] Golden Answer: [http://dbpedia.org/resource/Massachusetts] [INFO] Predicted Answer: [http://dbpedia.org/resource/Massachusetts] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 219, P: 0.566, R: 0.623, macro F1: 0.573, macro F1*: 0.593 -[INFO] Current time: 6279 ms, average: 2821.21 ms/ques, entity detection: 367.42 ms/ques (linking tool: 275.54 ms/ques), relation detection: 900.75 ms/ques (similarity: 715.72 ms/ques), EDG generation: 74.37 ms/ques +[INFO] Cumulative metrics, sample: 219, P: 0.566, R: 0.623, macro F1: 0.593 +[INFO] Current question consumed time: 4893 ms, average: 2656.66 ms/ques, entity detection: 347.61 ms/ques (linking tool: 263.17 ms/ques), relation detection: 873.00 ms/ques (similarity: 711.09 ms/ques), EDG generation: 73.33 ms/ques [INFO] Entity Linking P: 0.7534246575342466, R: 0.7123287671232876, micro F1: 0.7252663622526635, macro F1: 0.7323006017155294 [INFO] Relation Linking P: 0.6027397260273972, R: 0.589041095890411, micro F1: 0.5893455098934554, macro F1: 0.5958116831994962 [INFO] Type Linking P: 0.5296803652968036, R: 0.5296803652968036, micro F1: 0.5296803652968036, macro F1: 0.5296803652968036 @@ -3074,8 +3074,8 @@ Question 219: Where is the school which is the rival of Somerset Berkley Regiona [INFO] Golden Answer: [http://dbpedia.org/resource/New_England] [INFO] Predicted Answer: [http://dbpedia.org/resource/Huw_Jones_(rugby_union)] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 220, P: 0.564, R: 0.620, macro F1: 0.570, macro F1*: 0.590 -[INFO] Current time: 4844 ms, average: 2830.41 ms/ques, entity detection: 367.15 ms/ques (linking tool: 275.48 ms/ques), relation detection: 907.09 ms/ques (similarity: 723.49 ms/ques), EDG generation: 74.16 ms/ques +[INFO] Cumulative metrics, sample: 220, P: 0.564, R: 0.620, macro F1: 0.590 +[INFO] Current question consumed time: 3967 ms, average: 2662.61 ms/ques, entity detection: 347.42 ms/ques (linking tool: 263.17 ms/ques), relation detection: 876.34 ms/ques (similarity: 716.69 ms/ques), EDG generation: 73.13 ms/ques [INFO] Entity Linking P: 0.75, R: 0.7090909090909091, micro F1: 0.7219696969696969, macro F1: 0.7289719626168225 [INFO] Relation Linking P: 0.6, R: 0.5863636363636363, micro F1: 0.5866666666666669, macro F1: 0.593103448275862 [INFO] Type Linking P: 0.5272727272727272, R: 0.5272727272727272, micro F1: 0.5272727272727272, macro F1: 0.5272727272727272 @@ -3088,8 +3088,8 @@ Question 220: How many soccer manager have been in the Spain nation football tea [INFO] Golden Answer: [217] [INFO] Predicted Answer: [217] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 221, P: 0.566, R: 0.621, macro F1: 0.572, macro F1*: 0.592 -[INFO] Current time: 2810 ms, average: 2830.32 ms/ques, entity detection: 367.00 ms/ques (linking tool: 275.39 ms/ques), relation detection: 907.03 ms/ques (similarity: 724.83 ms/ques), EDG generation: 73.90 ms/ques +[INFO] Cumulative metrics, sample: 221, P: 0.566, R: 0.621, macro F1: 0.592 +[INFO] Current question consumed time: 2634 ms, average: 2662.48 ms/ques, entity detection: 347.17 ms/ques (linking tool: 263.04 ms/ques), relation detection: 876.27 ms/ques (similarity: 718.02 ms/ques), EDG generation: 72.90 ms/ques [INFO] Entity Linking P: 0.751131221719457, R: 0.7104072398190046, micro F1: 0.7232277526395172, macro F1: 0.7302018687922894 [INFO] Relation Linking P: 0.6018099547511312, R: 0.5882352941176471, micro F1: 0.5885369532428358, macro F1: 0.594945202415567 [INFO] Type Linking P: 0.5248868778280543, R: 0.5248868778280543, micro F1: 0.5248868778280543, macro F1: 0.5248868778280543 @@ -3102,8 +3102,8 @@ Question 221: Name the common alma mater of the Henry Morton Dexter and Jack McG [INFO] Golden Answer: [http://dbpedia.org/resource/Yale_University] [INFO] Predicted Answer: [http://dbpedia.org/resource/Yale_University] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 222, P: 0.568, R: 0.623, macro F1: 0.574, macro F1*: 0.594 -[INFO] Current time: 2316 ms, average: 2828.00 ms/ques, entity detection: 367.41 ms/ques (linking tool: 275.32 ms/ques), relation detection: 907.27 ms/ques (similarity: 725.25 ms/ques), EDG generation: 73.82 ms/ques +[INFO] Cumulative metrics, sample: 222, P: 0.568, R: 0.623, macro F1: 0.594 +[INFO] Current question consumed time: 1693 ms, average: 2658.12 ms/ques, entity detection: 347.45 ms/ques (linking tool: 262.93 ms/ques), relation detection: 874.37 ms/ques (similarity: 716.23 ms/ques), EDG generation: 72.78 ms/ques [INFO] Entity Linking P: 0.7522522522522522, R: 0.7117117117117117, micro F1: 0.7244744744744744, macro F1: 0.7314206514206514 [INFO] Relation Linking P: 0.6036036036036037, R: 0.5900900900900901, micro F1: 0.5903903903903907, macro F1: 0.5967703552609213 [INFO] Type Linking P: 0.527027027027027, R: 0.527027027027027, micro F1: 0.527027027027027, macro F1: 0.527027027027027 @@ -3116,8 +3116,8 @@ Question 222: What are some companies which are founded in Dallas Jurisdiction? [INFO] Golden Answer: [http://dbpedia.org/resource/Wyndham_Hotels_&_Resorts, http://dbpedia.org/resource/Michaels, http://dbpedia.org/resource/Rosewood_Hotels_&_Resorts, http://dbpedia.org/resource/Viewzi, http://dbpedia.org/resource/Ensco_plc, http://dbpedia.org/resource/Harwood_International, http://dbpedia.org/resource/The_Container_Store, http://dbpedia.org/resource/Credit_Solutions_of_America, http://dbpedia.org/resource/Beck_Technology, http://dbpedia.org/resource/Heritage_Auctions, http://dbpedia.org/resource/Mary_Kay, http://dbpedia.org/resource/Hitachi_Consulting, http://dbpedia.org/resource/Sevin_Rosen_Funds, http://dbpedia.org/resource/Borden_Milk_Products, http://dbpedia.org/resource/Brook_Mays, http://dbpedia.org/resource/Room_Key, http://dbpedia.org/resource/Susan_G._Komen_for_the_Cure, http://dbpedia.org/resource/GameStop, http://dbpedia.org/resource/Abeona_Therapeutics, http://dbpedia.org/resource/Rave_Cinemas, http://dbpedia.org/resource/Pizza_Inn, http://dbpedia.org/resource/TuneIn, http://dbpedia.org/resource/I_Can't_Believe_It's_Yogurt!, http://dbpedia.org/resource/Dave_&_Buster's, http://dbpedia.org/resource/Luminant, http://dbpedia.org/resource/La_Madeleine_(restaurant_chain), http://dbpedia.org/resource/Tin_Star_(restaurant), http://dbpedia.org/resource/North_Texas_Church_of_Freethought, http://dbpedia.org/resource/Energy_Future_Holdings, http://dbpedia.org/resource/Trinity_Industries, http://dbpedia.org/resource/Talenti, http://dbpedia.org/resource/Golden_Harvest_Seeds, http://dbpedia.org/resource/Corporate_Magic, http://dbpedia.org/resource/Sports_Media,_Inc., http://dbpedia.org/resource/Sonny_Bryan's_Smokehouse, http://dbpedia.org/resource/CardLab, http://dbpedia.org/resource/Magic_Logix, http://dbpedia.org/resource/R._William_Funk_&_Associates] [INFO] Predicted Answer: [http://dbpedia.org/resource/Akhil_Amar] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 223, P: 0.565, R: 0.620, macro F1: 0.572, macro F1*: 0.591 -[INFO] Current time: 573 ms, average: 2817.89 ms/ques, entity detection: 366.37 ms/ques (linking tool: 274.50 ms/ques), relation detection: 903.82 ms/ques (similarity: 722.52 ms/ques), EDG generation: 73.57 ms/ques +[INFO] Cumulative metrics, sample: 223, P: 0.565, R: 0.620, macro F1: 0.591 +[INFO] Current question consumed time: 540 ms, average: 2648.62 ms/ques, entity detection: 346.43 ms/ques (linking tool: 262.08 ms/ques), relation detection: 871.06 ms/ques (similarity: 713.61 ms/ques), EDG generation: 72.53 ms/ques [INFO] Entity Linking P: 0.7488789237668162, R: 0.7085201793721974, micro F1: 0.7212257100149476, macro F1: 0.7281407381855813 [INFO] Relation Linking P: 0.600896860986547, R: 0.5874439461883408, micro F1: 0.5877428998505234, macro F1: 0.5940942550131144 [INFO] Type Linking P: 0.5246636771300448, R: 0.5246636771300448, micro F1: 0.5246636771300448, macro F1: 0.5246636771300448 @@ -3130,8 +3130,8 @@ Question 223: How many bands collaborated with artists who signed up with EMI? [INFO] Golden Answer: [1303] [INFO] Predicted Answer: [1] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 224, P: 0.563, R: 0.618, macro F1: 0.569, macro F1*: 0.589 -[INFO] Current time: 2750 ms, average: 2817.58 ms/ques, entity detection: 367.53 ms/ques (linking tool: 275.61 ms/ques), relation detection: 902.71 ms/ques (similarity: 720.54 ms/ques), EDG generation: 73.34 ms/ques +[INFO] Cumulative metrics, sample: 224, P: 0.563, R: 0.618, macro F1: 0.589 +[INFO] Current question consumed time: 2516 ms, average: 2648.03 ms/ques, entity detection: 347.35 ms/ques (linking tool: 263.06 ms/ques), relation detection: 869.89 ms/ques (similarity: 711.67 ms/ques), EDG generation: 72.34 ms/ques [INFO] Entity Linking P: 0.75, R: 0.7098214285714286, micro F1: 0.722470238095238, macro F1: 0.7293577981651376 [INFO] Relation Linking P: 0.5982142857142857, R: 0.5848214285714286, micro F1: 0.5851190476190479, macro F1: 0.5914420485175201 [INFO] Type Linking P: 0.5223214285714286, R: 0.5223214285714286, micro F1: 0.5223214285714286, macro F1: 0.5223214285714286 @@ -3144,8 +3144,8 @@ Question 224: What is the purpose of New Zealand Chess Federation ? [INFO] Golden Answer: [http://dbpedia.org/resource/Chess] [INFO] Predicted Answer: [Chess, http://dbpedia.org/resource/Chess] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 225, P: 0.565, R: 0.619, macro F1: 0.571, macro F1*: 0.591 -[INFO] Current time: 523 ms, average: 2807.39 ms/ques, entity detection: 366.37 ms/ques (linking tool: 274.72 ms/ques), relation detection: 899.27 ms/ques (similarity: 717.68 ms/ques), EDG generation: 73.05 ms/ques +[INFO] Cumulative metrics, sample: 225, P: 0.565, R: 0.619, macro F1: 0.591 +[INFO] Current question consumed time: 432 ms, average: 2638.18 ms/ques, entity detection: 346.16 ms/ques (linking tool: 262.17 ms/ques), relation detection: 866.58 ms/ques (similarity: 708.86 ms/ques), EDG generation: 72.02 ms/ques [INFO] Entity Linking P: 0.7511111111111111, R: 0.7111111111111111, micro F1: 0.7237037037037036, macro F1: 0.730563998649105 [INFO] Relation Linking P: 0.6, R: 0.5866666666666667, micro F1: 0.5869629629629632, macro F1: 0.5932584269662922 [INFO] Type Linking P: 0.5244444444444445, R: 0.5244444444444445, micro F1: 0.5244444444444445, macro F1: 0.5244444444444445 @@ -3158,8 +3158,8 @@ Question 225: How many TV shows are of the company which are of company which is [INFO] Golden Answer: [255] [INFO] Predicted Answer: [1] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 226, P: 0.562, R: 0.617, macro F1: 0.569, macro F1*: 0.588 -[INFO] Current time: 3911 ms, average: 2812.27 ms/ques, entity detection: 365.91 ms/ques (linking tool: 274.18 ms/ques), relation detection: 900.44 ms/ques (similarity: 718.39 ms/ques), EDG generation: 72.98 ms/ques +[INFO] Cumulative metrics, sample: 226, P: 0.562, R: 0.617, macro F1: 0.588 +[INFO] Current question consumed time: 3003 ms, average: 2639.79 ms/ques, entity detection: 345.45 ms/ques (linking tool: 261.50 ms/ques), relation detection: 867.43 ms/ques (similarity: 709.67 ms/ques), EDG generation: 71.92 ms/ques [INFO] Entity Linking P: 0.7522123893805309, R: 0.7123893805309734, micro F1: 0.7249262536873156, macro F1: 0.7317594845333263 [INFO] Relation Linking P: 0.6002949852507374, R: 0.588495575221239, micro F1: 0.5879056047197643, macro F1: 0.5943367223698368 [INFO] Type Linking P: 0.5221238938053098, R: 0.5221238938053098, micro F1: 0.5221238938053098, macro F1: 0.5221238938053098 @@ -3172,8 +3172,8 @@ Question 226: Who operates twin Cities 400 ? [INFO] Golden Answer: [http://dbpedia.org/resource/Chicago] [INFO] Predicted Answer: [Chicago and North Western Railway@en] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 227, P: 0.560, R: 0.614, macro F1: 0.566, macro F1*: 0.585 -[INFO] Current time: 1036 ms, average: 2804.44 ms/ques, entity detection: 364.90 ms/ques (linking tool: 273.49 ms/ques), relation detection: 897.44 ms/ques (similarity: 715.82 ms/ques), EDG generation: 72.68 ms/ques +[INFO] Cumulative metrics, sample: 227, P: 0.560, R: 0.614, macro F1: 0.585 +[INFO] Current question consumed time: 923 ms, average: 2632.23 ms/ques, entity detection: 344.50 ms/ques (linking tool: 260.85 ms/ques), relation detection: 864.48 ms/ques (similarity: 707.12 ms/ques), EDG generation: 71.62 ms/ques [INFO] Entity Linking P: 0.7533039647577092, R: 0.7136563876651982, micro F1: 0.7261380323054331, macro F1: 0.7329443981426359 [INFO] Relation Linking P: 0.5976505139500734, R: 0.5859030837004405, micro F1: 0.5853157121879592, macro F1: 0.5917184989232737 [INFO] Type Linking P: 0.5242290748898678, R: 0.5242290748898678, micro F1: 0.5242290748898678, macro F1: 0.5242290748898678 @@ -3186,8 +3186,8 @@ Question 227: What is the city of the Maine School of Science and Mathematics is [INFO] Golden Answer: [http://dbpedia.org/resource/Maine] [INFO] Predicted Answer: [http://dbpedia.org/resource/Maine, http://dbpedia.org/resource/Mount_Hope_Cemetery_(Bangor,_Maine)] [INFO] P: 0.5, R: 1.0, F: 0.6666666666666666 -[INFO] Cumulative metrics, sample: 228, P: 0.559, R: 0.616, macro F1: 0.566, macro F1*: 0.586 -[INFO] Current time: 2100 ms, average: 2801.36 ms/ques, entity detection: 365.81 ms/ques (linking tool: 274.20 ms/ques), relation detection: 895.79 ms/ques (similarity: 714.11 ms/ques), EDG generation: 72.59 ms/ques +[INFO] Cumulative metrics, sample: 228, P: 0.559, R: 0.616, macro F1: 0.586 +[INFO] Current question consumed time: 1886 ms, average: 2628.96 ms/ques, entity detection: 345.32 ms/ques (linking tool: 261.55 ms/ques), relation detection: 862.89 ms/ques (similarity: 705.57 ms/ques), EDG generation: 71.53 ms/ques [INFO] Entity Linking P: 0.7543859649122807, R: 0.7127192982456141, micro F1: 0.725877192982456, macro F1: 0.7329609524558781 [INFO] Relation Linking P: 0.5950292397660818, R: 0.5833333333333334, micro F1: 0.5827485380116962, macro F1: 0.5891232423490488 [INFO] Type Linking P: 0.5263157894736842, R: 0.5263157894736842, micro F1: 0.5263157894736842, macro F1: 0.5263157894736842 @@ -3200,8 +3200,8 @@ Question 228: Give me a count of royalties buried in Rome ? [INFO] Golden Answer: [9] [INFO] Predicted Answer: [3] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 229, P: 0.557, R: 0.613, macro F1: 0.564, macro F1*: 0.583 -[INFO] Current time: 9645 ms, average: 2831.24 ms/ques, entity detection: 369.24 ms/ques (linking tool: 277.11 ms/ques), relation detection: 908.07 ms/ques (similarity: 727.22 ms/ques), EDG generation: 72.38 ms/ques +[INFO] Cumulative metrics, sample: 229, P: 0.557, R: 0.613, macro F1: 0.583 +[INFO] Current question consumed time: 8787 ms, average: 2655.85 ms/ques, entity detection: 348.49 ms/ques (linking tool: 264.31 ms/ques), relation detection: 873.37 ms/ques (similarity: 718.71 ms/ques), EDG generation: 71.31 ms/ques [INFO] Entity Linking P: 0.7554585152838428, R: 0.7139737991266376, micro F1: 0.7270742358078601, macro F1: 0.7341305631435858 [INFO] Relation Linking P: 0.5924308588064047, R: 0.5807860262008734, micro F1: 0.5802037845705971, macro F1: 0.5865506517711053 [INFO] Type Linking P: 0.5240174672489083, R: 0.5240174672489083, micro F1: 0.5240174672489083, macro F1: 0.5240174672489083 @@ -3214,8 +3214,8 @@ Question 229: In how many places are Marjas found? [INFO] Golden Answer: [15] [INFO] Predicted Answer: [2] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 230, P: 0.554, R: 0.610, macro F1: 0.562, macro F1*: 0.581 -[INFO] Current time: 1314 ms, average: 2824.65 ms/ques, entity detection: 368.30 ms/ques (linking tool: 276.26 ms/ques), relation detection: 905.82 ms/ques (similarity: 725.93 ms/ques), EDG generation: 72.10 ms/ques +[INFO] Cumulative metrics, sample: 230, P: 0.554, R: 0.610, macro F1: 0.581 +[INFO] Current question consumed time: 1161 ms, average: 2649.35 ms/ques, entity detection: 347.45 ms/ques (linking tool: 263.43 ms/ques), relation detection: 871.12 ms/ques (similarity: 717.38 ms/ques), EDG generation: 71.07 ms/ques [INFO] Entity Linking P: 0.7565217391304347, R: 0.7152173913043478, micro F1: 0.7282608695652173, macro F1: 0.735289962109049 [INFO] Relation Linking P: 0.5898550724637681, R: 0.5782608695652174, micro F1: 0.5776811594202901, macro F1: 0.5840004315460136 [INFO] Type Linking P: 0.5217391304347826, R: 0.5217391304347826, micro F1: 0.5217391304347826, macro F1: 0.5217391304347826 @@ -3228,8 +3228,8 @@ Question 230: List the soccer players whose current team has the ground Stoke-on [INFO] Golden Answer: [http://dbpedia.org/resource/Charlie_Adam, http://dbpedia.org/resource/Peter_Crouch, http://dbpedia.org/resource/Carl_Dickinson, http://dbpedia.org/resource/Joselu, http://dbpedia.org/resource/Geoff_Cameron, http://dbpedia.org/resource/Bojan_Krkić, http://dbpedia.org/resource/Erik_Pieters, http://dbpedia.org/resource/Glen_Johnson_(English_footballer), http://dbpedia.org/resource/Ibrahim_Afellay, http://dbpedia.org/resource/Richard_Duffy, http://dbpedia.org/resource/Mark_Hughes, http://dbpedia.org/resource/Jack_Butland, http://dbpedia.org/resource/Chris_Neal, http://dbpedia.org/resource/Louis_Dodds, http://dbpedia.org/resource/Shay_Given, http://dbpedia.org/resource/Mame_Biram_Diouf, http://dbpedia.org/resource/Xherdan_Shaqiri, http://dbpedia.org/resource/Phil_Bardsley, http://dbpedia.org/resource/Theo_Robinson, http://dbpedia.org/resource/Peter_Odemwingie, http://dbpedia.org/resource/Stephen_Ireland, http://dbpedia.org/resource/Earl_Barrett, http://dbpedia.org/resource/Ryan_Shawcross, http://dbpedia.org/resource/Glenn_Whelan, http://dbpedia.org/resource/AJ_Leitch-Smith, http://dbpedia.org/resource/Achille_Campion, http://dbpedia.org/resource/Adam_Yates_(footballer), http://dbpedia.org/resource/Andy_Quy, http://dbpedia.org/resource/Anthony_Grant_(footballer), http://dbpedia.org/resource/Ben_Purkiss, http://dbpedia.org/resource/Byron_Moore, http://dbpedia.org/resource/Chris_Birchall, http://dbpedia.org/resource/Daniel_Bachmann, http://dbpedia.org/resource/Dionatan_Teixeira, http://dbpedia.org/resource/Dom_Telford, http://dbpedia.org/resource/Eddy_Lecygne, http://dbpedia.org/resource/Enoch_Andoh, http://dbpedia.org/resource/Giannelli_Imbula, http://dbpedia.org/resource/Glyn_Hodges, http://dbpedia.org/resource/JJ_Hooper, http://dbpedia.org/resource/Jak_Alnwick, http://dbpedia.org/resource/Jakob_Haugaard, http://dbpedia.org/resource/Jonathan_Walters, http://dbpedia.org/resource/Marc_Muniesa, http://dbpedia.org/resource/Marc_Wilson_(Irish_footballer), http://dbpedia.org/resource/Mark_Waddington_(footballer), http://dbpedia.org/resource/Marko_Arnautović, http://dbpedia.org/resource/Matty_Kennedy, http://dbpedia.org/resource/Michael_Brown_(footballer,_born_1977), http://dbpedia.org/resource/Michael_O'Connor_(footballer,_born_1987), http://dbpedia.org/resource/Ollie_Shenton, http://dbpedia.org/resource/Philipp_Wollscheid, http://dbpedia.org/resource/Remie_Streete, http://dbpedia.org/resource/Ryan_Inniss, http://dbpedia.org/resource/Ryan_McGivern, http://dbpedia.org/resource/Sam_Foley, http://dbpedia.org/resource/Sam_Kelly_(footballer)] [INFO] Predicted Answer: [England@en] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 231, P: 0.552, R: 0.608, macro F1: 0.559, macro F1*: 0.578 -[INFO] Current time: 1301 ms, average: 2818.05 ms/ques, entity detection: 367.64 ms/ques (linking tool: 275.65 ms/ques), relation detection: 902.92 ms/ques (similarity: 723.78 ms/ques), EDG generation: 71.94 ms/ques +[INFO] Cumulative metrics, sample: 231, P: 0.552, R: 0.608, macro F1: 0.578 +[INFO] Current question consumed time: 1089 ms, average: 2642.59 ms/ques, entity detection: 346.77 ms/ques (linking tool: 262.83 ms/ques), relation detection: 868.29 ms/ques (similarity: 715.35 ms/ques), EDG generation: 70.91 ms/ques [INFO] Entity Linking P: 0.7532467532467533, R: 0.7121212121212122, micro F1: 0.725108225108225, macro F1: 0.7321068886800054 [INFO] Relation Linking P: 0.5873015873015872, R: 0.5757575757575758, micro F1: 0.5751803751803755, macro F1: 0.5814722911497104 [INFO] Type Linking P: 0.5194805194805194, R: 0.5194805194805194, micro F1: 0.5194805194805194, macro F1: 0.5194805194805194 @@ -3242,8 +3242,8 @@ Question 231: Which are the countires where Pioneer Corporation has flourished? [INFO] Golden Answer: [http://dbpedia.org/resource/Hong_Kong, http://dbpedia.org/resource/Italy, http://dbpedia.org/resource/Japan, http://dbpedia.org/resource/Malaysia, http://dbpedia.org/resource/Singapore, http://dbpedia.org/resource/South_Korea, http://dbpedia.org/resource/Taiwan, http://dbpedia.org/resource/Thailand] [INFO] Predicted Answer: [http://dbpedia.org/resource/Hong_Kong, http://dbpedia.org/resource/Italy, http://dbpedia.org/resource/Japan, http://dbpedia.org/resource/Malaysia, http://dbpedia.org/resource/Singapore, http://dbpedia.org/resource/South_Korea, http://dbpedia.org/resource/Taiwan, http://dbpedia.org/resource/Thailand] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 232, P: 0.554, R: 0.609, macro F1: 0.561, macro F1*: 0.580 -[INFO] Current time: 1513 ms, average: 2812.43 ms/ques, entity detection: 367.32 ms/ques (linking tool: 275.30 ms/ques), relation detection: 901.05 ms/ques (similarity: 722.98 ms/ques), EDG generation: 71.70 ms/ques +[INFO] Cumulative metrics, sample: 232, P: 0.554, R: 0.609, macro F1: 0.580 +[INFO] Current question consumed time: 1438 ms, average: 2637.40 ms/ques, entity detection: 346.48 ms/ques (linking tool: 262.50 ms/ques), relation detection: 866.47 ms/ques (similarity: 714.50 ms/ques), EDG generation: 70.68 ms/ques [INFO] Entity Linking P: 0.7543103448275862, R: 0.7133620689655172, micro F1: 0.7262931034482757, macro F1: 0.7332649754417945 [INFO] Relation Linking P: 0.5890804597701149, R: 0.5775862068965517, micro F1: 0.5770114942528738, macro F1: 0.5832767113979956 [INFO] Type Linking P: 0.521551724137931, R: 0.521551724137931, micro F1: 0.521551724137931, macro F1: 0.521551724137931 @@ -3256,8 +3256,8 @@ Question 232: Who is the president of Carl Stokes nad Wyche Fowler ? [INFO] Golden Answer: [http://dbpedia.org/resource/Bill_Clinton] [INFO] Predicted Answer: [http://dbpedia.org/resource/Bill_Clinton, http://dbpedia.org/resource/George_W._Bush] [INFO] P: 0.5, R: 1.0, F: 0.6666666666666666 -[INFO] Cumulative metrics, sample: 233, P: 0.554, R: 0.611, macro F1: 0.561, macro F1*: 0.581 -[INFO] Current time: 885 ms, average: 2804.16 ms/ques, entity detection: 366.28 ms/ques (linking tool: 274.55 ms/ques), relation detection: 898.52 ms/ques (similarity: 720.78 ms/ques), EDG generation: 71.48 ms/ques +[INFO] Cumulative metrics, sample: 233, P: 0.554, R: 0.611, macro F1: 0.581 +[INFO] Current question consumed time: 798 ms, average: 2629.51 ms/ques, entity detection: 345.52 ms/ques (linking tool: 261.80 ms/ques), relation detection: 864.03 ms/ques (similarity: 712.33 ms/ques), EDG generation: 70.44 ms/ques [INFO] Entity Linking P: 0.7553648068669528, R: 0.7124463519313304, micro F1: 0.7260371959942774, macro F1: 0.7332781166076852 [INFO] Relation Linking P: 0.5908440629470672, R: 0.5793991416309013, micro F1: 0.5788268955650933, macro F1: 0.585065636903575 [INFO] Type Linking P: 0.5236051502145923, R: 0.5236051502145923, micro F1: 0.5236051502145923, macro F1: 0.5236051502145923 @@ -3270,8 +3270,8 @@ Question 233: Is God Save the Queen the national anthem of United Kingdom? [INFO] Golden Answer: [true] [INFO] Predicted Answer: [true] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 234, P: 0.556, R: 0.613, macro F1: 0.563, macro F1*: 0.583 -[INFO] Current time: 9858 ms, average: 2834.30 ms/ques, entity detection: 389.65 ms/ques (linking tool: 297.62 ms/ques), relation detection: 896.78 ms/ques (similarity: 718.96 ms/ques), EDG generation: 71.24 ms/ques +[INFO] Cumulative metrics, sample: 234, P: 0.556, R: 0.613, macro F1: 0.583 +[INFO] Current question consumed time: 8412 ms, average: 2654.22 ms/ques, entity detection: 365.01 ms/ques (linking tool: 281.13 ms/ques), relation detection: 862.33 ms/ques (similarity: 710.56 ms/ques), EDG generation: 70.20 ms/ques [INFO] Entity Linking P: 0.7564102564102564, R: 0.7136752136752137, micro F1: 0.727207977207977, macro F1: 0.7344215861657721 [INFO] Relation Linking P: 0.5925925925925926, R: 0.5811965811965812, micro F1: 0.5806267806267809, macro F1: 0.5868392664509169 [INFO] Type Linking P: 0.5256410256410257, R: 0.5256410256410257, micro F1: 0.5256410256410257, macro F1: 0.5256410256410257 @@ -3284,8 +3284,8 @@ Question 234: List few Swedish language movies ? [INFO] Golden Answer: [http://dbpedia.org/resource/Ship_of_Theseus_(film), http://dbpedia.org/resource/Aida_(1987_film), http://dbpedia.org/resource/Emil_&_Ida_i_Lönneberga, http://dbpedia.org/resource/Karlsson_på_taket_(film), http://dbpedia.org/resource/Lotta_flyttar_hemifrån, http://dbpedia.org/resource/Lotta_på_Bråkmakargatan, http://dbpedia.org/resource/Pippi_Longstocking_(1969_film), http://dbpedia.org/resource/Pippi_in_the_South_Seas_(film), http://dbpedia.org/resource/Pippi_on_the_Run, http://dbpedia.org/resource/Ronia_the_Robber's_Daughter, http://dbpedia.org/resource/The_Brothers_Lionheart, http://dbpedia.org/resource/Flight_of_the_Eagle, http://dbpedia.org/resource/Hamilton_(1998_film), http://dbpedia.org/resource/Morsarvet, http://dbpedia.org/resource/Frida,_en_trotjänarinna, http://dbpedia.org/resource/Thunder_and_Lightning_(1938_film), http://dbpedia.org/resource/Deadline_(2001_film), http://dbpedia.org/resource/Jönssonligan_får_guldfeber, http://dbpedia.org/resource/Jönssonligan_och_Dynamit-Harry, http://dbpedia.org/resource/Varning_för_Jönssonligan, http://dbpedia.org/resource/Tjenare_kungen, http://dbpedia.org/resource/My_Sister,_My_Love_(film), http://dbpedia.org/resource/I_Miss_You,_I_Miss_You!, http://dbpedia.org/resource/Ocean_Breakers, http://dbpedia.org/resource/Beyond_the_Front_Line, http://dbpedia.org/resource/Den_store_amatören, http://dbpedia.org/resource/Peter-No-Tail_(1997_film), http://dbpedia.org/resource/Stall-Erik_och_snapphanarna, http://dbpedia.org/resource/Fanny_and_Alexander, http://dbpedia.org/resource/The_Island_at_the_Top_of_the_World, http://dbpedia.org/resource/The_Virgin_Spring, http://dbpedia.org/resource/Insomnia_(1997_film), http://dbpedia.org/resource/Ebba_the_Movie, http://dbpedia.org/resource/Badoo, http://dbpedia.org/resource/Pusher_(1996_film), http://dbpedia.org/resource/Frostbite_(2006_film), http://dbpedia.org/resource/1939_(1989_film), http://dbpedia.org/resource/Jönssonligan_spelar_högt, http://dbpedia.org/resource/Making_Babies, http://dbpedia.org/resource/Suddenly_(2006_film), http://dbpedia.org/resource/Jag_är_din_krigare, http://dbpedia.org/resource/Desmond_&_the_Swamp_Barbarian_Trap, http://dbpedia.org/resource/Lilla_Märta_kommer_tillbaka, http://dbpedia.org/resource/Each_to_His_Own_Way, http://dbpedia.org/resource/Sjätte_skottet, http://dbpedia.org/resource/The_Nuthouse_(film), http://dbpedia.org/resource/Into_A_Raging_Blaze, http://dbpedia.org/resource/Håkan_Bråkan_&_Josef, http://dbpedia.org/resource/To_Kill_a_Child_(1953_film), http://dbpedia.org/resource/30:e_november, http://dbpedia.org/resource/Nattbuss_807, http://dbpedia.org/resource/Katrina_(novel), http://dbpedia.org/resource/Elvira_Madigan_(1943_film), http://dbpedia.org/resource/I_manegen_med_Glenn_Killing, http://dbpedia.org/resource/Dragonetti:_The_Ruthless_Contract_Killer, http://dbpedia.org/resource/Egen_ingång, http://dbpedia.org/resource/Baldevins_bröllop, http://dbpedia.org/resource/On_the_Loose_(1985_film), http://dbpedia.org/resource/Jönssonligan_dyker_upp_igen, http://dbpedia.org/resource/Jönssonligan_på_Mallorca, http://dbpedia.org/resource/Lilla_Jönssonligan, http://dbpedia.org/resource/Darling_(2007_Swedish_film), http://dbpedia.org/resource/Den_ofrivillige_golfaren, http://dbpedia.org/resource/Stade_81, http://dbpedia.org/resource/Hotel_St._Pauli, http://dbpedia.org/resource/Charlotte_Löwensköld_(1930_film), http://dbpedia.org/resource/Ride_Tonight!, http://dbpedia.org/resource/Under_False_Flag_(1935_film), http://dbpedia.org/resource/The_Emperor_of_Portugallia, http://dbpedia.org/resource/Blodsbröder, http://dbpedia.org/resource/Jönssonligan_och_den_svarta_diamanten, http://dbpedia.org/resource/Fäbodjäntan, http://dbpedia.org/resource/Sandor_slash_Ida_(film), http://dbpedia.org/resource/Hur_många_kramar_finns_det_i_världen%3F, http://dbpedia.org/resource/You,_the_Living, http://dbpedia.org/resource/Arn_–_The_Kingdom_at_Road's_End, http://dbpedia.org/resource/Arn_–_The_Knight_Templar, http://dbpedia.org/resource/Show_Me_Love_(film), http://dbpedia.org/resource/Elvira_Madigan_(1967_film), http://dbpedia.org/resource/Joe_Hill_(film), http://dbpedia.org/resource/Vandring_med_månen, http://dbpedia.org/resource/Bang_Bang_Orangutang, http://dbpedia.org/resource/Blackjack_(1990_film), http://dbpedia.org/resource/Sleepwalker_(2000_film), http://dbpedia.org/resource/The_Other_Sport, http://dbpedia.org/resource/Giliap, http://dbpedia.org/resource/Miss_Kicki, http://dbpedia.org/resource/Repmånad, http://dbpedia.org/resource/Storm_(2005_film), http://dbpedia.org/resource/Zozo, http://dbpedia.org/resource/Credo_(1997_film), http://dbpedia.org/resource/Family_Matters_(film), http://dbpedia.org/resource/Livets_vår, http://dbpedia.org/resource/9_millimeter, http://dbpedia.org/resource/Solstorm, http://dbpedia.org/resource/Sökarna:_Återkomsten, http://dbpedia.org/resource/Banketten, http://dbpedia.org/resource/Flickan_från_tredje_raden, http://dbpedia.org/resource/Kungliga_patrasket, http://dbpedia.org/resource/Kärlekens_decimaler, http://dbpedia.org/resource/Agaton_Sax_och_Byköpings_gästabud, http://dbpedia.org/resource/Flames_in_the_Dark, http://dbpedia.org/resource/Fröken_Chic, http://dbpedia.org/resource/Hjärter_Knekt, http://dbpedia.org/resource/Love_Goes_Up_and_Down, http://dbpedia.org/resource/Lyckan_kommer, http://dbpedia.org/resource/Med_glorian_på_sned, http://dbpedia.org/resource/Meeting_in_the_Night, http://dbpedia.org/resource/Ombyte_av_tåg, http://dbpedia.org/resource/One_Swallow_Does_Not_Make_a_Summer, http://dbpedia.org/resource/På_en_bänk_i_en_park, http://dbpedia.org/resource/Rififi_in_Stockholm, http://dbpedia.org/resource/Som_folk_är_mest, http://dbpedia.org/resource/Stopp!_Tänk_på_något_annat, http://dbpedia.org/resource/The_White_Cat_(film), http://dbpedia.org/resource/We_Three_Debutantes, http://dbpedia.org/resource/You_Are_My_Adventure, http://dbpedia.org/resource/Varannan_vecka, http://dbpedia.org/resource/Sällskapsresan_2_–_Snowroller, http://dbpedia.org/resource/Blonde_in_Bondage, http://dbpedia.org/resource/Kärleksön, http://dbpedia.org/resource/Hamsun_(film), http://dbpedia.org/resource/Offside_(2006_Swedish_film), http://dbpedia.org/resource/Pippi_Goes_on_Board_(film), http://dbpedia.org/resource/Jönssonligans_största_kupp, http://dbpedia.org/resource/Peter-No-Tail_in_Americat, http://dbpedia.org/resource/Rapport_till_himlen, http://dbpedia.org/resource/After_the_Wedding, http://dbpedia.org/resource/Sökarna, http://dbpedia.org/resource/The_Architecture_of_Doom, http://dbpedia.org/resource/Resan_till_dej, http://dbpedia.org/resource/Swinging_at_the_Castle, http://dbpedia.org/resource/Space_Invasion_of_Lapland, http://dbpedia.org/resource/Executive_Protection_(film), http://dbpedia.org/resource/Kronjuvelerna, http://dbpedia.org/resource/Med_dej_i_mina_armar, http://dbpedia.org/resource/Love_Is_the_Only_Master_I'll_Serve, http://dbpedia.org/resource/Fram_för_lilla_Märta, http://dbpedia.org/resource/Ålder_okänd, http://dbpedia.org/resource/Excellensen, http://dbpedia.org/resource/Zero_Tolerance_(1999_film), http://dbpedia.org/resource/Lilla_Jönssonligan_och_cornflakeskuppen, http://dbpedia.org/resource/Lilla_Jönssonligan_på_kollo, http://dbpedia.org/resource/Lilla_Jönssonligan_på_styva_linan, http://dbpedia.org/resource/S.O.S._–_En_segelsällskapsresa, http://dbpedia.org/resource/Expedition_Linné, http://dbpedia.org/resource/Ratty_(film), http://dbpedia.org/resource/Peter-No-Tail_and_the_Great_Treasure_Hunt, http://dbpedia.org/resource/Prince_Gustaf_(film), http://dbpedia.org/resource/491_(film), http://dbpedia.org/resource/Once_in_a_Lifetime_(2000_film), http://dbpedia.org/resource/Illusive_Tracks, http://dbpedia.org/resource/Jalla!_Jalla!, http://dbpedia.org/resource/Beck_(film), http://dbpedia.org/resource/Beck_–_Annonsmannen, http://dbpedia.org/resource/Beck_–_Den_svaga_länken, http://dbpedia.org/resource/Beck_–_Det_tysta_skriket, http://dbpedia.org/resource/Beck_–_Flickan_i_jordkällaren, http://dbpedia.org/resource/Beck_–_Mannen_med_ikonerna, http://dbpedia.org/resource/Beck_–_Mannen_utan_ansikte, http://dbpedia.org/resource/Beck_–_Monstret, http://dbpedia.org/resource/Beck_–_Okänd_avsändare, http://dbpedia.org/resource/Beck_–_Pojken_i_glaskulan, http://dbpedia.org/resource/Beck_–_Sista_vittnet, http://dbpedia.org/resource/Beck_–_Skarpt_läge, http://dbpedia.org/resource/The_Man_on_the_Balcony, http://dbpedia.org/resource/The_Gladiators_(film), http://dbpedia.org/resource/Die_Zombiejäger, http://dbpedia.org/resource/Blå_måndag, http://dbpedia.org/resource/As_It_Is_in_Heaven, http://dbpedia.org/resource/Made_in_YU, http://dbpedia.org/resource/Maedeli_la_brèche, http://dbpedia.org/resource/Första_divisionen, http://dbpedia.org/resource/Seventh_Heaven_(1956_film), http://dbpedia.org/resource/Gabrielle_(1954_film), http://dbpedia.org/resource/Kalle_Stropp,_Grodan_Boll_och_deras_vänner, http://dbpedia.org/resource/Kalle_Stropp_och_Grodan_Boll_räddar_Hönan, http://dbpedia.org/resource/Wallander:_Innan_frosten, http://dbpedia.org/resource/Intermezzo_(1936_film), http://dbpedia.org/resource/The_Firebird_(1952_film), http://dbpedia.org/resource/Forelsket_i_København, http://dbpedia.org/resource/Terrorists:_The_Kids_They_Sentenced, http://dbpedia.org/resource/Elina:_As_If_I_Wasn't_There, http://dbpedia.org/resource/Bert:_The_Last_Virgin, http://dbpedia.org/resource/Dirty_Diaries, http://dbpedia.org/resource/Don't_Give_Up_(film), http://dbpedia.org/resource/L'imitateur, http://dbpedia.org/resource/Charlie_Strapp_and_Froggy_Ball_Flying_High, http://dbpedia.org/resource/The_Women_Around_Larsson, http://dbpedia.org/resource/Modiga_mindre_män, http://dbpedia.org/resource/Pappa_polis_(TV_series), http://dbpedia.org/resource/While_the_Door_Was_Locked, http://dbpedia.org/resource/The_Referee_(film), http://dbpedia.org/resource/The_Final_Countdown_Tour_1986:_Live_in_Sweden_–_20th_Anniversary_Edition, http://dbpedia.org/resource/Johansson_and_Vestman, http://dbpedia.org/resource/Don't_Cry_Wolf_(film), http://dbpedia.org/resource/Rocket_Brothers] [INFO] Predicted Answer: [http://dbpedia.org/resource/Day_by_Day_(hymn), http://dbpedia.org/resource/Thy_Holy_Wings] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 235, P: 0.553, R: 0.610, macro F1: 0.561, macro F1*: 0.580 -[INFO] Current time: 1238 ms, average: 2827.51 ms/ques, entity detection: 388.73 ms/ques (linking tool: 296.90 ms/ques), relation detection: 894.99 ms/ques (similarity: 716.86 ms/ques), EDG generation: 70.95 ms/ques +[INFO] Cumulative metrics, sample: 235, P: 0.553, R: 0.610, macro F1: 0.580 +[INFO] Current question consumed time: 1114 ms, average: 2647.66 ms/ques, entity detection: 364.09 ms/ques (linking tool: 280.42 ms/ques), relation detection: 860.46 ms/ques (similarity: 708.49 ms/ques), EDG generation: 69.91 ms/ques [INFO] Entity Linking P: 0.7574468085106383, R: 0.7148936170212766, micro F1: 0.728368794326241, macro F1: 0.7355552822531053 [INFO] Relation Linking P: 0.5900709219858156, R: 0.5787234042553191, micro F1: 0.5781560283687946, macro F1: 0.5843420780830407 [INFO] Type Linking P: 0.5234042553191489, R: 0.5234042553191489, micro F1: 0.5234042553191489, macro F1: 0.5234042553191489 @@ -3298,8 +3298,8 @@ Question 235: Was christina crawford trained by Booker T? [INFO] Golden Answer: [true] [INFO] Predicted Answer: [true] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 236, P: 0.555, R: 0.612, macro F1: 0.563, macro F1*: 0.582 -[INFO] Current time: 1460 ms, average: 2821.72 ms/ques, entity detection: 388.21 ms/ques (linking tool: 296.36 ms/ques), relation detection: 893.19 ms/ques (similarity: 715.04 ms/ques), EDG generation: 70.71 ms/ques +[INFO] Cumulative metrics, sample: 236, P: 0.555, R: 0.612, macro F1: 0.582 +[INFO] Current question consumed time: 1238 ms, average: 2641.69 ms/ques, entity detection: 363.54 ms/ques (linking tool: 279.90 ms/ques), relation detection: 858.81 ms/ques (similarity: 706.62 ms/ques), EDG generation: 69.69 ms/ques [INFO] Entity Linking P: 0.7584745762711864, R: 0.7161016949152542, micro F1: 0.7295197740112993, macro F1: 0.736679329826612 [INFO] Relation Linking P: 0.5918079096045198, R: 0.5805084745762712, micro F1: 0.5799435028248591, macro F1: 0.5861037369818255 [INFO] Type Linking P: 0.5254237288135594, R: 0.5254237288135594, micro F1: 0.5254237288135594, macro F1: 0.5254237288135594 @@ -3312,8 +3312,8 @@ Question 236: What is the currency of Kerguelen Islands ? [INFO] Golden Answer: [http://dbpedia.org/resource/Euro] [INFO] Predicted Answer: [http://dbpedia.org/resource/Euro] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 237, P: 0.557, R: 0.613, macro F1: 0.565, macro F1*: 0.584 -[INFO] Current time: 1922 ms, average: 2817.92 ms/ques, entity detection: 387.32 ms/ques (linking tool: 295.60 ms/ques), relation detection: 893.51 ms/ques (similarity: 714.57 ms/ques), EDG generation: 70.46 ms/ques +[INFO] Cumulative metrics, sample: 237, P: 0.557, R: 0.613, macro F1: 0.584 +[INFO] Current question consumed time: 1813 ms, average: 2638.19 ms/ques, entity detection: 362.67 ms/ques (linking tool: 279.12 ms/ques), relation detection: 859.21 ms/ques (similarity: 706.21 ms/ques), EDG generation: 69.46 ms/ques [INFO] Entity Linking P: 0.759493670886076, R: 0.7172995780590717, micro F1: 0.730661040787623, macro F1: 0.7377938517179023 [INFO] Relation Linking P: 0.5935302390998594, R: 0.5822784810126582, micro F1: 0.5817158931082984, macro F1: 0.5878505238931622 [INFO] Type Linking P: 0.5274261603375527, R: 0.5274261603375527, micro F1: 0.5274261603375527, macro F1: 0.5274261603375527 @@ -3326,8 +3326,8 @@ Question 237: What are the musicals whose musicians worked under Sony Music Ente [INFO] Golden Answer: [http://dbpedia.org/resource/Hands_on_a_Hardbody_(musical), http://dbpedia.org/resource/The_Real_Ambassadors, http://dbpedia.org/resource/Good_Vibrations_(musical), http://dbpedia.org/resource/The_SpongeBob_Musical, http://dbpedia.org/resource/Working_(musical), http://dbpedia.org/resource/Daddy_Cool_(musical), http://dbpedia.org/resource/Finding_Neverland_(musical), http://dbpedia.org/resource/Never_Forget_(musical), http://dbpedia.org/resource/Really_Rosie] [INFO] Predicted Answer: [http://dbpedia.org/resource/Dimitris_Giarmenitis] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 238, P: 0.555, R: 0.611, macro F1: 0.562, macro F1*: 0.581 -[INFO] Current time: 2644 ms, average: 2817.19 ms/ques, entity detection: 388.33 ms/ques (linking tool: 296.72 ms/ques), relation detection: 892.77 ms/ques (similarity: 714.19 ms/ques), EDG generation: 70.22 ms/ques +[INFO] Cumulative metrics, sample: 238, P: 0.555, R: 0.611, macro F1: 0.581 +[INFO] Current question consumed time: 2568 ms, average: 2637.90 ms/ques, entity detection: 363.69 ms/ques (linking tool: 280.26 ms/ques), relation detection: 858.56 ms/ques (similarity: 705.83 ms/ques), EDG generation: 69.24 ms/ques [INFO] Entity Linking P: 0.7605042016806722, R: 0.7184873949579832, micro F1: 0.7317927170868346, macro F1: 0.7388989686783803 [INFO] Relation Linking P: 0.5910364145658263, R: 0.5798319327731093, micro F1: 0.5792717086834737, macro F1: 0.5853805637087371 [INFO] Type Linking P: 0.5252100840336135, R: 0.5252100840336135, micro F1: 0.5252100840336135, macro F1: 0.5252100840336135 @@ -3340,8 +3340,8 @@ Question 238: Whose shrines are in a place governed by Justin Trudeau? [INFO] Golden Answer: [http://dbpedia.org/resource/Kateri_Tekakwitha, http://dbpedia.org/resource/Marguerite_Bourgeoys, http://dbpedia.org/resource/Gabriel_Lalemant, http://dbpedia.org/resource/Jean_de_Brébeuf, http://dbpedia.org/resource/Vasyl_Velychkovsky] [INFO] Predicted Answer: [] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 239, P: 0.552, R: 0.608, macro F1: 0.560, macro F1*: 0.579 -[INFO] Current time: 16788 ms, average: 2875.64 ms/ques, entity detection: 387.43 ms/ques (linking tool: 295.92 ms/ques), relation detection: 930.31 ms/ques (similarity: 729.16 ms/ques), EDG generation: 70.00 ms/ques +[INFO] Cumulative metrics, sample: 239, P: 0.552, R: 0.608, macro F1: 0.579 +[INFO] Current question consumed time: 15127 ms, average: 2690.15 ms/ques, entity detection: 362.82 ms/ques (linking tool: 279.49 ms/ques), relation detection: 892.89 ms/ques (similarity: 720.87 ms/ques), EDG generation: 69.03 ms/ques [INFO] Entity Linking P: 0.7573221757322176, R: 0.7154811715481172, micro F1: 0.7287308228730821, macro F1: 0.7358073411943705 [INFO] Relation Linking P: 0.5885634588563459, R: 0.5774058577405857, micro F1: 0.576847977684798, macro F1: 0.5829312726471942 [INFO] Type Linking P: 0.5271966527196653, R: 0.5271966527196653, micro F1: 0.5271966527196653, macro F1: 0.5271966527196653 @@ -3354,8 +3354,8 @@ Question 239: What are the academic interests of the past members of Senser? [INFO] Golden Answer: [http://dbpedia.org/resource/Buddhism, http://dbpedia.org/resource/Metaphysics, http://dbpedia.org/resource/Philosophical_realism, http://dbpedia.org/resource/Theoretical_ecology, http://dbpedia.org/resource/Object-oriented_ontology] [INFO] Predicted Answer: [http://dbpedia.org/resource/Buddhism, http://dbpedia.org/resource/Metaphysics, http://dbpedia.org/resource/Philosophical_realism, http://dbpedia.org/resource/Theoretical_ecology, http://dbpedia.org/resource/Object-oriented_ontology] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 240, P: 0.554, R: 0.610, macro F1: 0.562, macro F1*: 0.581 -[INFO] Current time: 1061 ms, average: 2868.08 ms/ques, entity detection: 386.39 ms/ques (linking tool: 295.09 ms/ques), relation detection: 928.50 ms/ques (similarity: 727.41 ms/ques), EDG generation: 69.81 ms/ques +[INFO] Cumulative metrics, sample: 240, P: 0.554, R: 0.610, macro F1: 0.581 +[INFO] Current question consumed time: 966 ms, average: 2682.97 ms/ques, entity detection: 361.77 ms/ques (linking tool: 278.61 ms/ques), relation detection: 891.15 ms/ques (similarity: 719.22 ms/ques), EDG generation: 68.84 ms/ques [INFO] Entity Linking P: 0.7583333333333333, R: 0.7166666666666667, micro F1: 0.729861111111111, macro F1: 0.7369114877589453 [INFO] Relation Linking P: 0.5902777777777778, R: 0.5791666666666667, micro F1: 0.5786111111111114, macro F1: 0.5846694378463976 [INFO] Type Linking P: 0.5291666666666667, R: 0.5291666666666667, micro F1: 0.5291666666666667, macro F1: 0.5291666666666667 @@ -3368,8 +3368,8 @@ Question 240: Who is the leader of Hackweiser ? [INFO] Golden Answer: [http://dbpedia.org/resource/North_America, p4ntera , BigHawk@en] [INFO] Predicted Answer: [http://dbpedia.org/resource/North_America, p4ntera , BigHawk@en] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 241, P: 0.556, R: 0.611, macro F1: 0.564, macro F1*: 0.582 -[INFO] Current time: 290 ms, average: 2857.39 ms/ques, entity detection: 384.93 ms/ques (linking tool: 293.96 ms/ques), relation detection: 925.05 ms/ques (similarity: 724.68 ms/ques), EDG generation: 69.56 ms/ques +[INFO] Cumulative metrics, sample: 241, P: 0.556, R: 0.611, macro F1: 0.582 +[INFO] Current question consumed time: 256 ms, average: 2672.90 ms/ques, entity detection: 360.39 ms/ques (linking tool: 277.54 ms/ques), relation detection: 887.86 ms/ques (similarity: 716.52 ms/ques), EDG generation: 68.59 ms/ques [INFO] Entity Linking P: 0.7593360995850622, R: 0.7178423236514523, micro F1: 0.730982019363762, macro F1: 0.7380064338663807 [INFO] Relation Linking P: 0.5919778699861687, R: 0.5809128630705395, micro F1: 0.5803596127247582, macro F1: 0.5863931730995068 [INFO] Type Linking P: 0.5311203319502075, R: 0.5311203319502075, micro F1: 0.5311203319502075, macro F1: 0.5311203319502075 @@ -3382,8 +3382,8 @@ Question 241: In which city is the sister station of KTXY located ? [INFO] Golden Answer: [http://dbpedia.org/resource/Missouri, http://dbpedia.org/resource/California,_Missouri, http://dbpedia.org/resource/Boonville,_Missouri, http://dbpedia.org/resource/Columbia,_Missouri, http://dbpedia.org/resource/Fulton,_Missouri, http://dbpedia.org/resource/Fayette,_Missouri] [INFO] Predicted Answer: [http://dbpedia.org/resource/Jefferson_City,_Missouri] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 242, P: 0.554, R: 0.609, macro F1: 0.561, macro F1*: 0.580 -[INFO] Current time: 637 ms, average: 2848.21 ms/ques, entity detection: 383.45 ms/ques (linking tool: 292.83 ms/ques), relation detection: 922.26 ms/ques (similarity: 722.89 ms/ques), EDG generation: 69.34 ms/ques +[INFO] Cumulative metrics, sample: 242, P: 0.554, R: 0.609, macro F1: 0.580 +[INFO] Current question consumed time: 575 ms, average: 2664.23 ms/ques, entity detection: 359.02 ms/ques (linking tool: 276.49 ms/ques), relation detection: 885.13 ms/ques (similarity: 714.75 ms/ques), EDG generation: 68.37 ms/ques [INFO] Entity Linking P: 0.7603305785123967, R: 0.71900826446281, micro F1: 0.7320936639118456, macro F1: 0.739092294196408 [INFO] Relation Linking P: 0.59366391184573, R: 0.5805785123966942, micro F1: 0.5807162534435264, macro F1: 0.5870483022709095 [INFO] Type Linking P: 0.5289256198347108, R: 0.5289256198347108, micro F1: 0.5289256198347108, macro F1: 0.5289256198347108 @@ -3396,8 +3396,8 @@ Question 242: Where was the successor of Hank Brown born? [INFO] Golden Answer: [http://dbpedia.org/resource/Colorado, http://dbpedia.org/resource/Fort_Collins,_Colorado] [INFO] Predicted Answer: [http://dbpedia.org/resource/James_Paul_Johnson, http://dbpedia.org/resource/Wayne_Allard, Wayne Allard@en] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 243, P: 0.552, R: 0.606, macro F1: 0.559, macro F1*: 0.578 -[INFO] Current time: 944 ms, average: 2840.37 ms/ques, entity detection: 382.69 ms/ques (linking tool: 292.12 ms/ques), relation detection: 919.53 ms/ques (similarity: 720.60 ms/ques), EDG generation: 69.10 ms/ques +[INFO] Cumulative metrics, sample: 243, P: 0.552, R: 0.606, macro F1: 0.578 +[INFO] Current question consumed time: 779 ms, average: 2656.47 ms/ques, entity detection: 358.18 ms/ques (linking tool: 275.73 ms/ques), relation detection: 882.50 ms/ques (similarity: 712.52 ms/ques), EDG generation: 68.14 ms/ques [INFO] Entity Linking P: 0.7613168724279835, R: 0.720164609053498, micro F1: 0.7331961591220849, macro F1: 0.7401691815272062 [INFO] Relation Linking P: 0.5953360768175583, R: 0.5802469135802469, micro F1: 0.5810699588477368, macro F1: 0.5876946569284182 [INFO] Type Linking P: 0.5308641975308642, R: 0.5308641975308642, micro F1: 0.5308641975308642, macro F1: 0.5308641975308642 @@ -3410,8 +3410,8 @@ Question 243: Name the constituency of Jonas Gahr where Sverre Krogh was born ? [INFO] Golden Answer: [http://dbpedia.org/resource/Oslo] [INFO] Predicted Answer: [http://dbpedia.org/resource/Oslo] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 244, P: 0.553, R: 0.608, macro F1: 0.561, macro F1*: 0.579 -[INFO] Current time: 1807 ms, average: 2836.14 ms/ques, entity detection: 382.66 ms/ques (linking tool: 291.80 ms/ques), relation detection: 917.95 ms/ques (similarity: 718.93 ms/ques), EDG generation: 68.93 ms/ques +[INFO] Cumulative metrics, sample: 244, P: 0.553, R: 0.608, macro F1: 0.579 +[INFO] Current question consumed time: 1473 ms, average: 2651.62 ms/ques, entity detection: 357.89 ms/ques (linking tool: 275.23 ms/ques), relation detection: 880.84 ms/ques (similarity: 710.89 ms/ques), EDG generation: 67.98 ms/ques [INFO] Entity Linking P: 0.7622950819672131, R: 0.7213114754098361, micro F1: 0.7342896174863387, macro F1: 0.7412372067747487 [INFO] Relation Linking P: 0.5969945355191256, R: 0.5819672131147541, micro F1: 0.5827868852459018, macro F1: 0.5893851034325551 [INFO] Type Linking P: 0.5327868852459017, R: 0.5327868852459017, micro F1: 0.5327868852459017, macro F1: 0.5327868852459017 @@ -3424,8 +3424,8 @@ Question 244: What is the common battle fought by Stephen Urban and Ali Habib Ma [INFO] Golden Answer: [http://dbpedia.org/resource/Gulf_War] [INFO] Predicted Answer: [http://dbpedia.org/resource/Gulf_War] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 245, P: 0.555, R: 0.610, macro F1: 0.563, macro F1*: 0.581 -[INFO] Current time: 1886 ms, average: 2832.26 ms/ques, entity detection: 382.47 ms/ques (linking tool: 291.40 ms/ques), relation detection: 917.21 ms/ques (similarity: 718.16 ms/ques), EDG generation: 68.84 ms/ques +[INFO] Cumulative metrics, sample: 245, P: 0.555, R: 0.610, macro F1: 0.581 +[INFO] Current question consumed time: 1470 ms, average: 2646.80 ms/ques, entity detection: 357.62 ms/ques (linking tool: 274.76 ms/ques), relation detection: 879.36 ms/ques (similarity: 709.51 ms/ques), EDG generation: 67.91 ms/ques [INFO] Entity Linking P: 0.763265306122449, R: 0.7224489795918367, micro F1: 0.7353741496598638, macro F1: 0.7422964790311729 [INFO] Relation Linking P: 0.5986394557823129, R: 0.5816326530612245, micro F1: 0.5831292517006804, macro F1: 0.5900135270246427 [INFO] Type Linking P: 0.5346938775510204, R: 0.5346938775510204, micro F1: 0.5346938775510204, macro F1: 0.5346938775510204 @@ -3438,8 +3438,8 @@ Question 245: What is the relegious affiliations of Katyayana ? [INFO] Golden Answer: [http://dbpedia.org/resource/Buddhism] [INFO] Predicted Answer: [http://dbpedia.org/resource/Buddhism] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 246, P: 0.557, R: 0.611, macro F1: 0.564, macro F1*: 0.583 -[INFO] Current time: 6487 ms, average: 2847.12 ms/ques, entity detection: 393.80 ms/ques (linking tool: 302.96 ms/ques), relation detection: 914.15 ms/ques (similarity: 715.60 ms/ques), EDG generation: 68.61 ms/ques +[INFO] Cumulative metrics, sample: 246, P: 0.557, R: 0.611, macro F1: 0.583 +[INFO] Current question consumed time: 5132 ms, average: 2656.90 ms/ques, entity detection: 366.40 ms/ques (linking tool: 283.78 ms/ques), relation detection: 876.31 ms/ques (similarity: 706.97 ms/ques), EDG generation: 67.68 ms/ques [INFO] Entity Linking P: 0.7642276422764228, R: 0.7235772357723578, micro F1: 0.7364498644986448, macro F1: 0.7433471056022036 [INFO] Relation Linking P: 0.6002710027100271, R: 0.5833333333333334, micro F1: 0.5848238482384825, macro F1: 0.5916809769128029 [INFO] Type Linking P: 0.5365853658536586, R: 0.5365853658536586, micro F1: 0.5365853658536586, macro F1: 0.5365853658536586 @@ -3452,8 +3452,8 @@ Question 246: Where was the engineer born who manufactured EP R.E.P. 1? [INFO] Golden Answer: [http://dbpedia.org/resource/France, http://dbpedia.org/resource/Paris] [INFO] Predicted Answer: [http://dbpedia.org/resource/Scomi_Rail, http://dbpedia.org/resource/Universal_Robina, http://dbpedia.org/resource/Digital_Telecommunications_Philippines] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 247, P: 0.555, R: 0.609, macro F1: 0.562, macro F1*: 0.580 -[INFO] Current time: 1884 ms, average: 2843.22 ms/ques, entity detection: 392.87 ms/ques (linking tool: 302.28 ms/ques), relation detection: 913.62 ms/ques (similarity: 714.47 ms/ques), EDG generation: 68.42 ms/ques +[INFO] Cumulative metrics, sample: 247, P: 0.555, R: 0.609, macro F1: 0.580 +[INFO] Current question consumed time: 1700 ms, average: 2653.03 ms/ques, entity detection: 365.53 ms/ques (linking tool: 283.15 ms/ques), relation detection: 875.93 ms/ques (similarity: 705.91 ms/ques), EDG generation: 67.49 ms/ques [INFO] Entity Linking P: 0.7611336032388664, R: 0.7206477732793523, micro F1: 0.7334682860998649, macro F1: 0.7403376031503729 [INFO] Relation Linking P: 0.5978407557354926, R: 0.5809716599190283, micro F1: 0.5824561403508773, macro F1: 0.5892855073706459 [INFO] Type Linking P: 0.5384615384615384, R: 0.5384615384615384, micro F1: 0.5384615384615384, macro F1: 0.5384615384615384 @@ -3466,8 +3466,8 @@ Question 247: who all have been a manager of english under twenty one football t [INFO] Golden Answer: [http://dbpedia.org/resource/Roy_Hodgson, http://dbpedia.org/resource/Nigel_Pearson, http://dbpedia.org/resource/Dave_Sexton, http://dbpedia.org/resource/Ray_Wilkins, http://dbpedia.org/resource/David_Platt_(footballer), http://dbpedia.org/resource/Peter_Taylor_(footballer,_born_1953), http://dbpedia.org/resource/Stuart_Pearce, http://dbpedia.org/resource/Peter_Reid, http://dbpedia.org/resource/Lawrie_McMenemy, http://dbpedia.org/resource/Howard_Wilkinson, http://dbpedia.org/resource/Gareth_Southgate] [INFO] Predicted Answer: [http://dbpedia.org/resource/A._B._A._Ghani_Khan_Choudhury, http://dbpedia.org/resource/Abu_Hasem_Khan_Choudhury, http://dbpedia.org/resource/Mausam_Noor] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 248, P: 0.552, R: 0.606, macro F1: 0.560, macro F1*: 0.578 -[INFO] Current time: 1230 ms, average: 2836.71 ms/ques, entity detection: 391.67 ms/ques (linking tool: 301.28 ms/ques), relation detection: 912.06 ms/ques (similarity: 712.97 ms/ques), EDG generation: 68.25 ms/ques +[INFO] Cumulative metrics, sample: 248, P: 0.552, R: 0.606, macro F1: 0.578 +[INFO] Current question consumed time: 1112 ms, average: 2646.81 ms/ques, entity detection: 364.46 ms/ques (linking tool: 282.22 ms/ques), relation detection: 874.40 ms/ques (similarity: 704.43 ms/ques), EDG generation: 67.33 ms/ques [INFO] Entity Linking P: 0.7580645161290323, R: 0.717741935483871, micro F1: 0.7305107526881719, macro F1: 0.7373523708796051 [INFO] Relation Linking P: 0.5954301075268816, R: 0.5786290322580645, micro F1: 0.5801075268817206, macro F1: 0.5869093561312481 [INFO] Type Linking P: 0.5403225806451613, R: 0.5403225806451613, micro F1: 0.5403225806451613, macro F1: 0.5403225806451613 @@ -3480,8 +3480,8 @@ Question 248: What is the military conflict whose commanders are John McEwen and [INFO] Golden Answer: [http://dbpedia.org/resource/Vietnam_War] [INFO] Predicted Answer: [http://dbpedia.org/resource/Vietnam_War] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 249, P: 0.554, R: 0.608, macro F1: 0.562, macro F1*: 0.580 -[INFO] Current time: 1409 ms, average: 2830.98 ms/ques, entity detection: 390.95 ms/ques (linking tool: 300.61 ms/ques), relation detection: 910.05 ms/ques (similarity: 711.99 ms/ques), EDG generation: 68.11 ms/ques +[INFO] Cumulative metrics, sample: 249, P: 0.554, R: 0.608, macro F1: 0.580 +[INFO] Current question consumed time: 1351 ms, average: 2641.61 ms/ques, entity detection: 363.76 ms/ques (linking tool: 281.54 ms/ques), relation detection: 872.56 ms/ques (similarity: 703.47 ms/ques), EDG generation: 67.18 ms/ques [INFO] Entity Linking P: 0.7590361445783133, R: 0.7168674698795181, micro F1: 0.7302543507362782, macro F1: 0.7373493975903613 [INFO] Relation Linking P: 0.5970548862115127, R: 0.5803212851405622, micro F1: 0.581793842034806, macro F1: 0.5885691715126566 [INFO] Type Linking P: 0.5381526104417671, R: 0.5381526104417671, micro F1: 0.5381526104417671, macro F1: 0.5381526104417671 @@ -3494,8 +3494,8 @@ Question 249: Which party has come in power in Mumbai North? [INFO] Golden Answer: [http://dbpedia.org/resource/Indian_National_Congress, http://dbpedia.org/resource/Bharatiya_Janata_Party] [INFO] Predicted Answer: [Independent@en, Samajwadi Party@en, Indian National Congress@en, Bharatiya Janata Party@en, Maharashtra Navnirman Sena@en, None of the above@en, Bahujan Samaj Party@en, Aam Aadmi Party@en, Kranti Kari Jai Hind Sena@en, Akhil Bharat Hindu Mahasabha@en, Naari Shakti Party@en] [INFO] P: 0.18181818181818182, R: 1.0, F: 0.3076923076923077 -[INFO] Cumulative metrics, sample: 250, P: 0.553, R: 0.609, macro F1: 0.561, macro F1*: 0.580 -[INFO] Current time: 1630 ms, average: 2826.18 ms/ques, entity detection: 392.48 ms/ques (linking tool: 302.15 ms/ques), relation detection: 907.14 ms/ques (similarity: 709.88 ms/ques), EDG generation: 67.90 ms/ques +[INFO] Cumulative metrics, sample: 250, P: 0.553, R: 0.609, macro F1: 0.580 +[INFO] Current question consumed time: 1507 ms, average: 2637.08 ms/ques, entity detection: 364.90 ms/ques (linking tool: 282.71 ms/ques), relation detection: 869.93 ms/ques (similarity: 701.71 ms/ques), EDG generation: 66.97 ms/ques [INFO] Entity Linking P: 0.76, R: 0.718, micro F1: 0.7313333333333332, macro F1: 0.738403247631935 [INFO] Relation Linking P: 0.5986666666666667, R: 0.58, micro F1: 0.5821333333333334, macro F1: 0.5891855203619909 [INFO] Type Linking P: 0.54, R: 0.54, micro F1: 0.54, macro F1: 0.54 @@ -3508,8 +3508,8 @@ Question 250: Name the tv show with judges Randy Jackson and Jennifer Lopez? [INFO] Golden Answer: [http://dbpedia.org/resource/American_Idol] [INFO] Predicted Answer: [http://dbpedia.org/resource/American_Idol, http://dbpedia.org/resource/American_Idol_(season_10), http://dbpedia.org/resource/American_Idol_(season_11)] [INFO] P: 0.3333333333333333, R: 1.0, F: 0.5 -[INFO] Cumulative metrics, sample: 251, P: 0.552, R: 0.611, macro F1: 0.560, macro F1*: 0.580 -[INFO] Current time: 5859 ms, average: 2838.26 ms/ques, entity detection: 396.12 ms/ques (linking tool: 305.27 ms/ques), relation detection: 910.49 ms/ques (similarity: 715.05 ms/ques), EDG generation: 67.78 ms/ques +[INFO] Cumulative metrics, sample: 251, P: 0.552, R: 0.611, macro F1: 0.580 +[INFO] Current question consumed time: 5499 ms, average: 2648.48 ms/ques, entity detection: 368.04 ms/ques (linking tool: 285.39 ms/ques), relation detection: 873.27 ms/ques (similarity: 707.01 ms/ques), EDG generation: 66.86 ms/ques [INFO] Entity Linking P: 0.7609561752988048, R: 0.7191235059760956, micro F1: 0.7324037184594951, macro F1: 0.7394486656335627 [INFO] Relation Linking P: 0.6002656042496679, R: 0.5816733067729084, micro F1: 0.5837981407702524, macro F1: 0.5908232239581003 [INFO] Type Linking P: 0.5378486055776892, R: 0.5378486055776892, micro F1: 0.5378486055776892, macro F1: 0.5378486055776892 @@ -3522,8 +3522,8 @@ Question 251: What is the largest city of the country which hosted the 1951 Asia [INFO] Golden Answer: [http://dbpedia.org/resource/Mumbai] [INFO] Predicted Answer: [http://dbpedia.org/resource/Mumbai] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 252, P: 0.554, R: 0.612, macro F1: 0.562, macro F1*: 0.582 -[INFO] Current time: 11166 ms, average: 2871.31 ms/ques, entity detection: 395.68 ms/ques (linking tool: 304.66 ms/ques), relation detection: 931.97 ms/ques (similarity: 722.78 ms/ques), EDG generation: 67.62 ms/ques +[INFO] Cumulative metrics, sample: 252, P: 0.554, R: 0.612, macro F1: 0.582 +[INFO] Current question consumed time: 10048 ms, average: 2677.84 ms/ques, entity detection: 367.51 ms/ques (linking tool: 284.75 ms/ques), relation detection: 892.14 ms/ques (similarity: 713.74 ms/ques), EDG generation: 66.70 ms/ques [INFO] Entity Linking P: 0.7619047619047619, R: 0.7202380952380952, micro F1: 0.7334656084656083, macro F1: 0.7404857525339453 [INFO] Relation Linking P: 0.6018518518518519, R: 0.5833333333333334, micro F1: 0.5854497354497356, macro F1: 0.5924479166666667 [INFO] Type Linking P: 0.5357142857142857, R: 0.5357142857142857, micro F1: 0.5357142857142857, macro F1: 0.5357142857142857 @@ -3536,8 +3536,8 @@ Question 252: Which royalty was married to ptolemy XIII Theos Philopator and had [INFO] Golden Answer: [http://dbpedia.org/resource/Cleopatra] [INFO] Predicted Answer: [http://dbpedia.org/resource/Cleopatra] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 253, P: 0.555, R: 0.614, macro F1: 0.564, macro F1*: 0.583 -[INFO] Current time: 1975 ms, average: 2867.77 ms/ques, entity detection: 395.49 ms/ques (linking tool: 304.36 ms/ques), relation detection: 930.40 ms/ques (similarity: 721.93 ms/ques), EDG generation: 67.45 ms/ques +[INFO] Cumulative metrics, sample: 253, P: 0.555, R: 0.614, macro F1: 0.583 +[INFO] Current question consumed time: 1903 ms, average: 2674.78 ms/ques, entity detection: 367.41 ms/ques (linking tool: 284.55 ms/ques), relation detection: 890.70 ms/ques (similarity: 712.91 ms/ques), EDG generation: 66.53 ms/ques [INFO] Entity Linking P: 0.7628458498023716, R: 0.7213438735177866, micro F1: 0.7345191040843213, macro F1: 0.7415146076640895 [INFO] Relation Linking P: 0.6034255599472991, R: 0.5849802371541502, micro F1: 0.5870882740447959, macro F1: 0.5940597530301569 [INFO] Type Linking P: 0.5335968379446641, R: 0.5335968379446641, micro F1: 0.5335968379446641, macro F1: 0.5335968379446641 @@ -3550,8 +3550,8 @@ Question 253: Whose theme music was composed by a band which collaborated with P [INFO] Golden Answer: [http://dbpedia.org/resource/South_Park] [INFO] Predicted Answer: [http://dbpedia.org/resource/South_Park, http://dbpedia.org/resource/Robot_Chicken:_Star_Wars, http://dbpedia.org/resource/Robot_Chicken:_Star_Wars_Episode_II, http://dbpedia.org/resource/Robot_Chicken:_Star_Wars_Episode_III] [INFO] P: 0.25, R: 1.0, F: 0.4 -[INFO] Cumulative metrics, sample: 254, P: 0.554, R: 0.615, macro F1: 0.563, macro F1*: 0.583 -[INFO] Current time: 3395 ms, average: 2869.84 ms/ques, entity detection: 394.79 ms/ques (linking tool: 303.74 ms/ques), relation detection: 932.45 ms/ques (similarity: 724.83 ms/ques), EDG generation: 67.29 ms/ques +[INFO] Cumulative metrics, sample: 254, P: 0.554, R: 0.615, macro F1: 0.583 +[INFO] Current question consumed time: 3276 ms, average: 2677.15 ms/ques, entity detection: 366.82 ms/ques (linking tool: 284.03 ms/ques), relation detection: 892.68 ms/ques (similarity: 715.87 ms/ques), EDG generation: 66.39 ms/ques [INFO] Entity Linking P: 0.7637795275590551, R: 0.7224409448818898, micro F1: 0.7355643044619421, macro F1: 0.7425353287792669 [INFO] Relation Linking P: 0.60498687664042, R: 0.5866141732283464, micro F1: 0.588713910761155, macro F1: 0.5956588851503694 [INFO] Type Linking P: 0.5354330708661418, R: 0.5354330708661418, micro F1: 0.5354330708661418, macro F1: 0.5354330708661418 @@ -3564,8 +3564,8 @@ Question 254: Is morrissey in the smiths? [INFO] Golden Answer: [true] [INFO] Predicted Answer: [false] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 255, P: 0.552, R: 0.613, macro F1: 0.561, macro F1*: 0.581 -[INFO] Current time: 271 ms, average: 2859.65 ms/ques, entity detection: 393.45 ms/ques (linking tool: 302.69 ms/ques), relation detection: 929.04 ms/ques (similarity: 722.16 ms/ques), EDG generation: 67.06 ms/ques +[INFO] Cumulative metrics, sample: 255, P: 0.552, R: 0.613, macro F1: 0.581 +[INFO] Current question consumed time: 242 ms, average: 2667.60 ms/ques, entity detection: 365.56 ms/ques (linking tool: 283.03 ms/ques), relation detection: 889.41 ms/ques (similarity: 713.23 ms/ques), EDG generation: 66.16 ms/ques [INFO] Entity Linking P: 0.7607843137254902, R: 0.7196078431372549, micro F1: 0.7326797385620913, macro F1: 0.739623425529152 [INFO] Relation Linking P: 0.6026143790849673, R: 0.5843137254901961, micro F1: 0.58640522875817, macro F1: 0.5933229679537013 [INFO] Type Linking P: 0.5372549019607843, R: 0.5372549019607843, micro F1: 0.5372549019607843, macro F1: 0.5372549019607843 @@ -3578,8 +3578,8 @@ Question 255: Which administrative headquarters of the Pulau Ubin is also the re [INFO] Golden Answer: [http://dbpedia.org/resource/Singapore] [INFO] Predicted Answer: [http://dbpedia.org/resource/Singapore] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 256, P: 0.554, R: 0.615, macro F1: 0.563, macro F1*: 0.583 -[INFO] Current time: 1893 ms, average: 2855.88 ms/ques, entity detection: 393.73 ms/ques (linking tool: 302.29 ms/ques), relation detection: 927.59 ms/ques (similarity: 720.66 ms/ques), EDG generation: 67.27 ms/ques +[INFO] Cumulative metrics, sample: 256, P: 0.554, R: 0.615, macro F1: 0.583 +[INFO] Current question consumed time: 1633 ms, average: 2663.56 ms/ques, entity detection: 365.53 ms/ques (linking tool: 282.55 ms/ques), relation detection: 887.85 ms/ques (similarity: 711.74 ms/ques), EDG generation: 66.38 ms/ques [INFO] Entity Linking P: 0.76171875, R: 0.720703125, micro F1: 0.7337239583333331, macro F1: 0.7406435276679841 [INFO] Relation Linking P: 0.6041666666666666, R: 0.5859375, micro F1: 0.5880208333333334, macro F1: 0.5949124726477025 [INFO] Type Linking P: 0.5390625, R: 0.5390625, micro F1: 0.5390625, macro F1: 0.5390625 @@ -3592,8 +3592,8 @@ Question 256: Where did the beverages sold by Refriango originate? [INFO] Golden Answer: [http://dbpedia.org/resource/Angola] [INFO] Predicted Answer: [] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 257, P: 0.552, R: 0.612, macro F1: 0.560, macro F1*: 0.580 -[INFO] Current time: 448 ms, average: 2846.51 ms/ques, entity detection: 392.19 ms/ques (linking tool: 301.11 ms/ques), relation detection: 925.08 ms/ques (similarity: 718.57 ms/ques), EDG generation: 67.07 ms/ques +[INFO] Cumulative metrics, sample: 257, P: 0.552, R: 0.612, macro F1: 0.580 +[INFO] Current question consumed time: 442 ms, average: 2654.91 ms/ques, entity detection: 364.11 ms/ques (linking tool: 281.46 ms/ques), relation detection: 885.38 ms/ques (similarity: 709.64 ms/ques), EDG generation: 66.34 ms/ques [INFO] Entity Linking P: 0.7587548638132295, R: 0.7178988326848249, micro F1: 0.7308690012970167, macro F1: 0.7377616462373694 [INFO] Relation Linking P: 0.6018158236057068, R: 0.5836575875486382, micro F1: 0.585732814526589, macro F1: 0.5925976381237814 [INFO] Type Linking P: 0.5369649805447471, R: 0.5369649805447471, micro F1: 0.5369649805447471, macro F1: 0.5369649805447471 @@ -3606,8 +3606,8 @@ Question 257: Who built the stadium which was rented by Costa Rica national foot [INFO] Golden Answer: [http://dbpedia.org/resource/China] [INFO] Predicted Answer: [http://dbpedia.org/resource/China] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 258, P: 0.553, R: 0.614, macro F1: 0.562, macro F1*: 0.582 -[INFO] Current time: 3607 ms, average: 2849.45 ms/ques, entity detection: 391.53 ms/ques (linking tool: 300.57 ms/ques), relation detection: 926.11 ms/ques (similarity: 722.48 ms/ques), EDG generation: 66.88 ms/ques +[INFO] Cumulative metrics, sample: 258, P: 0.553, R: 0.614, macro F1: 0.582 +[INFO] Current question consumed time: 2819 ms, average: 2655.55 ms/ques, entity detection: 363.43 ms/ques (linking tool: 280.88 ms/ques), relation detection: 886.19 ms/ques (similarity: 711.79 ms/ques), EDG generation: 66.17 ms/ques [INFO] Entity Linking P: 0.7596899224806202, R: 0.7189922480620154, micro F1: 0.7319121447028422, macro F1: 0.7387810255316123 [INFO] Relation Linking P: 0.6014211886304909, R: 0.5833333333333334, micro F1: 0.5854005167958657, macro F1: 0.5922391857506362 [INFO] Type Linking P: 0.5348837209302325, R: 0.5348837209302325, micro F1: 0.5348837209302325, macro F1: 0.5348837209302325 @@ -3620,8 +3620,8 @@ Question 258: In which country does the Auckland rugby union team play ? [INFO] Golden Answer: [http://dbpedia.org/resource/New_Zealand] [INFO] Predicted Answer: [http://dbpedia.org/resource/Blues_(Super_Rugby)] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 259, P: 0.551, R: 0.611, macro F1: 0.560, macro F1*: 0.580 -[INFO] Current time: 1703 ms, average: 2845.03 ms/ques, entity detection: 391.81 ms/ques (linking tool: 300.91 ms/ques), relation detection: 923.82 ms/ques (similarity: 721.09 ms/ques), EDG generation: 66.66 ms/ques +[INFO] Cumulative metrics, sample: 259, P: 0.551, R: 0.611, macro F1: 0.580 +[INFO] Current question consumed time: 1569 ms, average: 2651.36 ms/ques, entity detection: 363.61 ms/ques (linking tool: 281.14 ms/ques), relation detection: 883.99 ms/ques (similarity: 710.44 ms/ques), EDG generation: 65.95 ms/ques [INFO] Entity Linking P: 0.7606177606177607, R: 0.7200772200772201, micro F1: 0.7329472329472327, macro F1: 0.7397925025043669 [INFO] Relation Linking P: 0.5990990990990991, R: 0.581081081081081, micro F1: 0.5831402831402832, macro F1: 0.5899525479678152 [INFO] Type Linking P: 0.5366795366795367, R: 0.5366795366795367, micro F1: 0.5366795366795367, macro F1: 0.5366795366795367 @@ -3634,8 +3634,8 @@ Question 259: What is the serving railway line of Warwick railway station, Perth [INFO] Golden Answer: [http://dbpedia.org/resource/Butler] [INFO] Predicted Answer: [http://dbpedia.org/resource/Warwick_railway_station,_Perth] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 260, P: 0.549, R: 0.609, macro F1: 0.558, macro F1*: 0.578 -[INFO] Current time: 1864 ms, average: 2841.25 ms/ques, entity detection: 390.93 ms/ques (linking tool: 300.15 ms/ques), relation detection: 922.62 ms/ques (similarity: 719.74 ms/ques), EDG generation: 66.49 ms/ques +[INFO] Cumulative metrics, sample: 260, P: 0.549, R: 0.609, macro F1: 0.578 +[INFO] Current question consumed time: 1640 ms, average: 2647.47 ms/ques, entity detection: 362.74 ms/ques (linking tool: 280.38 ms/ques), relation detection: 882.77 ms/ques (similarity: 709.13 ms/ques), EDG generation: 65.78 ms/ques [INFO] Entity Linking P: 0.7615384615384615, R: 0.7211538461538461, micro F1: 0.7339743589743588, macro F1: 0.7407961688117329 [INFO] Relation Linking P: 0.6006410256410256, R: 0.5826923076923077, micro F1: 0.5847435897435899, macro F1: 0.5915305442120177 [INFO] Type Linking P: 0.5384615384615384, R: 0.5384615384615384, micro F1: 0.5384615384615384, macro F1: 0.5384615384615384 @@ -3648,8 +3648,8 @@ Question 260: What is the radio show created by Timoteo Jeffries, about? [INFO] Golden Answer: [http://dbpedia.org/resource/Latin_America, http://dbpedia.org/resource/Current_affairs_(news_format)] [INFO] Predicted Answer: [http://dbpedia.org/resource/Plaza_del_Mercado_de_Ponce] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 261, P: 0.547, R: 0.607, macro F1: 0.556, macro F1*: 0.575 -[INFO] Current time: 1165 ms, average: 2834.84 ms/ques, entity detection: 389.94 ms/ques (linking tool: 299.41 ms/ques), relation detection: 921.36 ms/ques (similarity: 718.94 ms/ques), EDG generation: 66.33 ms/ques +[INFO] Cumulative metrics, sample: 261, P: 0.547, R: 0.607, macro F1: 0.575 +[INFO] Current question consumed time: 673 ms, average: 2639.90 ms/ques, entity detection: 361.73 ms/ques (linking tool: 279.60 ms/ques), relation detection: 880.09 ms/ques (similarity: 706.87 ms/ques), EDG generation: 65.61 ms/ques [INFO] Entity Linking P: 0.7586206896551724, R: 0.7183908045977011, micro F1: 0.7311621966794378, macro F1: 0.7379578693143701 [INFO] Relation Linking P: 0.5983397190293742, R: 0.5804597701149425, micro F1: 0.5825031928480205, macro F1: 0.5892641436594812 [INFO] Type Linking P: 0.5363984674329502, R: 0.5363984674329502, micro F1: 0.5363984674329502, macro F1: 0.5363984674329502 @@ -3662,8 +3662,8 @@ Question 261: Where are all the protected areas near Bend, Oregon? [INFO] Golden Answer: [http://dbpedia.org/resource/United_States, http://dbpedia.org/resource/Oregon, http://dbpedia.org/resource/Deschutes_County,_Oregon, http://dbpedia.org/resource/Crook_County,_Oregon, http://dbpedia.org/resource/Lane_County,_Oregon, http://dbpedia.org/resource/Lake_County,_Oregon] [INFO] Predicted Answer: [http://dbpedia.org/resource/Mount_McLoughlin, http://dbpedia.org/resource/Wizard_Island, http://dbpedia.org/resource/Diamond_Peak_(Oregon), http://dbpedia.org/resource/Hermiston_Butte, http://dbpedia.org/resource/Jordan_Craters, http://dbpedia.org/resource/Mount_Scott_(Clackamas_County,_Oregon), http://dbpedia.org/resource/Neahkahnie_Mountain, http://dbpedia.org/resource/Burlington_Northern_Railroad_Bridge_5.1, http://dbpedia.org/resource/Hood_River_Bridge, http://dbpedia.org/resource/Grizzly_Mountain_(Crook_County,_Oregon), http://dbpedia.org/resource/Hogback_Mountain_(Klamath_County,_Oregon), http://dbpedia.org/resource/Malheur_Butte, http://dbpedia.org/resource/Mount_Howard_(Oregon), http://dbpedia.org/resource/Mount_Mitchell_(Oregon), http://dbpedia.org/resource/Mount_Baldy_(Polk_County,_Oregon), http://dbpedia.org/resource/Mount_Talbert, http://dbpedia.org/resource/Cinnamon_Butte, http://dbpedia.org/resource/Mount_Emily, http://dbpedia.org/resource/Belknap_Crater, http://dbpedia.org/resource/Larch_Mountain_(Multnomah_County,_Oregon), http://dbpedia.org/resource/Columbia_Heights_(Oregon), http://dbpedia.org/resource/Eagle_Cap, http://dbpedia.org/resource/Newberry_Volcano, http://dbpedia.org/resource/Mount_Multnomah, http://dbpedia.org/resource/Three_Sisters_(Oregon), http://dbpedia.org/resource/Mount_Sylvania, http://dbpedia.org/resource/Mount_Ireland, http://dbpedia.org/resource/Bosley_Butte, http://dbpedia.org/resource/Chetco_Peak, http://dbpedia.org/resource/Humbug_Mountain, http://dbpedia.org/resource/Pearsoll_Peak, http://dbpedia.org/resource/Four_Craters_Lava_Field, http://dbpedia.org/resource/Mount_Hood, http://dbpedia.org/resource/Hayrick_Butte, http://dbpedia.org/resource/Hogg_Rock, http://dbpedia.org/resource/Hoodoo_Butte, http://dbpedia.org/resource/Crooked_River_caldera, http://dbpedia.org/resource/Battle_Ax, http://dbpedia.org/resource/Mount_Jefferson_(Oregon), http://dbpedia.org/resource/Mount_Washington_(Oregon), http://dbpedia.org/resource/Hart_Mountain, http://dbpedia.org/resource/Tom_Dick_and_Harry_Mountain, http://dbpedia.org/resource/Pilot_Rock_(Jackson_County,_Oregon), http://dbpedia.org/resource/Morrison_Bridge, http://dbpedia.org/resource/Pilot_Butte_(Oregon), http://dbpedia.org/resource/Powell_Butte, http://dbpedia.org/resource/Rocky_Butte, http://dbpedia.org/resource/Mount_Ashland, http://dbpedia.org/resource/Abert_Rim, http://dbpedia.org/resource/Broken_Top, http://dbpedia.org/resource/Fort_Rock, http://dbpedia.org/resource/Hole-in-the-Ground, http://dbpedia.org/resource/Lava_Butte, http://dbpedia.org/resource/Maiden_Peak_(Oregon), http://dbpedia.org/resource/Mount_Hebo, http://dbpedia.org/resource/Rogers_Peak, http://dbpedia.org/resource/Saddle_Mountain_(Clatsop_County,_Oregon), http://dbpedia.org/resource/Strawberry_Mountain_(Oregon), http://dbpedia.org/resource/Trask_Mountain, http://dbpedia.org/resource/Tumalo_Mountain, http://dbpedia.org/resource/Sacajawea_Peak, http://dbpedia.org/resource/Cooper_Mountain_(Oregon), http://dbpedia.org/resource/Diamond_Craters, http://dbpedia.org/resource/Zigzag_Mountain, http://dbpedia.org/resource/Mahogany_Mountain, http://dbpedia.org/resource/Quartz_Mountain_(Douglas_County,_Oregon), http://dbpedia.org/resource/Pelican_Butte, http://dbpedia.org/resource/Hayden_Mountain_(Oregon), http://dbpedia.org/resource/Brown_Mountain_(Klamath_County,_Oregon), http://dbpedia.org/resource/Lookout_Mountain_(Oregon), http://dbpedia.org/resource/Steens_Mountain, http://dbpedia.org/resource/Marys_Peak, http://dbpedia.org/resource/South_Saddle_Mountain, http://dbpedia.org/resource/Mount_Thielsen, http://dbpedia.org/resource/Garfield_Peak_(Oregon), http://dbpedia.org/resource/Mount_Scott_(Klamath_County,_Oregon), http://dbpedia.org/resource/Union_Peak, http://dbpedia.org/resource/Three_Fingered_Jack, http://dbpedia.org/resource/Bohemia_Mountain, http://dbpedia.org/resource/Olallie_Butte, http://dbpedia.org/resource/Aspen_Butte, http://dbpedia.org/resource/Howlock_Mountain, http://dbpedia.org/resource/Bald_Peak, http://dbpedia.org/resource/Black_Crater, http://dbpedia.org/resource/Devil's_Garden_volcanic_field, http://dbpedia.org/resource/Squaw_Ridge_Lava_Field, http://dbpedia.org/resource/Yamsay_Mountain, http://dbpedia.org/resource/Baldy_(Jackson_County,_Oregon), http://dbpedia.org/resource/New_Youngs_Bay_Bridge, http://dbpedia.org/resource/Round_Top_(Oregon), http://dbpedia.org/resource/Williamson_Mountain] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 262, P: 0.545, R: 0.604, macro F1: 0.554, macro F1*: 0.573 -[INFO] Current time: 4887 ms, average: 2842.67 ms/ques, entity detection: 390.58 ms/ques (linking tool: 299.93 ms/ques), relation detection: 926.31 ms/ques (similarity: 721.10 ms/ques), EDG generation: 66.14 ms/ques +[INFO] Cumulative metrics, sample: 262, P: 0.545, R: 0.604, macro F1: 0.573 +[INFO] Current question consumed time: 4419 ms, average: 2646.69 ms/ques, entity detection: 362.27 ms/ques (linking tool: 280.04 ms/ques), relation detection: 884.78 ms/ques (similarity: 709.02 ms/ques), EDG generation: 65.42 ms/ques [INFO] Entity Linking P: 0.7557251908396947, R: 0.7156488549618321, micro F1: 0.7283715012722645, macro F1: 0.735141236225384 [INFO] Relation Linking P: 0.5960559796437659, R: 0.5782442748091603, micro F1: 0.5802798982188296, macro F1: 0.5870150438745214 [INFO] Type Linking P: 0.5343511450381679, R: 0.5343511450381679, micro F1: 0.5343511450381679, macro F1: 0.5343511450381679 @@ -3676,8 +3676,8 @@ Question 262: Was True Grit a novel of the western genre? [INFO] Golden Answer: [true] [INFO] Predicted Answer: [true] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 263, P: 0.547, R: 0.606, macro F1: 0.555, macro F1*: 0.575 -[INFO] Current time: 1625 ms, average: 2838.04 ms/ques, entity detection: 390.54 ms/ques (linking tool: 299.83 ms/ques), relation detection: 924.23 ms/ques (similarity: 718.96 ms/ques), EDG generation: 65.95 ms/ques +[INFO] Cumulative metrics, sample: 263, P: 0.547, R: 0.606, macro F1: 0.575 +[INFO] Current question consumed time: 1385 ms, average: 2641.90 ms/ques, entity detection: 362.16 ms/ques (linking tool: 279.89 ms/ques), relation detection: 882.78 ms/ques (similarity: 706.93 ms/ques), EDG generation: 65.24 ms/ques [INFO] Entity Linking P: 0.7566539923954373, R: 0.7167300380228137, micro F1: 0.7294043092522178, macro F1: 0.7361511100208513 [INFO] Relation Linking P: 0.5975918884664131, R: 0.5798479087452472, micro F1: 0.581875792141952, macro F1: 0.5885861980051323 [INFO] Type Linking P: 0.5361216730038023, R: 0.5361216730038023, micro F1: 0.5361216730038023, macro F1: 0.5361216730038023 @@ -3690,8 +3690,8 @@ Question 263: Which notable series of Russell T Davies is also the series of Sho [INFO] Golden Answer: [http://dbpedia.org/resource/Doctor_Who] [INFO] Predicted Answer: [http://dbpedia.org/resource/Doctor_Who] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 264, P: 0.548, R: 0.607, macro F1: 0.557, macro F1*: 0.576 -[INFO] Current time: 2285 ms, average: 2835.94 ms/ques, entity detection: 390.82 ms/ques (linking tool: 299.58 ms/ques), relation detection: 922.47 ms/ques (similarity: 717.23 ms/ques), EDG generation: 66.41 ms/ques +[INFO] Cumulative metrics, sample: 264, P: 0.548, R: 0.607, macro F1: 0.576 +[INFO] Current question consumed time: 1969 ms, average: 2639.35 ms/ques, entity detection: 362.40 ms/ques (linking tool: 279.56 ms/ques), relation detection: 881.02 ms/ques (similarity: 705.23 ms/ques), EDG generation: 65.67 ms/ques [INFO] Entity Linking P: 0.7556818181818182, R: 0.7159090909090909, micro F1: 0.7285353535353534, macro F1: 0.7352579852579852 [INFO] Relation Linking P: 0.5991161616161615, R: 0.5814393939393939, micro F1: 0.5834595959595961, macro F1: 0.5901454383406255 [INFO] Type Linking P: 0.5378787878787878, R: 0.5378787878787878, micro F1: 0.5378787878787878, macro F1: 0.5378787878787878 @@ -3704,8 +3704,8 @@ Question 264: Which battles were fought under the president when Chung Won Shik [INFO] Golden Answer: [http://dbpedia.org/resource/Vietnam_War] [INFO] Predicted Answer: [http://dbpedia.org/resource/Vietnam_War] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 265, P: 0.550, R: 0.609, macro F1: 0.559, macro F1*: 0.578 -[INFO] Current time: 2960 ms, average: 2836.41 ms/ques, entity detection: 389.91 ms/ques (linking tool: 298.90 ms/ques), relation detection: 923.17 ms/ques (similarity: 719.16 ms/ques), EDG generation: 66.31 ms/ques +[INFO] Cumulative metrics, sample: 265, P: 0.550, R: 0.609, macro F1: 0.578 +[INFO] Current question consumed time: 2789 ms, average: 2639.91 ms/ques, entity detection: 361.61 ms/ques (linking tool: 278.97 ms/ques), relation detection: 881.62 ms/ques (similarity: 707.15 ms/ques), EDG generation: 65.55 ms/ques [INFO] Entity Linking P: 0.7566037735849057, R: 0.7169811320754716, micro F1: 0.7295597484276728, macro F1: 0.7362597540646969 [INFO] Relation Linking P: 0.5987421383647799, R: 0.5811320754716981, micro F1: 0.5831446540880504, macro F1: 0.5898056885384398 [INFO] Type Linking P: 0.5358490566037736, R: 0.5358490566037736, micro F1: 0.5358490566037736, macro F1: 0.5358490566037736 @@ -3718,8 +3718,8 @@ Question 265: Which office holder's resting place is in Palo Alto, California a [INFO] Golden Answer: [http://dbpedia.org/resource/Shirley_Temple] [INFO] Predicted Answer: [http://dbpedia.org/resource/United_EMS_Workers_–_AFSCME_Local_4911, http://dbpedia.org/resource/International_Longshore_and_Warehouse_Union] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 266, P: 0.548, R: 0.607, macro F1: 0.557, macro F1*: 0.576 -[INFO] Current time: 17266 ms, average: 2890.66 ms/ques, entity detection: 395.61 ms/ques (linking tool: 304.23 ms/ques), relation detection: 939.97 ms/ques (similarity: 734.23 ms/ques), EDG generation: 66.47 ms/ques +[INFO] Cumulative metrics, sample: 266, P: 0.548, R: 0.607, macro F1: 0.576 +[INFO] Current question consumed time: 16118 ms, average: 2690.58 ms/ques, entity detection: 367.21 ms/ques (linking tool: 284.27 ms/ques), relation detection: 897.56 ms/ques (similarity: 722.24 ms/ques), EDG generation: 65.70 ms/ques [INFO] Entity Linking P: 0.7537593984962406, R: 0.7142857142857143, micro F1: 0.7268170426065161, macro F1: 0.7334918602524236 [INFO] Relation Linking P: 0.5964912280701754, R: 0.5789473684210527, micro F1: 0.580952380952381, macro F1: 0.5875883739198742 [INFO] Type Linking P: 0.5338345864661654, R: 0.5338345864661654, micro F1: 0.5338345864661654, macro F1: 0.5338345864661654 @@ -3732,8 +3732,8 @@ Question 266: which movies have been published by Cinema International Corporati [INFO] Golden Answer: [http://dbpedia.org/resource/Monty_Python's_Life_of_Brian, http://dbpedia.org/resource/Serpico, http://dbpedia.org/resource/The_Outsider_(1980_film), http://dbpedia.org/resource/Brother_Sun,_Sister_Moon, http://dbpedia.org/resource/The_Lost_Honour_of_Katharina_Blum_(film), http://dbpedia.org/resource/Watership_Down_(film), http://dbpedia.org/resource/Full_Circle_(1977_film), http://dbpedia.org/resource/Carmen_Miranda:_Bananas_is_My_Business, http://dbpedia.org/resource/The_Eagle_Has_Landed_(film), http://dbpedia.org/resource/Black_Moon_(1975_film), http://dbpedia.org/resource/The_Twelve_Tasks_of_Asterix, http://dbpedia.org/resource/Gallipoli_(1981_film), http://dbpedia.org/resource/Rooster_Cogburn_(film), http://dbpedia.org/resource/UFOria, http://dbpedia.org/resource/Bread_and_Chocolate, http://dbpedia.org/resource/Comeback_(film), http://dbpedia.org/resource/Let_Joy_Reign_Supreme, http://dbpedia.org/resource/Burning_Rubber, http://dbpedia.org/resource/North_Sea_Hijack, http://dbpedia.org/resource/The_Slipper_and_the_Rose, http://dbpedia.org/resource/Gangster_Wars, http://dbpedia.org/resource/Lacombe,_Lucien] [INFO] Predicted Answer: [http://dbpedia.org/resource/Monty_Python's_Life_of_Brian, http://dbpedia.org/resource/Serpico, http://dbpedia.org/resource/The_Outsider_(1980_film), http://dbpedia.org/resource/Brother_Sun,_Sister_Moon, http://dbpedia.org/resource/The_Lost_Honour_of_Katharina_Blum_(film), http://dbpedia.org/resource/Watership_Down_(film), http://dbpedia.org/resource/Full_Circle_(1977_film), http://dbpedia.org/resource/Carmen_Miranda:_Bananas_is_My_Business, http://dbpedia.org/resource/The_Eagle_Has_Landed_(film), http://dbpedia.org/resource/Black_Moon_(1975_film), http://dbpedia.org/resource/The_Twelve_Tasks_of_Asterix, http://dbpedia.org/resource/Gallipoli_(1981_film), http://dbpedia.org/resource/Rooster_Cogburn_(film), http://dbpedia.org/resource/UFOria, http://dbpedia.org/resource/Bread_and_Chocolate, http://dbpedia.org/resource/Comeback_(film), http://dbpedia.org/resource/Let_Joy_Reign_Supreme, http://dbpedia.org/resource/Burning_Rubber, http://dbpedia.org/resource/North_Sea_Hijack, http://dbpedia.org/resource/The_Slipper_and_the_Rose, http://dbpedia.org/resource/Gangster_Wars, http://dbpedia.org/resource/Lacombe,_Lucien, http://dbpedia.org/resource/The_Valachi_Papers] [INFO] P: 0.9565217391304348, R: 1.0, F: 0.9777777777777777 -[INFO] Cumulative metrics, sample: 267, P: 0.550, R: 0.608, macro F1: 0.558, macro F1*: 0.577 -[INFO] Current time: 694 ms, average: 2882.43 ms/ques, entity detection: 394.84 ms/ques (linking tool: 303.47 ms/ques), relation detection: 936.89 ms/ques (similarity: 731.81 ms/ques), EDG generation: 66.26 ms/ques +[INFO] Cumulative metrics, sample: 267, P: 0.550, R: 0.608, macro F1: 0.577 +[INFO] Current question consumed time: 643 ms, average: 2682.91 ms/ques, entity detection: 366.47 ms/ques (linking tool: 283.54 ms/ques), relation detection: 894.64 ms/ques (similarity: 719.88 ms/ques), EDG generation: 65.49 ms/ques [INFO] Entity Linking P: 0.7546816479400749, R: 0.7153558052434457, micro F1: 0.7278401997503119, macro F1: 0.7344927121353085 [INFO] Relation Linking P: 0.5980024968789014, R: 0.5805243445692884, micro F1: 0.5825218476903871, macro F1: 0.589133815781121 [INFO] Type Linking P: 0.5318352059925093, R: 0.5318352059925093, micro F1: 0.5318352059925093, macro F1: 0.5318352059925093 @@ -3746,8 +3746,8 @@ Question 267: What is the total number of other destinations of the airlines who [INFO] Golden Answer: [8] [INFO] Predicted Answer: [8] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 268, P: 0.551, R: 0.609, macro F1: 0.560, macro F1*: 0.579 -[INFO] Current time: 3269 ms, average: 2883.87 ms/ques, entity detection: 395.77 ms/ques (linking tool: 304.46 ms/ques), relation detection: 937.14 ms/ques (similarity: 733.39 ms/ques), EDG generation: 66.41 ms/ques +[INFO] Cumulative metrics, sample: 268, P: 0.551, R: 0.609, macro F1: 0.579 +[INFO] Current question consumed time: 2889 ms, average: 2683.68 ms/ques, entity detection: 367.31 ms/ques (linking tool: 284.45 ms/ques), relation detection: 894.82 ms/ques (similarity: 720.42 ms/ques), EDG generation: 65.62 ms/ques [INFO] Entity Linking P: 0.7555970149253731, R: 0.7164179104477612, micro F1: 0.7288557213930347, macro F1: 0.735486067760059 [INFO] Relation Linking P: 0.597636815920398, R: 0.582089552238806, micro F1: 0.5828358208955224, macro F1: 0.5897607376926647 [INFO] Type Linking P: 0.5298507462686567, R: 0.5298507462686567, micro F1: 0.5298507462686567, macro F1: 0.5298507462686567 @@ -3760,8 +3760,8 @@ Question 268: Count the total number of academic discipline of the journals whos [INFO] Golden Answer: [6] [INFO] Predicted Answer: [6] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 269, P: 0.553, R: 0.611, macro F1: 0.561, macro F1*: 0.580 -[INFO] Current time: 1256 ms, average: 2877.82 ms/ques, entity detection: 394.87 ms/ques (linking tool: 303.62 ms/ques), relation detection: 935.53 ms/ques (similarity: 732.04 ms/ques), EDG generation: 66.34 ms/ques +[INFO] Cumulative metrics, sample: 269, P: 0.553, R: 0.611, macro F1: 0.580 +[INFO] Current question consumed time: 1126 ms, average: 2677.89 ms/ques, entity detection: 366.42 ms/ques (linking tool: 283.65 ms/ques), relation detection: 893.19 ms/ques (similarity: 719.14 ms/ques), EDG generation: 65.56 ms/ques [INFO] Entity Linking P: 0.7565055762081785, R: 0.7174721189591078, micro F1: 0.7298636926889713, macro F1: 0.7364720111383528 [INFO] Relation Linking P: 0.5991325898389095, R: 0.5836431226765799, micro F1: 0.5843866171003718, macro F1: 0.5912864322978028 [INFO] Type Linking P: 0.5315985130111525, R: 0.5315985130111525, micro F1: 0.5315985130111525, macro F1: 0.5315985130111525 @@ -3774,8 +3774,8 @@ Question 269: List the award which have been won by both Charles LeMaire and Nic [INFO] Golden Answer: [http://dbpedia.org/resource/Academy_Awards] [INFO] Predicted Answer: [http://dbpedia.org/resource/Academy_Awards] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 270, P: 0.555, R: 0.612, macro F1: 0.563, macro F1*: 0.582 -[INFO] Current time: 2083 ms, average: 2874.88 ms/ques, entity detection: 395.03 ms/ques (linking tool: 303.46 ms/ques), relation detection: 933.69 ms/ques (similarity: 731.09 ms/ques), EDG generation: 66.33 ms/ques +[INFO] Cumulative metrics, sample: 270, P: 0.555, R: 0.612, macro F1: 0.582 +[INFO] Current question consumed time: 1851 ms, average: 2674.83 ms/ques, entity detection: 366.49 ms/ques (linking tool: 283.46 ms/ques), relation detection: 891.39 ms/ques (similarity: 718.18 ms/ques), EDG generation: 65.54 ms/ques [INFO] Entity Linking P: 0.7574074074074074, R: 0.7185185185185186, micro F1: 0.7308641975308641, macro F1: 0.7374506250290442 [INFO] Relation Linking P: 0.6006172839506172, R: 0.5851851851851851, micro F1: 0.585925925925926, macro F1: 0.592800817475466 [INFO] Type Linking P: 0.5296296296296297, R: 0.5296296296296297, micro F1: 0.5296296296296297, macro F1: 0.5296296296296297 @@ -3788,8 +3788,8 @@ Question 270: What are some cities which have a Ferris wheel? [INFO] Golden Answer: [http://dbpedia.org/resource/Seattle, http://dbpedia.org/resource/Brighton] [INFO] Predicted Answer: [http://dbpedia.org/resource/List_of_licensed_and_localized_editions_of_Monopoly:_USA, http://dbpedia.org/resource/Monopoly_Junior] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 271, P: 0.553, R: 0.610, macro F1: 0.561, macro F1*: 0.580 -[INFO] Current time: 900 ms, average: 2867.59 ms/ques, entity detection: 394.33 ms/ques (linking tool: 302.82 ms/ques), relation detection: 930.82 ms/ques (similarity: 728.99 ms/ques), EDG generation: 66.14 ms/ques +[INFO] Cumulative metrics, sample: 271, P: 0.553, R: 0.610, macro F1: 0.580 +[INFO] Current question consumed time: 828 ms, average: 2668.01 ms/ques, entity detection: 365.83 ms/ques (linking tool: 282.86 ms/ques), relation detection: 888.65 ms/ques (similarity: 716.13 ms/ques), EDG generation: 65.36 ms/ques [INFO] Entity Linking P: 0.7564575645756457, R: 0.7195571955719557, micro F1: 0.7306273062730625, macro F1: 0.7375461254612546 [INFO] Relation Linking P: 0.5984009840098401, R: 0.5830258302583026, micro F1: 0.5837638376383765, macro F1: 0.5906133605844126 [INFO] Type Linking P: 0.5313653136531366, R: 0.5313653136531366, micro F1: 0.5313653136531366, macro F1: 0.5313653136531366 @@ -3802,8 +3802,8 @@ Question 271: Name the movie who has screenplay by Akiva Goldsman and is edited [INFO] Golden Answer: [http://dbpedia.org/resource/Batman_Forever] [INFO] Predicted Answer: [http://dbpedia.org/resource/The_Da_Vinci_Code_(film), http://dbpedia.org/resource/I,_Robot_(film), http://dbpedia.org/resource/Angels_&_Demons_(film), http://dbpedia.org/resource/Batman_Forever, http://dbpedia.org/resource/The_Client_(1994_film), http://dbpedia.org/resource/A_Beautiful_Mind_(film), http://dbpedia.org/resource/The_Divergent_Series:_Insurgent, http://dbpedia.org/resource/A_Time_to_Kill_(1996_film), http://dbpedia.org/resource/The_5th_Wave_(film)] [INFO] P: 0.1111111111111111, R: 1.0, F: 0.19999999999999998 -[INFO] Cumulative metrics, sample: 272, P: 0.551, R: 0.612, macro F1: 0.560, macro F1*: 0.580 -[INFO] Current time: 2840 ms, average: 2867.49 ms/ques, entity detection: 394.28 ms/ques (linking tool: 302.52 ms/ques), relation detection: 930.68 ms/ques (similarity: 729.93 ms/ques), EDG generation: 66.06 ms/ques +[INFO] Cumulative metrics, sample: 272, P: 0.551, R: 0.612, macro F1: 0.580 +[INFO] Current question consumed time: 2546 ms, average: 2667.57 ms/ques, entity detection: 365.74 ms/ques (linking tool: 282.49 ms/ques), relation detection: 888.34 ms/ques (similarity: 717.13 ms/ques), EDG generation: 65.27 ms/ques [INFO] Entity Linking P: 0.7573529411764706, R: 0.71875, micro F1: 0.7303921568627448, macro F1: 0.737546699875467 [INFO] Relation Linking P: 0.5998774509803921, R: 0.5827205882352942, micro F1: 0.5840686274509804, macro F1: 0.5911745656811948 [INFO] Type Linking P: 0.5294117647058824, R: 0.5294117647058824, micro F1: 0.5294117647058824, macro F1: 0.5294117647058824 @@ -3816,8 +3816,8 @@ Question 272: Count the movies in Swedish language? [INFO] Golden Answer: [197] [INFO] Predicted Answer: [] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 273, P: 0.549, R: 0.609, macro F1: 0.558, macro F1*: 0.578 -[INFO] Current time: 734 ms, average: 2859.68 ms/ques, entity detection: 393.49 ms/ques (linking tool: 301.88 ms/ques), relation detection: 927.56 ms/ques (similarity: 727.44 ms/ques), EDG generation: 65.85 ms/ques +[INFO] Cumulative metrics, sample: 273, P: 0.549, R: 0.609, macro F1: 0.578 +[INFO] Current question consumed time: 630 ms, average: 2660.10 ms/ques, entity detection: 364.96 ms/ques (linking tool: 281.84 ms/ques), relation detection: 885.36 ms/ques (similarity: 714.68 ms/ques), EDG generation: 65.06 ms/ques [INFO] Entity Linking P: 0.7545787545787546, R: 0.7161172161172161, micro F1: 0.7277167277167275, macro F1: 0.7348450636121868 [INFO] Relation Linking P: 0.5976800976800977, R: 0.5805860805860806, micro F1: 0.581929181929182, macro F1: 0.58900909108163 [INFO] Type Linking P: 0.5274725274725275, R: 0.5274725274725275, micro F1: 0.5274725274725275, macro F1: 0.5274725274725275 @@ -3830,8 +3830,8 @@ Question 273: Give me the total number of television shows whose theme music com [INFO] Golden Answer: [9] [INFO] Predicted Answer: [57, 68] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 274, P: 0.547, R: 0.607, macro F1: 0.556, macro F1*: 0.575 -[INFO] Current time: 1918 ms, average: 2856.24 ms/ques, entity detection: 393.43 ms/ques (linking tool: 301.60 ms/ques), relation detection: 925.47 ms/ques (similarity: 726.08 ms/ques), EDG generation: 65.89 ms/ques +[INFO] Cumulative metrics, sample: 274, P: 0.547, R: 0.607, macro F1: 0.575 +[INFO] Current question consumed time: 1698 ms, average: 2656.59 ms/ques, entity detection: 364.78 ms/ques (linking tool: 281.51 ms/ques), relation detection: 883.27 ms/ques (similarity: 713.36 ms/ques), EDG generation: 65.10 ms/ques [INFO] Entity Linking P: 0.7554744525547445, R: 0.7153284671532847, micro F1: 0.727493917274939, macro F1: 0.734853561790223 [INFO] Relation Linking P: 0.5991484184914841, R: 0.5802919708029197, micro F1: 0.5822384428223845, macro F1: 0.5895694597636677 [INFO] Type Linking P: 0.5255474452554745, R: 0.5255474452554745, micro F1: 0.5255474452554745, macro F1: 0.5255474452554745 @@ -3844,8 +3844,8 @@ Question 274: Which cloud platform of Google cloud connect also provides service [INFO] Golden Answer: [http://dbpedia.org/resource/Microsoft_Office] [INFO] Predicted Answer: [http://dbpedia.org/resource/Microsoft_Office] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 275, P: 0.549, R: 0.608, macro F1: 0.557, macro F1*: 0.577 -[INFO] Current time: 1325 ms, average: 2850.67 ms/ques, entity detection: 393.27 ms/ques (linking tool: 301.37 ms/ques), relation detection: 922.84 ms/ques (similarity: 723.90 ms/ques), EDG generation: 65.73 ms/ques +[INFO] Cumulative metrics, sample: 275, P: 0.549, R: 0.608, macro F1: 0.577 +[INFO] Current question consumed time: 1175 ms, average: 2651.20 ms/ques, entity detection: 364.52 ms/ques (linking tool: 281.23 ms/ques), relation detection: 880.79 ms/ques (similarity: 711.23 ms/ques), EDG generation: 64.95 ms/ques [INFO] Entity Linking P: 0.7563636363636363, R: 0.7163636363636363, micro F1: 0.7284848484848482, macro F1: 0.7358204264870931 [INFO] Relation Linking P: 0.6006060606060606, R: 0.5818181818181818, micro F1: 0.5837575757575757, macro F1: 0.591062858207912 [INFO] Type Linking P: 0.5272727272727272, R: 0.5272727272727272, micro F1: 0.5272727272727272, macro F1: 0.5272727272727272 @@ -3858,8 +3858,8 @@ Question 275: Name the office holder whose constituency is Haight-Ashbury? [INFO] Golden Answer: [http://dbpedia.org/resource/Harvey_Milk] [INFO] Predicted Answer: [http://dbpedia.org/resource/Harvey_Milk] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 276, P: 0.550, R: 0.610, macro F1: 0.559, macro F1*: 0.579 -[INFO] Current time: 1426 ms, average: 2845.51 ms/ques, entity detection: 392.49 ms/ques (linking tool: 300.75 ms/ques), relation detection: 921.41 ms/ques (similarity: 723.37 ms/ques), EDG generation: 65.56 ms/ques +[INFO] Cumulative metrics, sample: 276, P: 0.550, R: 0.610, macro F1: 0.579 +[INFO] Current question consumed time: 1301 ms, average: 2646.31 ms/ques, entity detection: 363.74 ms/ques (linking tool: 280.61 ms/ques), relation detection: 879.33 ms/ques (similarity: 710.75 ms/ques), EDG generation: 64.78 ms/ques [INFO] Entity Linking P: 0.7572463768115942, R: 0.717391304347826, micro F1: 0.7294685990338161, macro F1: 0.736780258519389 [INFO] Relation Linking P: 0.6020531400966184, R: 0.5833333333333334, micro F1: 0.5852657004830918, macro F1: 0.5925454236712515 [INFO] Type Linking P: 0.5253623188405797, R: 0.5253623188405797, micro F1: 0.5253623188405797, macro F1: 0.5253623188405797 @@ -3872,8 +3872,8 @@ Question 276: Name the loocation of Lekh Castle which is also the location of Sa [INFO] Golden Answer: [http://dbpedia.org/resource/Azerbaijan] [INFO] Predicted Answer: [http://dbpedia.org/resource/Aquinas_School, http://dbpedia.org/resource/EastWest_Bank, http://dbpedia.org/resource/Equicom_Savings_Bank, http://dbpedia.org/resource/Land_Bank_of_the_Philippines, http://dbpedia.org/resource/Radio_Philippines_Network, http://dbpedia.org/resource/Union_Bank_of_the_Philippines, http://dbpedia.org/resource/House_of_Representatives_of_the_Philippines, http://dbpedia.org/resource/National_Intelligence_Coordinating_Agency, http://dbpedia.org/resource/Glorietta, http://dbpedia.org/resource/Gateway_Mall_(Araneta_Center), http://dbpedia.org/resource/TriNoma, http://dbpedia.org/resource/Asia_Theological_Association, http://dbpedia.org/resource/Aso_ni_San_Roque, http://dbpedia.org/resource/United_Defense_Manufacturing_Corporation, http://dbpedia.org/resource/Rizal_Memorial_Stadium, http://dbpedia.org/resource/Movie_and_Television_Review_and_Classification_Board, http://dbpedia.org/resource/Ormoc_Bay, http://dbpedia.org/resource/Philippine_Broadcasting_Service, http://dbpedia.org/resource/MacArthur_Landing_Memorial_National_Park, http://dbpedia.org/resource/The_Gramercy_Residences, http://dbpedia.org/resource/The_Knightsbridge_Residences, http://dbpedia.org/resource/Isetann_Cinerama_Recto, http://dbpedia.org/resource/De_La_Salle_University_Science_and_Technology_Complex, http://dbpedia.org/resource/National_Police_Commission_(Philippines), http://dbpedia.org/resource/Naval_Station_Sangley_Point, http://dbpedia.org/resource/Philippine_Amateur_Baseball_Association, http://dbpedia.org/resource/Philippine_Veterans_Bank, http://dbpedia.org/resource/RCBC_Plaza, http://dbpedia.org/resource/SM_City_Bacolod, http://dbpedia.org/resource/SM_Investments_Corporation, http://dbpedia.org/resource/International_Exchange_Bank, http://dbpedia.org/resource/National_Citizens'_Movement_for_Free_Elections, http://dbpedia.org/resource/SM_City_Bacoor, http://dbpedia.org/resource/Ozine_Fest, http://dbpedia.org/resource/9th_Cinemalaya_Independent_Film_Festival, http://dbpedia.org/resource/Asia_Brewery, http://dbpedia.org/resource/Magnolia,_Inc., http://dbpedia.org/resource/Magnolia_Purewater_Wizards, http://dbpedia.org/resource/United_Coconut_Planters_Bank, http://dbpedia.org/resource/Philippine_Amateur_Radio_Association, http://dbpedia.org/resource/Cebu_Provincial_Detention_and_Rehabilitation_Center, http://dbpedia.org/resource/Pasig_River_Light, http://dbpedia.org/resource/Yellow_Cab_Pizza, http://dbpedia.org/resource/Arch_of_the_Centuries, http://dbpedia.org/resource/University_of_Santo_Tomas_Field, http://dbpedia.org/resource/ABL_Sports_&_Cultural_Complex, http://dbpedia.org/resource/Cebu_Coliseum, http://dbpedia.org/resource/KJC_King_Dome, http://dbpedia.org/resource/Tacloban_City_Convention_Center, http://dbpedia.org/resource/Ynares_Sports_Arena, http://dbpedia.org/resource/Zamboanga_City_Coliseum, http://dbpedia.org/resource/Association_for_Theological_Education_in_South_East_Asia, http://dbpedia.org/resource/Robinsons_Summit_Center, http://dbpedia.org/resource/Punta_Fuego, http://dbpedia.org/resource/Mounts_Banahaw–San_Cristobal_Protected_Landscape, http://dbpedia.org/resource/ABS-CBN_Insider, http://dbpedia.org/resource/Magdalena_(TV_series), http://dbpedia.org/resource/Philippine_Association_of_State_Universities_and_Colleges, http://dbpedia.org/resource/United_Central_Luzon_Athletic_Association, http://dbpedia.org/resource/Export_and_Industry_Bank, http://dbpedia.org/resource/PLDT_Communication_and_Energy_Ventures, http://dbpedia.org/resource/St._Joseph_the_Worker_Chapel,_Victorias, http://dbpedia.org/resource/Adik_Sa'Yo, http://dbpedia.org/resource/All_About_Eve_(Philippine_TV_series), http://dbpedia.org/resource/All_My_Life_(TV_series), http://dbpedia.org/resource/Broken_Vow_(TV_series), http://dbpedia.org/resource/Claudine_(TV_series), http://dbpedia.org/resource/Gagambino, http://dbpedia.org/resource/Hiram_na_Puso, http://dbpedia.org/resource/I_Heart_You,_Pare!, http://dbpedia.org/resource/Kaya_Kong_Abutin_Ang_Langit, http://dbpedia.org/resource/TV5_Network_Inc., http://dbpedia.org/resource/The_Good_Daughter, http://dbpedia.org/resource/GMA_New_Media, http://dbpedia.org/resource/Iglot_(TV_series), http://dbpedia.org/resource/Guns_and_Roses_(TV_series), http://dbpedia.org/resource/Joaquin_Bordado, http://dbpedia.org/resource/Kailangan_Ko'y_Ikaw_(TV_series), http://dbpedia.org/resource/PBN_Broadcasting_Network,_Inc., http://dbpedia.org/resource/Philippine_International_Hot_Air_Balloon_Fiesta, http://dbpedia.org/resource/Penshoppe, http://dbpedia.org/resource/PhilSports_Football_and_Athletics_Stadium, http://dbpedia.org/resource/The_Message_(Philippine_TV_series), http://dbpedia.org/resource/Yagit_(TV_series), http://dbpedia.org/resource/Rosario_Sports_Complex, http://dbpedia.org/resource/Antonio_Bautista_Air_Base, http://dbpedia.org/resource/Biñan_Football_Stadium, http://dbpedia.org/resource/Paroa:_Ang_Kuwento_ni_Mariposa, http://dbpedia.org/resource/Baycomms_Broadcasting_Corporation, http://dbpedia.org/resource/FlordeLiza, http://dbpedia.org/resource/Momay, http://dbpedia.org/resource/Altimax_Broadcasting_Company, http://dbpedia.org/resource/Maxandrea_Hotel, http://dbpedia.org/resource/Tiger_International_Resources, http://dbpedia.org/resource/Chowtime_Na!, http://dbpedia.org/resource/Somewhere_in_My_Heart_(TV_series), http://dbpedia.org/resource/Youth_Congress_on_Information_Technology, http://dbpedia.org/resource/Alculympics, http://dbpedia.org/resource/University_of_Makati_Stadium, http://dbpedia.org/resource/Student_Government_in_the_Philippines, http://dbpedia.org/resource/Aguinaldo_International_School_Manila, http://dbpedia.org/resource/McKinley_Hill_Stadium, http://dbpedia.org/resource/Las_Piñas_Baptist_Academy, http://dbpedia.org/resource/CSI_Bayambang, http://dbpedia.org/resource/Arki_Alumni_Basketball, http://dbpedia.org/resource/Manila_Light_Rail_Transit_System_Line_1, http://dbpedia.org/resource/Bohol_Sea, http://dbpedia.org/resource/Bureau_of_Jail_Management_and_Penology, http://dbpedia.org/resource/Davao_Gulf, http://dbpedia.org/resource/Illana_Bay, http://dbpedia.org/resource/National_Power_Corporation, http://dbpedia.org/resource/Securities_and_Exchange_Commission_(Philippines), http://dbpedia.org/resource/National_Shrine_of_Our_Lady_of_Fatima, http://dbpedia.org/resource/Ambuklao_Dam, http://dbpedia.org/resource/Angat_Dam, http://dbpedia.org/resource/Bustos_Dam, http://dbpedia.org/resource/San_Roque_Dam_(Philippines), http://dbpedia.org/resource/UFC_Fight_Night:_Edgar_vs._Faber, http://dbpedia.org/resource/Kapisanan_ng_mga_Brodkaster_ng_Pilipinas, http://dbpedia.org/resource/Quirino_Grandstand, http://dbpedia.org/resource/Callao_Cave, http://dbpedia.org/resource/San_Miguel_Brewery, http://dbpedia.org/resource/Verde_Island_Passage, http://dbpedia.org/resource/National_Anti-Poverty_Commission_(Philippines), http://dbpedia.org/resource/Presidential_Commission_on_Good_Government, http://dbpedia.org/resource/Radio_Veritas, http://dbpedia.org/resource/SM_City_San_Pablo, http://dbpedia.org/resource/1322_Golden_Empire_Tower, http://dbpedia.org/resource/Arroceros_Forest_Park, http://dbpedia.org/resource/Greenhills_Shopping_Center, http://dbpedia.org/resource/National_Archives_of_the_Philippines, http://dbpedia.org/resource/South_SEED_LPDH_College, http://dbpedia.org/resource/Southville_International_School_and_Colleges, http://dbpedia.org/resource/Stonyhurst_Southville_International_School, http://dbpedia.org/resource/British_School_Manila, http://dbpedia.org/resource/InterContinental_Manila, http://dbpedia.org/resource/The_SM_Store, http://dbpedia.org/resource/Ang_Probinsyano, http://dbpedia.org/resource/Del_Carmen_Airfield, http://dbpedia.org/resource/Moon_of_Desire, http://dbpedia.org/resource/Tasya_Fantasya, http://dbpedia.org/resource/YCO_Shine_Masters, http://dbpedia.org/resource/Fuyot_Springs_National_Park, http://dbpedia.org/resource/First_Time_(TV_series), http://dbpedia.org/resource/Shangri-La_at_the_Fort,_Manila, http://dbpedia.org/resource/Biritera, http://dbpedia.org/resource/San_Miguel-Team_Pilipinas, http://dbpedia.org/resource/Basahang_Ginto, http://dbpedia.org/resource/Cape_Melville_Lighthouse, http://dbpedia.org/resource/MORESCO_1, http://dbpedia.org/resource/Pelaez_Ancestral_House, http://dbpedia.org/resource/Vega_Ancestral_House, http://dbpedia.org/resource/The_Philippine_Order_of_Narnians, http://dbpedia.org/resource/Love_Is_Only_in_the_Movies, http://dbpedia.org/resource/Bakit_Manipis_ang_Ulap%3F, http://dbpedia.org/resource/Larawan:_A_Special_Drama_Engagement, http://dbpedia.org/resource/Project_NOAH_(Philippines), http://dbpedia.org/resource/Kambal_sa_Uma_(TV_series), http://dbpedia.org/resource/My_Little_Juan, http://dbpedia.org/resource/Lumang_Piso_Para_sa_Puso, http://dbpedia.org/resource/Office_for_Transportation_Security, http://dbpedia.org/resource/Maradeka, http://dbpedia.org/resource/You're_Mine,_Only_Mine, http://dbpedia.org/resource/11th_Cinemalaya_Independent_Film_Festival, http://dbpedia.org/resource/2nd_Cinemalaya_Independent_Film_Festival, http://dbpedia.org/resource/3rd_Cinemalaya_Independent_Film_Festival, http://dbpedia.org/resource/4th_Cinemalaya_Independent_Film_Festival, http://dbpedia.org/resource/5th_Cinemalaya_Independent_Film_Festival, http://dbpedia.org/resource/6th_Cinemalaya_Independent_Film_Festival, http://dbpedia.org/resource/7th_Cinemalaya_Independent_Film_Festival, http://dbpedia.org/resource/8th_Cinemalaya_Independent_Film_Festival, http://dbpedia.org/resource/The_Weekend_News_(ABS-CBN), http://dbpedia.org/resource/Oh_My_G!, http://dbpedia.org/resource/Leyte_Provincial_Board, http://dbpedia.org/resource/Southern_Leyte_Provincial_Board, http://dbpedia.org/resource/Negros_Occidental_Private_Schools_Sports_Cultural_Educational_Association, http://dbpedia.org/resource/The_Beacon_–_Arnaiz_Tower, http://dbpedia.org/resource/TicketNet, http://dbpedia.org/resource/Pelaéz_Sports_Complex, http://dbpedia.org/resource/ASEAN_Wildlife_Enforcement_Network, http://dbpedia.org/resource/Almazora_Motors_Corporation, http://dbpedia.org/resource/Cafe_France-CEU_Bakers, http://dbpedia.org/resource/Mimbilisan_Protected_Landscape, http://dbpedia.org/resource/UP_Physics_Association, http://dbpedia.org/resource/Pila_Historic_Town_Center, http://dbpedia.org/resource/San_Carlos_Solar_Energy, http://dbpedia.org/resource/Philippine_Sports_Center, http://dbpedia.org/resource/Ronda_Pilipinas, http://dbpedia.org/resource/Iloilo_Accord, http://dbpedia.org/resource/Premier_Business_Inc., http://dbpedia.org/resource/Numoni, http://dbpedia.org/resource/Golden_Sarok_Rotonda, http://dbpedia.org/resource/Eraño_Manalo, http://dbpedia.org/resource/Nielson_Field, http://dbpedia.org/resource/Ninoy_Aquino_International_Airport, http://dbpedia.org/resource/Puerto_Princesa_Subterranean_River_National_Park, http://dbpedia.org/resource/Codename:_Asero, http://dbpedia.org/resource/Eduardo_V._Manalo, http://dbpedia.org/resource/Philippine_Atheists_and_Agnostics_Society, http://dbpedia.org/resource/Bicutan_Automated_Guideway_Transit_System, http://dbpedia.org/resource/Chowking, http://dbpedia.org/resource/Greenwich_Pizza, http://dbpedia.org/resource/Leyte_Gulf, http://dbpedia.org/resource/San_Pedro_Bay_(Philippines), http://dbpedia.org/resource/BDO_Unibank, http://dbpedia.org/resource/Mactan–Cebu_International_Airport, http://dbpedia.org/resource/Bataan_National_Park, http://dbpedia.org/resource/Battery_Way, http://dbpedia.org/resource/Camotes_Sea, http://dbpedia.org/resource/Cebu_Strait, http://dbpedia.org/resource/Corregidor_Island_Lighthouse, http://dbpedia.org/resource/Roosevelt_Protected_Landscape, http://dbpedia.org/resource/SM_Seaside_City_Cebu, http://dbpedia.org/resource/Tañon_Strait, http://dbpedia.org/resource/Zorro_(Philippines_TV_series), http://dbpedia.org/resource/Touch_Mobile, http://dbpedia.org/resource/Philippine_Postal_Corporation, http://dbpedia.org/resource/Radio_Corporation_of_the_Philippines, http://dbpedia.org/resource/Society_of_Industrial_Engineering, http://dbpedia.org/resource/Got_to_Believe, http://dbpedia.org/resource/Lake_Pinatubo, http://dbpedia.org/resource/Lake_Venado, http://dbpedia.org/resource/Maqueda_Channel, http://dbpedia.org/resource/Panguil_Bay, http://dbpedia.org/resource/Philippine_Rice_Research_Institute, http://dbpedia.org/resource/Cebu_City_Sports_Complex, http://dbpedia.org/resource/Crowne_Plaza_Manila_Galleria_Philippines, http://dbpedia.org/resource/Fisher_Mall, http://dbpedia.org/resource/Jose_R._Gullas_Halad_Museum, http://dbpedia.org/resource/Robinsons_Novaliches, http://dbpedia.org/resource/SM_City_Fairview, http://dbpedia.org/resource/SM_City_Novaliches, http://dbpedia.org/resource/SM_City_Sta._Mesa, http://dbpedia.org/resource/TV_Patrol_Socsksargen, http://dbpedia.org/resource/First_Philippine_Holdings_Corporation, http://dbpedia.org/resource/Fox_International_Channels_Philippines, http://dbpedia.org/resource/Mangaldan_Airfield, http://dbpedia.org/resource/National_Academy_of_Science_and_Technology, http://dbpedia.org/resource/OFW_Diaries, http://dbpedia.org/resource/Cinemalaya_Philippine_Independent_Film_Festival, http://dbpedia.org/resource/1990_Philippines_men's_Asian_Games_basketball_team, http://dbpedia.org/resource/Northern_Cement_basketball_team, http://dbpedia.org/resource/Philippine_Centennial_Team, http://dbpedia.org/resource/Chinese_International_School_Manila, http://dbpedia.org/resource/Elemento, http://dbpedia.org/resource/Forum_Robinsons, http://dbpedia.org/resource/GMA_Network_Center, http://dbpedia.org/resource/MGC-New_Life_Christian_Academy, http://dbpedia.org/resource/Manila_Japanese_School, http://dbpedia.org/resource/Cabra_Island_Lighthouse, http://dbpedia.org/resource/Liwasang_Bonifacio, http://dbpedia.org/resource/Plaza_Moraga, http://dbpedia.org/resource/Philippine_Aerospace_Development_Corporation, http://dbpedia.org/resource/Banco_San_Juan, http://dbpedia.org/resource/Rizal_Commercial_Banking_Corporation, http://dbpedia.org/resource/GMA_Records, http://dbpedia.org/resource/Pilipinas_MX3_Kings, http://dbpedia.org/resource/Totoy_Bato, http://dbpedia.org/resource/Waterfront_Cebu_City_Hotel_&_Casino, http://dbpedia.org/resource/Bandila_(TV_program), http://dbpedia.org/resource/La_Mesa_Dam_and_Reservoir, http://dbpedia.org/resource/News_Patrol, http://dbpedia.org/resource/Philippine_Postal_Savings_Bank, http://dbpedia.org/resource/T.G.I.S., http://dbpedia.org/resource/Stairway_to_Heaven_(Philippines_TV_series), http://dbpedia.org/resource/Apoy_Sa_Dagat, http://dbpedia.org/resource/Green_Rose_(Philippine_TV_series), http://dbpedia.org/resource/Iisa_Pa_Lamang, http://dbpedia.org/resource/Impostor_(TV_series), http://dbpedia.org/resource/Mana_Po, http://dbpedia.org/resource/Midnight_DJ, http://dbpedia.org/resource/Two_Wives_(Philippine_TV_series), http://dbpedia.org/resource/Honda_Bay, http://dbpedia.org/resource/Ang_Lihim_ni_Annasandra, http://dbpedia.org/resource/Babaeng_Hampaslupa, http://dbpedia.org/resource/Calle_Real,_Iloilo, http://dbpedia.org/resource/Galema:_Anak_ni_Zuma, http://dbpedia.org/resource/Grazilda, http://dbpedia.org/resource/Iloilo_Convention_Center, http://dbpedia.org/resource/Luna_Mystika, http://dbpedia.org/resource/Rosalinda_(Philippines_TV_series), http://dbpedia.org/resource/NE_Pacific_Mall, http://dbpedia.org/resource/Robinsons_Cabanatuan, http://dbpedia.org/resource/Federal_Land_Tower, http://dbpedia.org/resource/One_Galleon_Place, http://dbpedia.org/resource/One_Rockwell_West_Tower, http://dbpedia.org/resource/Pacific_Plaza_Towers, http://dbpedia.org/resource/Philamlife_Tower, http://dbpedia.org/resource/Philippine_Diamond_Tower, http://dbpedia.org/resource/Skycity_(Mandaluyong), http://dbpedia.org/resource/Batangas_Blades, http://dbpedia.org/resource/Harbour_Centre_Batang_Pier, http://dbpedia.org/resource/GMA_Kapuso_Foundation, http://dbpedia.org/resource/Edwin_Andrews_Air_Base, http://dbpedia.org/resource/My_Daddy_Dearest, http://dbpedia.org/resource/My_Beloved, http://dbpedia.org/resource/JAKA_Tower, http://dbpedia.org/resource/LKG_Tower, http://dbpedia.org/resource/Amazing_Cooking_Kids, http://dbpedia.org/resource/Seasons_of_Love_(TV_series), http://dbpedia.org/resource/Wish_Come_True, http://dbpedia.org/resource/Iligan_Crusaders, http://dbpedia.org/resource/Solaire_Open, http://dbpedia.org/resource/Robinsons_Place_Pangasinan, http://dbpedia.org/resource/Robinsons_Town_Mall_Malabon, http://dbpedia.org/resource/UST_Student_Organizations_Coordinating_Council, http://dbpedia.org/resource/Joaquin_F._Enriquez_Memorial_Sports_Complex, http://dbpedia.org/resource/South_Mansfield_College, http://dbpedia.org/resource/Bacolod_Arts_&_Youth_Sports_Center, http://dbpedia.org/resource/Negros_Occidental_Multi-Purpose_Activity_Center, http://dbpedia.org/resource/Beverly_Hills_6750, http://dbpedia.org/resource/Hilutangan_Channel, http://dbpedia.org/resource/Blessed_Trinity_School, http://dbpedia.org/resource/Bulwagan_ng_Panginoon, http://dbpedia.org/resource/Magellan_Bay, http://dbpedia.org/resource/Miss_Possibilities, http://dbpedia.org/resource/Apollo_Quiboloy, http://dbpedia.org/resource/National_Bureau_of_Investigation_(Philippines), http://dbpedia.org/resource/Association_of_Local_Colleges_and_Universities, http://dbpedia.org/resource/Bonifacio_Global_City, http://dbpedia.org/resource/Clark_Air_Base, http://dbpedia.org/resource/Philippine_Institute_of_Civil_Engineers, http://dbpedia.org/resource/Balintang_Channel, http://dbpedia.org/resource/Metropolitan_Bank_and_Trust_Company, http://dbpedia.org/resource/Samar_Sea, http://dbpedia.org/resource/San_Bernardino_Strait, http://dbpedia.org/resource/Expeditie_Robinson, http://dbpedia.org/resource/National_Disaster_Risk_Reduction_and_Management_Council, http://dbpedia.org/resource/Manila_Metrostars, http://dbpedia.org/resource/Pasig_River_Rehabilitation_Commission, http://dbpedia.org/resource/Philippines–Thailand_Friendship_Circle, http://dbpedia.org/resource/Remedios_Circle, http://dbpedia.org/resource/Rizal_Memorial_Baseball_Stadium, http://dbpedia.org/resource/SM_City_Manila, http://dbpedia.org/resource/SM_City_San_Lazaro, http://dbpedia.org/resource/SM_Quiapo, http://dbpedia.org/resource/Aliw_Broadcasting_Corporation, http://dbpedia.org/resource/Bombo_Radyo_Philippines, http://dbpedia.org/resource/Catholic_Media_Network, http://dbpedia.org/resource/Manila_Broadcasting_Company, http://dbpedia.org/resource/Mareco_Broadcasting_Network, http://dbpedia.org/resource/Nation_Broadcasting_Corporation, http://dbpedia.org/resource/Canigao_Channel, http://dbpedia.org/resource/Robinsons_Place_Santiago, http://dbpedia.org/resource/Balay_Negrense, http://dbpedia.org/resource/Butuan_Bay, http://dbpedia.org/resource/MassKara_Festival, http://dbpedia.org/resource/Negros_Slashers, http://dbpedia.org/resource/Paglaum_Sports_Complex, http://dbpedia.org/resource/Panaad_Park_and_Stadium, http://dbpedia.org/resource/Panaad_sa_Negros_Festival, http://dbpedia.org/resource/CDO_(company), http://dbpedia.org/resource/Manila_International_Airport_Authority, http://dbpedia.org/resource/Ayala_Tower_One, http://dbpedia.org/resource/MFI_Foundation_Inc., http://dbpedia.org/resource/Marikina_River_Park, http://dbpedia.org/resource/Marikina_Sports_Complex, http://dbpedia.org/resource/SMX_Convention_Center, http://dbpedia.org/resource/SM_City_Marikina, http://dbpedia.org/resource/Shangri-La_Plaza, http://dbpedia.org/resource/Urban_Zone, http://dbpedia.org/resource/World_Trade_Center_Metro_Manila, http://dbpedia.org/resource/Del_Monte_Motor_Works,_Inc., http://dbpedia.org/resource/Mutya_ng_Pilipinas, http://dbpedia.org/resource/Filipino_Freethinkers, http://dbpedia.org/resource/Joy-Nostalg_Center, http://dbpedia.org/resource/One_Corporate_Centre, http://dbpedia.org/resource/One_San_Miguel_Avenue, http://dbpedia.org/resource/Orient_Square, http://dbpedia.org/resource/Robinsons_Equitable_Tower, http://dbpedia.org/resource/St._Francis_Square_Mall, http://dbpedia.org/resource/The_St._Francis_Shangri-La_Place, http://dbpedia.org/resource/Ultrasonic_Broadcasting_System, http://dbpedia.org/resource/UnionBank_Plaza, http://dbpedia.org/resource/SM_City_Calamba, http://dbpedia.org/resource/Abreeza, http://dbpedia.org/resource/Chinabank, http://dbpedia.org/resource/Davao_del_Norte_Sports_Complex, http://dbpedia.org/resource/Gaisano_Mall_of_Davao, http://dbpedia.org/resource/One_Network_Bank, http://dbpedia.org/resource/Phoenix_Petroleum, http://dbpedia.org/resource/SM_City_Davao, http://dbpedia.org/resource/SM_Lanang_Premier, http://dbpedia.org/resource/San_Diego_de_Alcala_Church, http://dbpedia.org/resource/Cañacao_Bay, http://dbpedia.org/resource/Honesto, http://dbpedia.org/resource/SM_City_Olongapo, http://dbpedia.org/resource/Marquee_Mall, http://dbpedia.org/resource/SM_City_Clark, http://dbpedia.org/resource/Point_of_Entry_(TV_series), http://dbpedia.org/resource/Iloilo_River_Esplanade, http://dbpedia.org/resource/Congo_Grille, http://dbpedia.org/resource/Dencio's, http://dbpedia.org/resource/Equitable_PCI_Bank, http://dbpedia.org/resource/Ortigas_Interchange, http://dbpedia.org/resource/2016_Manila_Challenger, http://dbpedia.org/resource/Laguna_Lakers, http://dbpedia.org/resource/SCQ_Reload, http://dbpedia.org/resource/YCO_Painters, http://dbpedia.org/resource/Captain_Barbell_(2011_TV_series), http://dbpedia.org/resource/Darna_(2009_TV_series), http://dbpedia.org/resource/Amro_River_Protected_Landscape, http://dbpedia.org/resource/Dinadiawan_River_Protected_Landscape, http://dbpedia.org/resource/Mount_Samat_National_Shrine, http://dbpedia.org/resource/San_Marcelino_Airfield, http://dbpedia.org/resource/Simbahan-Talagas_Protected_Landscape, http://dbpedia.org/resource/Talaytay_Protected_Landscape, http://dbpedia.org/resource/Little_Champ, http://dbpedia.org/resource/Magkano_Ang_Iyong_Dangal%3F, http://dbpedia.org/resource/Super_D_(TV_series), http://dbpedia.org/resource/Trip_na_Trip, http://dbpedia.org/resource/Alamat_(TV_series), http://dbpedia.org/resource/Dear_Uge, http://dbpedia.org/resource/M-Zet_Productions, http://dbpedia.org/resource/Naku,_Boss_Ko!, http://dbpedia.org/resource/Poor_Señorita, http://dbpedia.org/resource/List_of_ONE_Championship_events, http://dbpedia.org/resource/Born_For_You, http://dbpedia.org/resource/Maria_Mercedes_(Philippine_telenovela), http://dbpedia.org/resource/Cielo_de_Angelina, http://dbpedia.org/resource/Dugong_Buhay, http://dbpedia.org/resource/Kristine_(TV_series), http://dbpedia.org/resource/Reel_Love, http://dbpedia.org/resource/My_Cheating_Heart, http://dbpedia.org/resource/Dear_Friend:_Bakasyonistas, http://dbpedia.org/resource/Liter_of_Light, http://dbpedia.org/resource/First_United_Broadcasting_Corporation, http://dbpedia.org/resource/InstaDad, http://dbpedia.org/resource/PEC_Broadcasting_Corporation, http://dbpedia.org/resource/The_Substitute_Bride, http://dbpedia.org/resource/1st_Cinemalaya_Independent_Film_Festival, http://dbpedia.org/resource/Blackwater_Sports, http://dbpedia.org/resource/Bais_City_National_Science_High_School, http://dbpedia.org/resource/Hot_Mama_(TV_series), http://dbpedia.org/resource/Crown_Regency_Hotel_and_Towers, http://dbpedia.org/resource/Exportbank_Plaza, http://dbpedia.org/resource/One_Roxas_Triangle, http://dbpedia.org/resource/Pearl_of_the_Orient_Tower, http://dbpedia.org/resource/The_Infinity_Fort_Bonifacio, http://dbpedia.org/resource/The_Residences_at_Greenbelt_–_Laguna_Tower, http://dbpedia.org/resource/The_Residences_at_Greenbelt_–_Manila_Tower, http://dbpedia.org/resource/The_Shang_Grand_Tower, http://dbpedia.org/resource/The_World_Centre, http://dbpedia.org/resource/AGHAM, http://dbpedia.org/resource/Filoil_Flying_V_Center, http://dbpedia.org/resource/Al-Amanah_Islamic_Bank, http://dbpedia.org/resource/Bacolod_Chicken_Inasal, http://dbpedia.org/resource/Mula_sa_Puso, http://dbpedia.org/resource/Why_News, http://dbpedia.org/resource/Aurora_National_Science_High_School, http://dbpedia.org/resource/Kabasalan_Science_and_Technology_High_School, http://dbpedia.org/resource/Misamis_Occidental_Science_and_Technology_High_School, http://dbpedia.org/resource/Puerto_Princesa_City_National_Science_High_School, http://dbpedia.org/resource/Tagaytay_City_National_Science_High_School, http://dbpedia.org/resource/Talisay_City_Science_High_School, http://dbpedia.org/resource/Festival_Alabang, http://dbpedia.org/resource/Century_Pacific_Food_Inc., http://dbpedia.org/resource/San_Miguel_Pure_Foods_Company_Inc., http://dbpedia.org/resource/Ginebra_San_Miguel, http://dbpedia.org/resource/Manila_Bay, http://dbpedia.org/resource/Subic_Bay, http://dbpedia.org/resource/Felix_Manalo, http://dbpedia.org/resource/The_Peninsula_Manila, http://dbpedia.org/resource/Cebu_International_School, http://dbpedia.org/resource/E.S.P._(TV_series), http://dbpedia.org/resource/International_Service_for_the_Acquisition_of_Agri-biotech_Applications, http://dbpedia.org/resource/Philippine_Arena, http://dbpedia.org/resource/Ayala_Corporation, http://dbpedia.org/resource/Ateneo_Art_Gallery, http://dbpedia.org/resource/Ayala_Land, http://dbpedia.org/resource/Ayala_Museum, http://dbpedia.org/resource/Camp_Atienza, http://dbpedia.org/resource/Cuneta_Astrodome, http://dbpedia.org/resource/Development_Bank_of_the_Philippines, http://dbpedia.org/resource/Farmers_Plaza, http://dbpedia.org/resource/Fort_Bonifacio, http://dbpedia.org/resource/Greenbelt_(Ayala_Center), http://dbpedia.org/resource/Lopez_Group_of_Companies, http://dbpedia.org/resource/Manila_Water, http://dbpedia.org/resource/Maynilad_Water_Services, http://dbpedia.org/resource/Meralco, http://dbpedia.org/resource/Metropolitan_Museum_of_Manila, http://dbpedia.org/resource/Museo_Pambata, http://dbpedia.org/resource/Paco_Park, http://dbpedia.org/resource/Philippine_International_Convention_Center, http://dbpedia.org/resource/Quezon_Memorial_Circle, http://dbpedia.org/resource/Rizal_Memorial_Sports_Complex, http://dbpedia.org/resource/SM_Aura_Premier, http://dbpedia.org/resource/SM_City_North_EDSA, http://dbpedia.org/resource/SM_Megamall, http://dbpedia.org/resource/SM_Prime_Holdings, http://dbpedia.org/resource/SM_Southmall, http://dbpedia.org/resource/San_Miguel_Corporation, http://dbpedia.org/resource/State_Colleges_and_Universities_Athletic_Association, http://dbpedia.org/resource/The_Mind_Museum, http://dbpedia.org/resource/Visayan_Sea, http://dbpedia.org/resource/Baguio_Athletic_Bowl, http://dbpedia.org/resource/Basa_Air_Base, http://dbpedia.org/resource/Bataan_Airfield, http://dbpedia.org/resource/Mines_View_Park, http://dbpedia.org/resource/Robinsons_Place_Las_Piñas, http://dbpedia.org/resource/SM_Center_Las_Piñas, http://dbpedia.org/resource/SM_City_Baguio, http://dbpedia.org/resource/Manila_Commodity_Exchange, http://dbpedia.org/resource/Rain_or_Shine_Elasto_Painters, http://dbpedia.org/resource/Brigada_Mass_Media_Corporation, http://dbpedia.org/resource/Malampaya_Sound, http://dbpedia.org/resource/REECS, http://dbpedia.org/resource/Plaza_Dilao, http://dbpedia.org/resource/Plaza_Lacson, http://dbpedia.org/resource/Plaza_Moriones, http://dbpedia.org/resource/Plaza_Rajah_Sulayman, http://dbpedia.org/resource/Plaza_San_Lorenzo_Ruiz, http://dbpedia.org/resource/Plaza_de_Armas_(Manila), http://dbpedia.org/resource/Plaza_de_Roma, http://dbpedia.org/resource/Malaybalay_Airport, http://dbpedia.org/resource/Timpoong_and_Hibok-Hibok_Natural_Monument, http://dbpedia.org/resource/Blue_Eagle_Gym, http://dbpedia.org/resource/Cherry_Mobile, http://dbpedia.org/resource/Oishi_(company), http://dbpedia.org/resource/Philippine_Savings_Bank, http://dbpedia.org/resource/Universal_Robina, http://dbpedia.org/resource/On_the_Wings_of_Love_(TV_series), http://dbpedia.org/resource/Mandaue_Presidencia, http://dbpedia.org/resource/Ripley's_Believe_It_or_Not!_(2008_TV_series), http://dbpedia.org/resource/Viva_Records_(Philippines), http://dbpedia.org/resource/SM_City_Dasmariñas, http://dbpedia.org/resource/Daldalita, http://dbpedia.org/resource/ADD_Convention_Center, http://dbpedia.org/resource/Irrigated_Rice_Research_Consortium, http://dbpedia.org/resource/Philippine_Commission_on_Women, http://dbpedia.org/resource/Pulso:_Aksyon_Balita, http://dbpedia.org/resource/SM_City_Lucena, http://dbpedia.org/resource/The_World_Tonight_(Philippine_newscast), http://dbpedia.org/resource/Marinara_(TV_series), http://dbpedia.org/resource/Kung_Ako'y_Iiwan_Mo, http://dbpedia.org/resource/Kung_Mawawala_Ka, http://dbpedia.org/resource/May_Isang_Pangarap, http://dbpedia.org/resource/Mirabella_(TV_series), http://dbpedia.org/resource/SM_Center_Pasig, http://dbpedia.org/resource/SM_City_Batangas, http://dbpedia.org/resource/SM_City_Consolacion, http://dbpedia.org/resource/SM_City_Lipa, http://dbpedia.org/resource/SM_City_Molino, http://dbpedia.org/resource/SM_City_Naga, http://dbpedia.org/resource/SM_City_Rosario, http://dbpedia.org/resource/SM_City_Tarlac, http://dbpedia.org/resource/Hawak_Kamay_(TV_series), http://dbpedia.org/resource/Niño_(TV_series), http://dbpedia.org/resource/Pavilion_Mall, http://dbpedia.org/resource/Balinsasayao_Twin_Lakes_Natural_Park, http://dbpedia.org/resource/Accrediting_Agency_of_Chartered_Colleges_and_Universities_in_the_Philippines, http://dbpedia.org/resource/Hilongos_Airport, http://dbpedia.org/resource/Rajah_Buayan_Air_Base, http://dbpedia.org/resource/Legacy_(2012_TV_series), http://dbpedia.org/resource/Bagacay_Point_Lighthouse, http://dbpedia.org/resource/Bureau_of_Fisheries_and_Aquatic_Resources, http://dbpedia.org/resource/CPU_Research_Center, http://dbpedia.org/resource/Coffee_Prince_(2012_TV_series), http://dbpedia.org/resource/Hanggang_Makita_Kang_Muli, http://dbpedia.org/resource/Hiram_na_Alaala, http://dbpedia.org/resource/Juan_Tamad_(TV_series), http://dbpedia.org/resource/Karelasyon, http://dbpedia.org/resource/Let_the_Love_Begin_(TV_series), http://dbpedia.org/resource/Little_Nanay, http://dbpedia.org/resource/Mga_Basang_Sisiw, http://dbpedia.org/resource/One_True_Love_(TV_series), http://dbpedia.org/resource/That's_My_Amboy, http://dbpedia.org/resource/The_Rich_Man's_Daughter, http://dbpedia.org/resource/Wish_I_May_(TV_series), http://dbpedia.org/resource/Spring_Films, http://dbpedia.org/resource/Ang_Panday_(2016_TV_series), http://dbpedia.org/resource/Unforgettable_(2013_TV_series), http://dbpedia.org/resource/Sa_Puso_ni_Dok, http://dbpedia.org/resource/The_Last_Romance_(TV_series), http://dbpedia.org/resource/Undercover_(2013_TV_series), http://dbpedia.org/resource/Dading_(TV_series), http://dbpedia.org/resource/1G_(TV_series), http://dbpedia.org/resource/Banana_Sundae, http://dbpedia.org/resource/I_Do_(Philippine_reality_show), http://dbpedia.org/resource/The_Story_of_Us_(TV_series), http://dbpedia.org/resource/Tubig_at_Langis, http://dbpedia.org/resource/We_Will_Survive_(TV_series), http://dbpedia.org/resource/Red_Cross_Youth_(Philippines), http://dbpedia.org/resource/Hiyas_(TV_series), http://dbpedia.org/resource/One_Rockwell_East_Tower, http://dbpedia.org/resource/Walang_Hanggan_(2012_TV_series), http://dbpedia.org/resource/Apo_Reef_Light, http://dbpedia.org/resource/Washington_Sycip_Park, http://dbpedia.org/resource/Philippine_International_Motor_Show, http://dbpedia.org/resource/Tour_de_Filipinas, http://dbpedia.org/resource/Philippine_Advertising_Congress, http://dbpedia.org/resource/Rufino_Pacific_Tower, http://dbpedia.org/resource/The_Residences_at_Greenbelt_–_San_Lorenzo_Tower, http://dbpedia.org/resource/The_Beacon_–_Amorsolo_Tower, http://dbpedia.org/resource/The_Beacon_–_Roces_Tower, http://dbpedia.org/resource/San_Lazaro_Leisure_Park, http://dbpedia.org/resource/Seaoil_Philippines, http://dbpedia.org/resource/Bulusan_Volcano_Natural_Park, http://dbpedia.org/resource/National_Teams_Training_Center, http://dbpedia.org/resource/Advance_Paper_Corporation, http://dbpedia.org/resource/Springboard_Foundation, http://dbpedia.org/resource/Hyatt_Terraces_Hotel, http://dbpedia.org/resource/Miss_Earth, http://dbpedia.org/resource/SkyCable, http://dbpedia.org/resource/PROGUN, http://dbpedia.org/resource/PhilSports_Arena, http://dbpedia.org/resource/Fairplay_For_All_Foundation, http://dbpedia.org/resource/St._La_Salle_Hall, http://dbpedia.org/resource/Department_of_Trade_and_Industry_(Philippines), http://dbpedia.org/resource/Kokak, http://dbpedia.org/resource/Manila_Hotel, http://dbpedia.org/resource/Alabang_Town_Center, http://dbpedia.org/resource/Insular_Life, http://dbpedia.org/resource/New_Bilibid_Prison, http://dbpedia.org/resource/Philippine_Amusement_and_Gaming_Corporation, http://dbpedia.org/resource/Puregold, http://dbpedia.org/resource/Robinsons_Supermarket, http://dbpedia.org/resource/SM_Center_Muntinlupa, http://dbpedia.org/resource/SM_City_Sucat, http://dbpedia.org/resource/SM_Hypermarket, http://dbpedia.org/resource/Good_Morning_Kuya, http://dbpedia.org/resource/Magandang_Umaga,_Pilipinas, http://dbpedia.org/resource/Umagang_Kay_Ganda, http://dbpedia.org/resource/Bacoor_Bay, http://dbpedia.org/resource/Healing_Hearts, http://dbpedia.org/resource/Light_Rail_Transit_Authority, http://dbpedia.org/resource/Mounts_Palay-Palay–Mataas-na-Gulod_Protected_Landscape, http://dbpedia.org/resource/People's_Park_in_the_Sky, http://dbpedia.org/resource/Batangas_Bay, http://dbpedia.org/resource/Full_House_(2009_TV_series), http://dbpedia.org/resource/Wowowin, http://dbpedia.org/resource/National_Book_Store, http://dbpedia.org/resource/Philippine_Science_Consortium, http://dbpedia.org/resource/Cebu_International_Convention_Center, http://dbpedia.org/resource/Tandag_National_Science_High_School, http://dbpedia.org/resource/Pueblo_de_Panay, http://dbpedia.org/resource/Sacred_Heart_of_Jesus_(Roxas,_Capiz), http://dbpedia.org/resource/Nacho_King!, http://dbpedia.org/resource/Allied_Banking_Corporation, http://dbpedia.org/resource/Gothong_Lines, http://dbpedia.org/resource/Philtrust_Bank, http://dbpedia.org/resource/Robinsons_Magnolia, http://dbpedia.org/resource/All_Youth_Channels, http://dbpedia.org/resource/Century_Properties, http://dbpedia.org/resource/DMCI_Homes, http://dbpedia.org/resource/Ever_Gotesco_Malls, http://dbpedia.org/resource/Jinisys_Software, http://dbpedia.org/resource/Lae_Garden_and_Landscapes, http://dbpedia.org/resource/Petron_Megaplaza, http://dbpedia.org/resource/Philippine_Business_Bank, http://dbpedia.org/resource/Philippine_National_Construction_Corporation, http://dbpedia.org/resource/Property_Company_of_Friends, http://dbpedia.org/resource/Republic_Biscuit_Corporation, http://dbpedia.org/resource/Starmobile, http://dbpedia.org/resource/Tokyo_Tokyo, http://dbpedia.org/resource/SRO_Cinemaserye, http://dbpedia.org/resource/The_Last_Prince, http://dbpedia.org/resource/Quezon_Convention_Center, http://dbpedia.org/resource/Tagum_Cooperative, http://dbpedia.org/resource/Calbayog_Pan-as–Hayiban_Protected_Landscape, http://dbpedia.org/resource/Capul_Island_Lighthouse, http://dbpedia.org/resource/Dumanquilas_Bay, http://dbpedia.org/resource/Land_Management_Bureau_(Philippines), http://dbpedia.org/resource/Mainit_Hot_Springs_Protected_Landscape, http://dbpedia.org/resource/Aliwagwag_Protected_Landscape, http://dbpedia.org/resource/Baganga_Protected_Landscape, http://dbpedia.org/resource/Pujada_Bay, http://dbpedia.org/resource/Urban_Bank, http://dbpedia.org/resource/Home_Sweet_Home_(2013_TV_series), http://dbpedia.org/resource/Balita_Alas_Singko_ng_Umaga_(news_show), http://dbpedia.org/resource/Lovers_in_Paris_(Philippine_TV_series), http://dbpedia.org/resource/Kalbario-Patapat_Natural_Park, http://dbpedia.org/resource/Maniguin_Island_Lighthouse, http://dbpedia.org/resource/One_McKinley_Place, http://dbpedia.org/resource/All_of_Me_(TV_series), http://dbpedia.org/resource/Kahit_Konting_Pagtingin, http://dbpedia.org/resource/You're_My_Home_(TV_series), http://dbpedia.org/resource/10th_Cinemalaya_Independent_Film_Festival, http://dbpedia.org/resource/Kirara,_Ano_Ang_Kulay_ng_Pag-ibig%3F, http://dbpedia.org/resource/Cossack_Blue_Spirits, http://dbpedia.org/resource/Stag_Pale_Pilseners, http://dbpedia.org/resource/Tanduay_Light_Rhum_Masters, http://dbpedia.org/resource/Ysmael_Steel_Admirals, http://dbpedia.org/resource/Teen_Gen, http://dbpedia.org/resource/AddMix_Transformers, http://dbpedia.org/resource/Big_Chill_Super_Chargers, http://dbpedia.org/resource/Cebuana_Lhuillier_Gems, http://dbpedia.org/resource/Crispa_400, http://dbpedia.org/resource/Excel_Roof_25ers, http://dbpedia.org/resource/Fern-C, http://dbpedia.org/resource/Mail_and_More_Comets, http://dbpedia.org/resource/Pharex_B-Complex, http://dbpedia.org/resource/RFM-Swift's, http://dbpedia.org/resource/A_Beautiful_Affair, http://dbpedia.org/resource/Lucky_Chinatown, http://dbpedia.org/resource/Robinsons_Metro_East, http://dbpedia.org/resource/Salcedo_Park_Twin_Towers, http://dbpedia.org/resource/ABS-CBN_News_Advisory, http://dbpedia.org/resource/Alyna_(TV_series), http://dbpedia.org/resource/Gimik_2010, http://dbpedia.org/resource/Love_Me_Again_(TV_series), http://dbpedia.org/resource/Alonte_Sports_Arena, http://dbpedia.org/resource/Bacolod_City_Arena, http://dbpedia.org/resource/Hoops_Dome, http://dbpedia.org/resource/Seaside_City_Arena, http://dbpedia.org/resource/Solaire_Arena, http://dbpedia.org/resource/Pasonanca_Natural_Park, http://dbpedia.org/resource/RADION_International, http://dbpedia.org/resource/PAGCOR_Tower, http://dbpedia.org/resource/Circumferential_Road_5–Kalayaan_Avenue_Interchange, http://dbpedia.org/resource/Circumferential_Road_5–Ortigas_Avenue_Interchange, http://dbpedia.org/resource/German_European_School_Manila, http://dbpedia.org/resource/National_Service_Reserve_Corps, http://dbpedia.org/resource/Mounts_Iglit–Baco_National_Park, http://dbpedia.org/resource/Quest_Broadcasting, http://dbpedia.org/resource/Hermosa_Festival, http://dbpedia.org/resource/Bagatao_Island_Lighthouse, http://dbpedia.org/resource/Capones_Island_Lighthouse, http://dbpedia.org/resource/Photoville_International, http://dbpedia.org/resource/Mama_Sita's_Holding_Company, http://dbpedia.org/resource/VMoney_Inc, http://dbpedia.org/resource/Correctional_Institution_for_Women_(Mandaluyong), http://dbpedia.org/resource/Sa_Sinungaling_Mong_Puso, http://dbpedia.org/resource/Mercury_Drug, http://dbpedia.org/resource/Albay_Gulf, http://dbpedia.org/resource/Armscor_(Philippines), http://dbpedia.org/resource/Banaue_Rice_Terraces, http://dbpedia.org/resource/Caramoan_Peninsula, http://dbpedia.org/resource/Clark_International_Airport, http://dbpedia.org/resource/Department_of_Science_and_Technology_(Philippines), http://dbpedia.org/resource/Goldilocks_Bakeshop, http://dbpedia.org/resource/Government_Arsenal, http://dbpedia.org/resource/Iligan_Bay, http://dbpedia.org/resource/It's_Showtime_(variety_show), http://dbpedia.org/resource/Jollibee, http://dbpedia.org/resource/Lagonoy_Gulf, http://dbpedia.org/resource/Lamon_Bay, http://dbpedia.org/resource/Mabuhay_Satellite_Corporation, http://dbpedia.org/resource/Manila_Light_Rail_Transit_System, http://dbpedia.org/resource/Manila_Light_Rail_Transit_System_Line_2, http://dbpedia.org/resource/Manila_Metro_Rail_Transit_System, http://dbpedia.org/resource/Mindoro_Strait, http://dbpedia.org/resource/Moro_Gulf, http://dbpedia.org/resource/National_Commission_on_Muslim_Filipinos, http://dbpedia.org/resource/Ombudsman_of_the_Philippines, http://dbpedia.org/resource/Philippine_Long_Distance_Telephone_Company, http://dbpedia.org/resource/Philippine_National_Oil_Company, http://dbpedia.org/resource/Philippine_Statistics_Authority, http://dbpedia.org/resource/Ragay_Gulf, http://dbpedia.org/resource/Rizal_Park, http://dbpedia.org/resource/SM_Mall_of_Asia, http://dbpedia.org/resource/San_Juanico_Strait, http://dbpedia.org/resource/San_Miguel_Bay, http://dbpedia.org/resource/Sibuyan_Sea, http://dbpedia.org/resource/Sorsogon_Bay, http://dbpedia.org/resource/Sulpicio_Lines, http://dbpedia.org/resource/Tablas_Strait, http://dbpedia.org/resource/Tayabas_Bay, http://dbpedia.org/resource/Technical_Education_and_Skills_Development_Authority, http://dbpedia.org/resource/University_of_the_Philippines_Diliman_Automated_Guideway_Transit_System, http://dbpedia.org/resource/Radio_Mindanao_Network, http://dbpedia.org/resource/PBCom_Tower, http://dbpedia.org/resource/Zuellig_Building, http://dbpedia.org/resource/Ali_Mall, http://dbpedia.org/resource/National_Historical_Commission_of_the_Philippines, http://dbpedia.org/resource/Toon_City, http://dbpedia.org/resource/Cagayan_Rising_Suns, http://dbpedia.org/resource/Biak-na-Bato_National_Park, http://dbpedia.org/resource/Micronet_co.,_Ltd., http://dbpedia.org/resource/Ang_Dating_Daan, http://dbpedia.org/resource/Ayala_Triangle_Gardens, http://dbpedia.org/resource/G.T._International_Tower, http://dbpedia.org/resource/Magallanes_Interchange, http://dbpedia.org/resource/Makati_Coliseum, http://dbpedia.org/resource/Philippine_Bank_of_Communications, http://dbpedia.org/resource/Power_Plant_Mall, http://dbpedia.org/resource/Rustan's, http://dbpedia.org/resource/Dolce_Amore, http://dbpedia.org/resource/Lake_Bunot, http://dbpedia.org/resource/Lake_Calibato, http://dbpedia.org/resource/Lake_Muhikap, http://dbpedia.org/resource/Lake_Palakpakin, http://dbpedia.org/resource/Lake_Sampaloc, http://dbpedia.org/resource/Lakes_Pandin_and_Yambo, http://dbpedia.org/resource/Philippine_Sports_Stadium, http://dbpedia.org/resource/Sunday_PinaSaya, http://dbpedia.org/resource/Lake_Tikub, http://dbpedia.org/resource/Mangarin_Bay, http://dbpedia.org/resource/The_Amazing_Race_Philippines, http://dbpedia.org/resource/Click_(Philippine_TV_series), http://dbpedia.org/resource/Pinaglabanan_Shrine, http://dbpedia.org/resource/Anino_Games, http://dbpedia.org/resource/Ina,_Kapatid,_Anak, http://dbpedia.org/resource/Juan_dela_Cruz_(TV_series), http://dbpedia.org/resource/Kantar_Media_Philippines, http://dbpedia.org/resource/Princess_and_I, http://dbpedia.org/resource/Tayong_Dalawa, http://dbpedia.org/resource/Notre_Dame_Broadcasting_Corporation, http://dbpedia.org/resource/Nichols_Field_(Philippines), http://dbpedia.org/resource/ABS-CBN_Broadcasting_Center, http://dbpedia.org/resource/Be_Careful_With_My_Heart, http://dbpedia.org/resource/Chicken_Pork_Adobo, http://dbpedia.org/resource/Millennium_Transmitter, http://dbpedia.org/resource/RoadRunner_(division), http://dbpedia.org/resource/Sine_Screen, http://dbpedia.org/resource/Skylight_Films, http://dbpedia.org/resource/The_Sharon_Cuneta_Show, http://dbpedia.org/resource/LBC_Express, http://dbpedia.org/resource/Pilar_Manalo_Danao, http://dbpedia.org/resource/Global_Cable, http://dbpedia.org/resource/Forever_in_My_Heart, http://dbpedia.org/resource/Buenavista_Protected_Landscape, http://dbpedia.org/resource/Jillian:_Namamasko_Po, http://dbpedia.org/resource/Sana_Ngayong_Pasko, http://dbpedia.org/resource/Splash_Corporation, http://dbpedia.org/resource/Pacific_Broadcasting_Systems, http://dbpedia.org/resource/Madam_Chairman, http://dbpedia.org/resource/Obra_(TV_series), http://dbpedia.org/resource/Pangako_Sa_'Yo_(2015_TV_series), http://dbpedia.org/resource/Pepito_Manaloto, http://dbpedia.org/resource/Tunay_Na_Buhay, http://dbpedia.org/resource/Narito_Ang_Puso_Ko, http://dbpedia.org/resource/Sarap_Diva, http://dbpedia.org/resource/ABS-CBN_Headlines, http://dbpedia.org/resource/Adarna_(TV_series), http://dbpedia.org/resource/Ako_si_Kim_Samsoon, http://dbpedia.org/resource/Ang_Dalawang_Mrs._Real, http://dbpedia.org/resource/Annaliza, http://dbpedia.org/resource/Artista_Academy, http://dbpedia.org/resource/BFGF_(TV_series), http://dbpedia.org/resource/Carmela_(TV_series), http://dbpedia.org/resource/Dwarfina, http://dbpedia.org/resource/Hanggang_Kailan, http://dbpedia.org/resource/I_Luv_NY, http://dbpedia.org/resource/Ikaw_Lang_Ang_Mamahalin, http://dbpedia.org/resource/Ikaw_Lang_ang_Mamahalin_(2011_TV_series), http://dbpedia.org/resource/Ikaw_Na_Sana, http://dbpedia.org/resource/Ikaw_Sana, http://dbpedia.org/resource/Ilumina_(TV_series), http://dbpedia.org/resource/Ito_Ang_Balita, http://dbpedia.org/resource/Kailan_Ba_Tama_ang_Mali%3F, http://dbpedia.org/resource/Kambal_Sirena, http://dbpedia.org/resource/Love_Bug_(Philippine_TV_series), http://dbpedia.org/resource/Majika, http://dbpedia.org/resource/More_Than_Words_(TV_series), http://dbpedia.org/resource/Once_Upon_a_Kiss, http://dbpedia.org/resource/Pahiram_ng_Sandali, http://dbpedia.org/resource/Panday_Kids, http://dbpedia.org/resource/Paraiso_Ko'y_Ikaw, http://dbpedia.org/resource/Pari_'Koy, http://dbpedia.org/resource/Rated_K, http://dbpedia.org/resource/Sana_Ay_Ikaw_Na_Nga_(2001_TV_series), http://dbpedia.org/resource/Sana_Ay_Ikaw_Na_Nga_(2012_TV_series), http://dbpedia.org/resource/Temptation_of_Wife_(2012_TV_series), http://dbpedia.org/resource/The_Borrowed_Wife, http://dbpedia.org/resource/The_Half_Sisters, http://dbpedia.org/resource/The_Legal_Wife, http://dbpedia.org/resource/International_Institute_of_Rural_Reconstruction, http://dbpedia.org/resource/KCC_Malls, http://dbpedia.org/resource/Tweets_For_My_Sweet, http://dbpedia.org/resource/Dear_Friend:_Karibal, http://dbpedia.org/resource/Futbolilits, http://dbpedia.org/resource/Halik_Sa_Apoy, http://dbpedia.org/resource/Hindi_Ka_Na_Mag-iisa, http://dbpedia.org/resource/Makapiling_Kang_Muli, http://dbpedia.org/resource/My_Faithful_Husband, http://dbpedia.org/resource/Spooky_Nights, http://dbpedia.org/resource/For_Love_or_Money_(2013_TV_series), http://dbpedia.org/resource/Forever_(Philippine_TV_series), http://dbpedia.org/resource/My_Binondo_Girl, http://dbpedia.org/resource/National_Labor_Relations_Commission_(Philippines), http://dbpedia.org/resource/Overseas_Workers_Welfare_Administration, http://dbpedia.org/resource/Dream_FM_Network, http://dbpedia.org/resource/Bridges_of_Love, http://dbpedia.org/resource/Dapat_Ka_Bang_Mahalin%3F, http://dbpedia.org/resource/Di_Ba't_Ikaw, http://dbpedia.org/resource/Growing_Up_(1997_Philippine_TV_series), http://dbpedia.org/resource/Gumapang_Ka_Sa_Lusak_(TV_series), http://dbpedia.org/resource/Hongkong_Flight_143, http://dbpedia.org/resource/Ina,_Kasusuklaman_Ba_Kita%3F, http://dbpedia.org/resource/Ngayon_at_Kailanman_(TV_series), http://dbpedia.org/resource/Sana_Bukas_pa_ang_Kahapon, http://dbpedia.org/resource/Sinner_or_Saint_(TV_series), http://dbpedia.org/resource/To_The_Top_(TV_series), http://dbpedia.org/resource/Nueva_Ecija_Patriots, http://dbpedia.org/resource/Olongapo_Volunteers, http://dbpedia.org/resource/San_Juan_Knights, http://dbpedia.org/resource/Give_Love_on_Christmas, http://dbpedia.org/resource/Kidlat, http://dbpedia.org/resource/Subkulture, http://dbpedia.org/resource/I-Shine_Talent_Camp, http://dbpedia.org/resource/Pare_&_Pare, http://dbpedia.org/resource/Agricultural_Training_Institute_(Philippines), http://dbpedia.org/resource/Bureau_of_Agricultural_Research_(Philippines), http://dbpedia.org/resource/Bureau_of_Agricultural_and_Fisheries_Product_Standards_(Philippines), http://dbpedia.org/resource/Bureau_of_Soils_and_Water_Management_(Philippines), http://dbpedia.org/resource/Cotton_Development_Administration_(Philippines), http://dbpedia.org/resource/Integrated_Micro-Electronics,_Inc., http://dbpedia.org/resource/Marimar_(2015_TV_series), http://dbpedia.org/resource/Bagets:_Just_Got_Lucky, http://dbpedia.org/resource/My_Fair_Lady_(Philippine_TV_series), http://dbpedia.org/resource/Positive_(TV_series), http://dbpedia.org/resource/Mactan-Benito_Ebuen_Air_Base, http://dbpedia.org/resource/San_Juan_Gym, http://dbpedia.org/resource/Celebrity_DAT_Com, http://dbpedia.org/resource/Centennial_Tower_(Philippines), http://dbpedia.org/resource/Philippine_Society_of_Information_Technology_Educators, http://dbpedia.org/resource/Dear_Friend:_Kay_Tagal_Kitang_Hinintay, http://dbpedia.org/resource/La_Consolacion_College_–_Novaliches, http://dbpedia.org/resource/The_Enterprise_Center_Tower_1, http://dbpedia.org/resource/GSIS_Museo_ng_Sining, http://dbpedia.org/resource/Iglesia_ni_Cristo_Museum_and_Gallery, http://dbpedia.org/resource/Plaza_de_Mexico_(Manila), http://dbpedia.org/resource/FlipTop_(rap_battle), http://dbpedia.org/resource/Satoshi_Citadel_Industries, http://dbpedia.org/resource/Nissan_Motor_Philippines, http://dbpedia.org/resource/Baler_National_High_School, http://dbpedia.org/resource/Malolos_Historic_Town_Center, http://dbpedia.org/resource/Ceres_Pasalubong, http://dbpedia.org/resource/TV_Patrol, http://dbpedia.org/resource/Talk_'N_Text, http://dbpedia.org/resource/U.S._Naval_Base_Subic_Bay, http://dbpedia.org/resource/Burnham_Park_(Philippines), http://dbpedia.org/resource/Del_Monte_Airfield, http://dbpedia.org/resource/Iba_Airfield, http://dbpedia.org/resource/Malinta_Tunnel, http://dbpedia.org/resource/The_Stratford_Residences, http://dbpedia.org/resource/Mehan_Garden, http://dbpedia.org/resource/Kawasaki_Motors_Philippines, http://dbpedia.org/resource/San_Fernando_de_Dilao_Church, http://dbpedia.org/resource/Petron_Corporation, http://dbpedia.org/resource/Binibining_Pilipinas, http://dbpedia.org/resource/Miss_World_Philippines, http://dbpedia.org/resource/Isetann_Department_Store, http://dbpedia.org/resource/Philippine_Sports_Commission, http://dbpedia.org/resource/Ilustrado_(TV_series), http://dbpedia.org/resource/Philippine_Tarsier_Foundation, http://dbpedia.org/resource/Universal_Records_(Philippines), http://dbpedia.org/resource/Agusan_Marsh_Wildlife_Sanctuary, http://dbpedia.org/resource/Aurora_Memorial_National_Park, http://dbpedia.org/resource/Balbalasang-Balbalan_National_Park, http://dbpedia.org/resource/Baler_Bay, http://dbpedia.org/resource/Bangan_Hill_National_Park, http://dbpedia.org/resource/Bessang_Pass_Natural_Monument, http://dbpedia.org/resource/Bicol_Natural_Park, http://dbpedia.org/resource/Bigbiga_Protected_Landscape, http://dbpedia.org/resource/Bongsanglay_Natural_Park, http://dbpedia.org/resource/Bread_of_Life_Ministries_International, http://dbpedia.org/resource/Bulabog_Putian_National_Park, http://dbpedia.org/resource/Bureau_of_Internal_Revenue_(Philippines), http://dbpedia.org/resource/Casecnan_Protected_Landscape, http://dbpedia.org/resource/Cassamata_Hill_National_Park, http://dbpedia.org/resource/Central_Cebu_Protected_Landscape, http://dbpedia.org/resource/Commission_on_Human_Rights_(Philippines), http://dbpedia.org/resource/Computer_Professionals'_Union, http://dbpedia.org/resource/Digital_Telecommunications_Philippines, http://dbpedia.org/resource/Guadalupe_Mabugnao_Mainit_Hot_Spring_National_Park, http://dbpedia.org/resource/Haribon_Foundation, http://dbpedia.org/resource/Hundred_Islands_National_Park, http://dbpedia.org/resource/Integrated_Bar_of_the_Philippines, http://dbpedia.org/resource/JG_Summit_Holdings, http://dbpedia.org/resource/José_Rizal_Memorial_Protected_Landscape, http://dbpedia.org/resource/Kuapnit_Balinsasayao_National_Park, http://dbpedia.org/resource/Lake_Butig_National_Park, http://dbpedia.org/resource/Lake_Wood_(Philippines), http://dbpedia.org/resource/Libmanan_Caves_National_Park, http://dbpedia.org/resource/Libunao_Protected_Landscape, http://dbpedia.org/resource/Lidlidda_Protected_Landscape, http://dbpedia.org/resource/Mado_Hot_Spring_National_Park, http://dbpedia.org/resource/Magapit_Protected_Landscape, http://dbpedia.org/resource/Manila_Film_Center, http://dbpedia.org/resource/Mati_Protected_Landscape, http://dbpedia.org/resource/Minalungao_National_Park, http://dbpedia.org/resource/Mount_Inayawan_Range_Natural_Park, http://dbpedia.org/resource/Mount_Timolan_Protected_Landscape, http://dbpedia.org/resource/National_Youth_Commission_(Philippines), http://dbpedia.org/resource/Northern_Luzon_Heroes_Hill_National_Park, http://dbpedia.org/resource/Northern_Negros_Natural_Park, http://dbpedia.org/resource/Northern_Sierra_Madre_Natural_Park, http://dbpedia.org/resource/Northwest_Panay_Peninsula_Natural_Park, http://dbpedia.org/resource/Pagsanjan_Gorge_National_Park, http://dbpedia.org/resource/Pamitinan_Cave, http://dbpedia.org/resource/Panay_Gulf, http://dbpedia.org/resource/Pantuwaraya_Lake_National_Park, http://dbpedia.org/resource/Pasaleng_Bay, http://dbpedia.org/resource/People's_Park_(Davao_City), http://dbpedia.org/resource/Peñablanca_Protected_Landscape_and_Seascape, http://dbpedia.org/resource/Philippine_Overseas_Employment_Administration, http://dbpedia.org/resource/Plaza_Miranda, http://dbpedia.org/resource/Rajah_Sikatuna_Protected_Landscape, http://dbpedia.org/resource/Red_Mobile, http://dbpedia.org/resource/Rungkunan_National_Park, http://dbpedia.org/resource/Sacred_Mountain_National_Park, http://dbpedia.org/resource/Salikata_National_Park, http://dbpedia.org/resource/Salinas_Natural_Monument, http://dbpedia.org/resource/Samahang_Basketbol_ng_Pilipinas, http://dbpedia.org/resource/Santa_Lucia_Protected_Landscape, http://dbpedia.org/resource/Sibalom_Natural_Park, http://dbpedia.org/resource/Sibuguey_Bay]... [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 277, P: 0.548, R: 0.608, macro F1: 0.557, macro F1*: 0.576 -[INFO] Current time: 6285 ms, average: 2857.93 ms/ques, entity detection: 391.95 ms/ques (linking tool: 300.06 ms/ques), relation detection: 930.82 ms/ques (similarity: 727.00 ms/ques), EDG generation: 65.72 ms/ques +[INFO] Cumulative metrics, sample: 277, P: 0.548, R: 0.608, macro F1: 0.576 +[INFO] Current question consumed time: 5759 ms, average: 2657.55 ms/ques, entity detection: 363.17 ms/ques (linking tool: 279.92 ms/ques), relation detection: 888.04 ms/ques (similarity: 714.44 ms/ques), EDG generation: 64.98 ms/ques [INFO] Entity Linking P: 0.7545126353790613, R: 0.7148014440433214, micro F1: 0.7268351383874847, macro F1: 0.7341204019904382 [INFO] Relation Linking P: 0.6034897713598074, R: 0.5830324909747292, micro F1: 0.5855595667870036, macro F1: 0.5930847753018798 [INFO] Type Linking P: 0.5270758122743683, R: 0.5270758122743683, micro F1: 0.5270758122743683, macro F1: 0.5270758122743683 @@ -3886,8 +3886,8 @@ Question 277: Count the number of families of the Animal kingdom. [INFO] Golden Answer: [8196] [INFO] Predicted Answer: [1] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 278, P: 0.546, R: 0.606, macro F1: 0.555, macro F1*: 0.574 -[INFO] Current time: 1603 ms, average: 2853.41 ms/ques, entity detection: 391.07 ms/ques (linking tool: 299.29 ms/ques), relation detection: 929.73 ms/ques (similarity: 725.68 ms/ques), EDG generation: 65.52 ms/ques +[INFO] Cumulative metrics, sample: 278, P: 0.546, R: 0.606, macro F1: 0.574 +[INFO] Current question consumed time: 1401 ms, average: 2653.03 ms/ques, entity detection: 362.32 ms/ques (linking tool: 279.18 ms/ques), relation detection: 886.81 ms/ques (similarity: 713.13 ms/ques), EDG generation: 64.78 ms/ques [INFO] Entity Linking P: 0.7517985611510791, R: 0.7122302158273381, micro F1: 0.7242206235011988, macro F1: 0.731479681119969 [INFO] Relation Linking P: 0.6013189448441246, R: 0.5809352517985612, micro F1: 0.5834532374100719, macro F1: 0.5909513768295708 [INFO] Type Linking P: 0.5287769784172662, R: 0.5287769784172662, micro F1: 0.5287769784172662, macro F1: 0.5287769784172662 @@ -3900,8 +3900,8 @@ Question 278: What is the political party of the person succeeded by A.K. Singh? [INFO] Golden Answer: [http://dbpedia.org/resource/Indian_National_Congress] [INFO] Predicted Answer: [http://dbpedia.org/resource/Jantar_Mantar_(Jaipur), http://dbpedia.org/resource/Qutb_complex, http://dbpedia.org/resource/Mahabodhi_Temple, http://dbpedia.org/resource/Khajuraho_Group_of_Monuments, http://dbpedia.org/resource/Kumbhalgarh, http://dbpedia.org/resource/Valley_of_Flowers_National_Park, http://dbpedia.org/resource/Ranthambore_Fort, http://dbpedia.org/resource/Hill_Forts_of_Rajasthan, http://dbpedia.org/resource/Fatehpur_Sikri, http://dbpedia.org/resource/Agra_Fort, http://dbpedia.org/resource/Elephanta_Caves, http://dbpedia.org/resource/Ellora_Caves, http://dbpedia.org/resource/Hampi, http://dbpedia.org/resource/Jaisalmer, http://dbpedia.org/resource/Bhimbetka_rock_shelters, http://dbpedia.org/resource/Champaner-Pavagadh_Archaeological_Park, http://dbpedia.org/resource/Group_of_Monuments_at_Mahabalipuram, http://dbpedia.org/resource/Cave_Temples_of_Mahabalipuram, http://dbpedia.org/resource/Darjeeling_Himalayan_Railway, http://dbpedia.org/resource/Nilgiri_Mountain_Railway, http://dbpedia.org/resource/Pattadakal, http://dbpedia.org/resource/Old_Goa, http://dbpedia.org/resource/Airavatesvara_Temple, http://dbpedia.org/resource/Gangaikonda_Cholapuram, http://dbpedia.org/resource/Great_Living_Chola_Temples, http://dbpedia.org/resource/Rani_ki_vav, http://dbpedia.org/resource/Varaha_Cave_Temple, http://dbpedia.org/resource/Kalka–Shimla_Railway, http://dbpedia.org/resource/Mountain_Railways_of_India, http://dbpedia.org/resource/Gagron_Fort, http://dbpedia.org/resource/Descent_of_the_Ganges_(Mahabalipuram), http://dbpedia.org/resource/Churches_and_convents_of_Goa, http://dbpedia.org/resource/Nakula_Sahadeva_Ratha] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 279, P: 0.544, R: 0.603, macro F1: 0.553, macro F1*: 0.572 -[INFO] Current time: 11776 ms, average: 2885.39 ms/ques, entity detection: 390.00 ms/ques (linking tool: 298.48 ms/ques), relation detection: 950.49 ms/ques (similarity: 732.58 ms/ques), EDG generation: 65.65 ms/ques +[INFO] Cumulative metrics, sample: 279, P: 0.544, R: 0.603, macro F1: 0.572 +[INFO] Current question consumed time: 11002 ms, average: 2682.95 ms/ques, entity detection: 361.32 ms/ques (linking tool: 278.41 ms/ques), relation detection: 906.33 ms/ques (similarity: 720.11 ms/ques), EDG generation: 64.91 ms/ques [INFO] Entity Linking P: 0.7491039426523297, R: 0.7096774193548387, micro F1: 0.7216248506571085, macro F1: 0.7288578901482128 [INFO] Relation Linking P: 0.5991636798088411, R: 0.578853046594982, micro F1: 0.5813620071684588, macro F1: 0.5888332715362748 [INFO] Type Linking P: 0.5268817204301075, R: 0.5268817204301075, micro F1: 0.5268817204301075, macro F1: 0.5268817204301075 @@ -3915,8 +3915,8 @@ Question 279: Who is the head coach of the teams in Middlesbrough FC? [INFO] Predicted Answer: [http://dbpedia.org/resource/Ruud_Gullit, http://dbpedia.org/resource/Kenny_Dalglish, http://dbpedia.org/resource/Bobby_Robson, http://dbpedia.org/resource/Brian_Clough, http://dbpedia.org/resource/George_Graham_(footballer), http://dbpedia.org/resource/José_Mourinho, http://dbpedia.org/resource/David_O'Leary, http://dbpedia.org/resource/Gary_Megson, http://dbpedia.org/resource/Sammy_Lee_(footballer), http://dbpedia.org/resource/Gary_Bowyer, http://dbpedia.org/resource/Paul_Lambert, http://dbpedia.org/resource/David_Moyes, http://dbpedia.org/resource/Gianluca_Vialli, http://dbpedia.org/resource/Graeme_Souness, http://dbpedia.org/resource/Mark_Hughes, http://dbpedia.org/resource/Paul_Ince, http://dbpedia.org/resource/Sam_Allardyce, http://dbpedia.org/resource/Martin_O'Neill, http://dbpedia.org/resource/Jimmy_Phillips_(footballer,_born_1966), http://dbpedia.org/resource/Owen_Coyle, http://dbpedia.org/resource/Dougie_Freedman, http://dbpedia.org/resource/Simon_Grayson, http://dbpedia.org/resource/Dean_Smith_(footballer,_born_1971), http://dbpedia.org/resource/Sean_Dyche, http://dbpedia.org/resource/Derek_McInnes, http://dbpedia.org/resource/Sean_O'Driscoll, http://dbpedia.org/resource/Dean_Holden, http://dbpedia.org/resource/Lee_Johnson_(footballer), http://dbpedia.org/resource/Steve_Evans_(footballer,_born_1962), http://dbpedia.org/resource/Ian_Branfoot, http://dbpedia.org/resource/Kevin_Keegan, http://dbpedia.org/resource/Sami_Hyypiä, http://dbpedia.org/resource/Chris_Hughton, http://dbpedia.org/resource/Darren_Wassall, http://dbpedia.org/resource/Paul_Clement_(football_coach), http://dbpedia.org/resource/David_Pleat, http://dbpedia.org/resource/Alan_Knill, http://dbpedia.org/resource/Mick_McCarthy, http://dbpedia.org/resource/Tommy_Burns_(footballer), http://dbpedia.org/resource/Alan_Pardew, http://dbpedia.org/resource/Terry_Bullivant, http://dbpedia.org/resource/Lawrence_Furniss, http://dbpedia.org/resource/Joseph_Wostinholm, http://dbpedia.org/resource/Bruce_Rioch, http://dbpedia.org/resource/Arsène_Wenger, http://dbpedia.org/resource/Pat_Rice, http://dbpedia.org/resource/Stewart_Houston, http://dbpedia.org/resource/Andrew_Hughes_(footballer,_born_1978), http://dbpedia.org/resource/Caretaker_manager, http://dbpedia.org/resource/Neil_Lennon, http://dbpedia.org/resource/Gus_Poyet, http://dbpedia.org/resource/Ronnie_Moore, http://dbpedia.org/resource/Paul_Murray_(footballer), http://dbpedia.org/resource/Colin_Cooper, http://dbpedia.org/resource/Alan_Curbishley, http://dbpedia.org/resource/Kenny_Jackett, http://dbpedia.org/resource/Aitor_Karanka, http://dbpedia.org/resource/Dave_Stringer, http://dbpedia.org/resource/Kit_Symons, http://dbpedia.org/resource/Peter_Grant_(footballer,_born_1965), http://dbpedia.org/resource/Slaviša_Jokanović, http://dbpedia.org/resource/Stuart_Gray_(footballer,_born_1960), http://dbpedia.org/resource/Gérard_Houllier, http://dbpedia.org/resource/Phil_Thompson, http://dbpedia.org/resource/Rafael_Benítez, http://dbpedia.org/resource/Brendan_Rodgers, http://dbpedia.org/resource/Steve_Clarke, http://dbpedia.org/resource/John_Carver_(footballer), http://dbpedia.org/resource/Glenn_Roeder, http://dbpedia.org/resource/Alan_Shearer, http://dbpedia.org/resource/Joe_Kinnear, http://dbpedia.org/resource/Colin_Calderwood, http://dbpedia.org/resource/Stan_Cullis, http://dbpedia.org/resource/Roy_Evans, http://dbpedia.org/resource/Roberto_Martínez, http://dbpedia.org/resource/Jean_Tigana, http://dbpedia.org/resource/Chris_Coleman_(footballer), http://dbpedia.org/resource/Terry_Venables, http://dbpedia.org/resource/Peter_Reid, http://dbpedia.org/resource/Tony_Mowbray, http://dbpedia.org/resource/Nigel_Adkins, http://dbpedia.org/resource/Eddie_Gray_(footballer), http://dbpedia.org/resource/Eddie_Gray_(footballer,_born_1948), http://dbpedia.org/resource/Uwe_Rösler, http://dbpedia.org/resource/Malky_Mackay, http://dbpedia.org/resource/Gary_Caldwell, http://dbpedia.org/resource/Steve_Cotterill, http://dbpedia.org/resource/Michael_Appleton, http://dbpedia.org/resource/David_Flitcroft, http://dbpedia.org/resource/Danny_Wilson_(footballer,_born_1960), http://dbpedia.org/resource/Neil_Warnock, http://dbpedia.org/resource/Nigel_Worthington, http://dbpedia.org/resource/Guy_Luzon, http://dbpedia.org/resource/José_Riga, http://dbpedia.org/resource/Karel_Fraeye, http://dbpedia.org/resource/Barry_Ferguson, http://dbpedia.org/resource/Harry_Redknapp, http://dbpedia.org/resource/Manuel_Pellegrini, http://dbpedia.org/resource/Dave_Jones_(footballer,_born_1956), http://dbpedia.org/resource/Brian_McDermott_(footballer), http://dbpedia.org/resource/Paul_Jewell, http://dbpedia.org/resource/Roy_Hodgson, http://dbpedia.org/resource/Lawrie_Sanchez, http://dbpedia.org/resource/Kevin_Nolan, http://dbpedia.org/resource/Andy_Hessenthaler, http://dbpedia.org/resource/Ian_Hendon, http://dbpedia.org/resource/Mark_Warburton, http://dbpedia.org/resource/Alex_McLeish, http://dbpedia.org/resource/Billy_Davies, http://dbpedia.org/resource/Gary_Brazil, http://dbpedia.org/resource/Stuart_Pearce, http://dbpedia.org/resource/Gary_Johnson_(footballer,_born_1955), http://dbpedia.org/resource/Keith_Millen, http://dbpedia.org/resource/Steve_Coppell, http://dbpedia.org/resource/Jon_Rudkin, http://dbpedia.org/resource/Mike_Stowell, http://dbpedia.org/resource/Nigel_Pearson, http://dbpedia.org/resource/Sven-Göran_Eriksson, http://dbpedia.org/resource/George_Burley, http://dbpedia.org/resource/Chris_Powell, http://dbpedia.org/resource/David_Wagner_(soccer), http://dbpedia.org/resource/Mark_Lillis, http://dbpedia.org/resource/Dean_Saunders, http://dbpedia.org/resource/Paul_Dickov, http://dbpedia.org/resource/Paul_Heckingbottom, http://dbpedia.org/resource/Tommy_Wright_(footballer,_born_1966), http://dbpedia.org/resource/Paul_Peschisolido, http://dbpedia.org/resource/Dave_Russell_(footballer), http://dbpedia.org/resource/Bill_Moore_(footballer), http://dbpedia.org/resource/Gianfranco_Zola, http://dbpedia.org/resource/Jimmy_Floyd_Hasselbaink, http://dbpedia.org/resource/Chris_Ramsey_(footballer), http://dbpedia.org/resource/Ståle_Solbakken, http://dbpedia.org/resource/Eddie_Howe, http://dbpedia.org/resource/Steve_Bruce, http://dbpedia.org/resource/Keith_Hill_(footballer), http://dbpedia.org/resource/Paul_Hart, http://dbpedia.org/resource/David_Dunn, http://dbpedia.org/resource/John_Sheridan_(footballer), http://dbpedia.org/resource/Darren_Kelly, http://dbpedia.org/resource/Alex_Neil_(footballer), http://dbpedia.org/resource/Neil_Adams_(footballer), http://dbpedia.org/resource/Rob_Kelly, http://dbpedia.org/resource/Steve_McClaren, http://dbpedia.org/resource/Lee_Clark_(footballer), http://dbpedia.org/resource/Bob_Peeters, http://dbpedia.org/resource/Nick_Barmby, http://dbpedia.org/resource/Michael_Laudrup, http://dbpedia.org/resource/Darren_Ferguson, http://dbpedia.org/resource/Greg_Abbott_(footballer), http://dbpedia.org/resource/Nigel_Clough, http://dbpedia.org/resource/John_Toshack, http://dbpedia.org/resource/Jimmy_Mullen_(footballer,_born_1952), http://dbpedia.org/resource/Felix_Magath, http://dbpedia.org/resource/Mark_Robins, http://dbpedia.org/resource/Micky_Adams, http://dbpedia.org/resource/Andy_Thorn_(footballer), http://dbpedia.org/resource/Ian_Holloway, http://dbpedia.org/resource/Steve_Lomas, http://dbpedia.org/resource/Neil_Harris_(footballer,_born_1977), http://dbpedia.org/resource/Trevor_Brooking, http://dbpedia.org/resource/Steven_Pressley, http://dbpedia.org/resource/Alex_Smith_(footballer,_born_1939), http://dbpedia.org/resource/Gary_Holt_(footballer), http://dbpedia.org/resource/Roberto_Mancini, http://dbpedia.org/resource/Iain_Dowie, http://dbpedia.org/resource/Martin_Jol, http://dbpedia.org/resource/Steve_Kean, http://dbpedia.org/resource/Henning_Berg, http://dbpedia.org/resource/Eric_Black, http://dbpedia.org/resource/James_Beattie_(footballer), http://dbpedia.org/resource/Lee_Carsley, http://dbpedia.org/resource/Marinus_Dijkhuizen, http://dbpedia.org/resource/Osvaldo_Ardiles, http://dbpedia.org/resource/Graham_Turner, http://dbpedia.org/resource/Carlos_Carvalhal, http://dbpedia.org/resource/Óscar_García_Junyent, http://dbpedia.org/resource/Giuseppe_Sannino, http://dbpedia.org/resource/Billy_McKinlay, http://dbpedia.org/resource/Neil_Redfearn, http://dbpedia.org/resource/Darko_Milanič, http://dbpedia.org/resource/Dave_Hockaday, http://dbpedia.org/resource/Graham_Taylor, http://dbpedia.org/resource/Paul_Sturrock, http://dbpedia.org/resource/Steve_Wigley, http://dbpedia.org/resource/Mark_Venus, http://dbpedia.org/resource/Gordon_Strachan, http://dbpedia.org/resource/Steve_Thompson_(footballer,_born_1964), http://dbpedia.org/resource/Velimir_Zajec, http://dbpedia.org/resource/Alain_Perrin, http://dbpedia.org/resource/Paulo_Sousa, http://dbpedia.org/resource/Martin_Allen, http://dbpedia.org/resource/Paul_Mariner, http://dbpedia.org/resource/Ole_Gunnar_Solskjær, http://dbpedia.org/resource/Russell_Slade, http://dbpedia.org/resource/Danny_Gabbidon, http://dbpedia.org/resource/Scott_Young_(Welsh_footballer), http://dbpedia.org/resource/Karl_Robinson, http://dbpedia.org/resource/Brian_Laws, http://dbpedia.org/resource/John_Nicholson_(football_secretary), http://dbpedia.org/resource/Martin_Kuhl, http://dbpedia.org/resource/Kevin_Blackwell, http://dbpedia.org/resource/Gary_Speed, http://dbpedia.org/resource/Claudio_Ranieri, http://dbpedia.org/resource/Avram_Grant, http://dbpedia.org/resource/Ray_Wilkins, http://dbpedia.org/resource/Guus_Hiddink, http://dbpedia.org/resource/Luiz_Felipe_Scolari, http://dbpedia.org/resource/Roberto_Di_Matteo, http://dbpedia.org/resource/Graham_Westley, http://dbpedia.org/resource/Mark_McGhee, http://dbpedia.org/resource/John_Pemberton_(footballer), http://dbpedia.org/resource/Mick_Gooding, http://dbpedia.org/resource/Fred_Stewart_(football_manager), http://dbpedia.org/resource/Stuart_McMillan_(footballer), Ruud Gullit@en, Kenny Dalglish@en, David O'Leary@en, Graeme Souness@en, ()@en, George Graham@en, Kevin Keegan@en, Darren Wassall@en, --03-18, --03-25, Alan Pardew ()@en, Lawrence Furniss@en, Bruce Rioch@en, & Lee Turner@en, Steve Clarke@en, Sir Bobby Robson@en, Roy Evans managers until November)@en, Peter Reid@en, Terry Venables@en, Gary Caldwell@en, ( )@en, ''@en, Barry Ferguson@en, --12-17, --12-16, Steve Cotterill@en, Billy Davies ()@en, Gary Brazil@en, Nigel Pearson@en, --11-15, --10-24, Jon Rudkin and Mike Stowell@en, Caretaker@en, Paul Heckingbottom Caretaker@en, Rob Kelly@en, Steve McClaren@en, Jimmy Mullen@en, Mark Robins@en, Mark Lillis@en, Chris Powell@en, Chris Coleman@en, Ian Holloway@en, Steve Lomas@en, Neil Harris & Scott Fitzgerald@en, Glenn Roeder@en, Sir Trevor Brooking@en, Steve Kean@en, Eric Black (') Henning Berg (')@en, Gary Bowyer (' ) Michael Appleton (')@en, Gary Bowyer ()@en, Osvaldo Ardiles@en, Roy Evans@en, Aitor Karanka@en, --10-01, --10-03, Chris Powell and Mike Stowell@en, &@en, None@en, (Interim)@en, http://dbpedia.org/resource/Boudewijn_Zenden, http://dbpedia.org/resource/Eddie_Newton, http://dbpedia.org/resource/Steve_Holland, )@en, http://dbpedia.org/resource/Jimmy_Quinn_(footballer,_born_1959), Fred Stewart@en, http://dbpedia.org/resource/2000–01_Middlesbrough_F.C._season, http://dbpedia.org/resource/1995–96_Middlesbrough_F.C._season, http://dbpedia.org/resource/2005–06_West_Bromwich_Albion_F.C._season, http://dbpedia.org/resource/2004–05_West_Bromwich_Albion_F.C._season, http://dbpedia.org/resource/2006–07_West_Bromwich_Albion_F.C._season, http://dbpedia.org/resource/2007–08_Sheffield_United_F.C._season, http://dbpedia.org/resource/1997–98_Middlesbrough_F.C._season, http://dbpedia.org/resource/1998–99_Middlesbrough_F.C._season, http://dbpedia.org/resource/1999–2000_Middlesbrough_F.C._season, http://dbpedia.org/resource/1996–97_Middlesbrough_F.C._season, http://dbpedia.org/resource/1994–95_Middlesbrough_F.C._season, http://dbpedia.org/resource/2011–12_Hartlepool_United_F.C._season, http://dbpedia.org/resource/2008–09_Blackburn_Rovers_F.C._season, http://dbpedia.org/resource/2013–14_Blackpool_F.C._season, http://dbpedia.org/resource/2012–13_Blackpool_F.C._season, http://dbpedia.org/resource/2015_São_Paulo_F.C._season, http://dbpedia.org/resource/Esporte_Clube_Bahia, http://dbpedia.org/resource/2015_CR_Vasco_da_Gama_season, http://dbpedia.org/resource/Calcio_Como, http://dbpedia.org/resource/2014–15_Cagliari_Calcio_season, http://dbpedia.org/resource/Associação_Portuguesa_de_Desportos, http://dbpedia.org/resource/2016_Associação_Portuguesa_de_Desportos_season, http://dbpedia.org/resource/The_New_Saints_F.C., http://dbpedia.org/resource/2009–10_Darlington_F.C._season, http://dbpedia.org/resource/1976–77_Burnley_F.C._season, http://dbpedia.org/resource/Bolton_Wanderers_F.C._Reserves_and_Academy, http://dbpedia.org/resource/2012–13_Bolton_Wanderers_F.C._season, http://dbpedia.org/resource/1945–46_Dumbarton_F.C._season, http://dbpedia.org/resource/1960–61_Everton_F.C._season, http://dbpedia.org/resource/2013–14_Zamalek_SC_season, http://dbpedia.org/resource/2015–16_Zamalek_SC_season, http://dbpedia.org/resource/2014–15_Burton_Albion_F.C._season, http://dbpedia.org/resource/Queens_Park_Rangers_F.C., http://dbpedia.org/resource/2015–16_Queens_Park_Rangers_F.C._season, http://dbpedia.org/resource/2015–16_Burton_Albion_F.C._season, http://dbpedia.org/resource/2013–14_SpVgg_Unterhaching_season, http://dbpedia.org/resource/CD_Atlético_Baleares, http://dbpedia.org/resource/Limerick_F.C., http://dbpedia.org/resource/2009–10_Leicester_City_F.C._season, http://dbpedia.org/resource/2008–09_Leicester_City_F.C._season, http://dbpedia.org/resource/2011–12_Leicester_City_F.C._season, http://dbpedia.org/resource/2012–13_Leicester_City_F.C._season, http://dbpedia.org/resource/2010–11_Hull_City_A.F.C._season, http://dbpedia.org/resource/2011–12_Hull_City_A.F.C._season, http://dbpedia.org/resource/2007–08_Southampton_F.C._season, http://dbpedia.org/resource/2013–14_Leicester_City_F.C._season, http://dbpedia.org/resource/2014–15_Leicester_City_F.C._season, http://dbpedia.org/resource/1995–96_Torquay_United_F.C._season, http://dbpedia.org/resource/Galway_W.F.C., http://dbpedia.org/resource/1988–89_Newport_County_A.F.C._season, http://dbpedia.org/resource/1998–99_Carmarthen_Town_A.F.C._season, http://dbpedia.org/resource/2013–14_Kilmarnock_F.C._season, http://dbpedia.org/resource/2011–12_Queen_of_the_South_F.C._season, http://dbpedia.org/resource/2014–15_Kilmarnock_F.C._season, http://dbpedia.org/resource/2015–16_Dunfermline_Athletic_F.C._season, http://dbpedia.org/resource/Dunfermline_Athletic_F.C., http://dbpedia.org/resource/2012–13_Queen_of_the_South_F.C._season, http://dbpedia.org/resource/1991–92_Birmingham_City_F.C._season, http://dbpedia.org/resource/1992–93_Birmingham_City_F.C._season, http://dbpedia.org/resource/1993–94_Birmingham_City_F.C._season, http://dbpedia.org/resource/West_Auckland_Town_F.C., http://dbpedia.org/resource/1997–98_Bradford_City_A.F.C._season, http://dbpedia.org/resource/1996–97_Bradford_City_A.F.C._season, http://dbpedia.org/resource/1997–98_Stoke_City_F.C._season, http://dbpedia.org/resource/2013–14_Carlisle_United_F.C._season, http://dbpedia.org/resource/2014–15_Carlisle_United_F.C._season, http://dbpedia.org/resource/2015–16_Hartlepool_United_F.C._season, http://dbpedia.org/resource/Hartlepool_United_F.C., http://dbpedia.org/resource/1984_Minnesota_Strikers_season, http://dbpedia.org/resource/Minnesota_Strikers, http://dbpedia.org/resource/1983_Fort_Lauderdale_Strikers_indoor_season, http://dbpedia.org/resource/1983_Fort_Lauderdale_Strikers_season, http://dbpedia.org/resource/1978–79_Nottingham_Forest_F.C._season, http://dbpedia.org/resource/1974–75_Leeds_United_A.F.C._season, http://dbpedia.org/resource/1977–78_Nottingham_Forest_F.C._season, http://dbpedia.org/resource/1990–91_Nottingham_Forest_F.C._season, http://dbpedia.org/resource/1991–92_Nottingham_Forest_F.C._season, http://dbpedia.org/resource/1992–93_Nottingham_Forest_F.C._season, http://dbpedia.org/resource/1971–72_Derby_County_F.C._season, http://dbpedia.org/resource/1972–73_Derby_County_F.C._season, http://dbpedia.org/resource/FFA_Centre_of_Excellence, http://dbpedia.org/resource/2004–05_Hibernian_F.C._season, http://dbpedia.org/resource/2005–06_Hibernian_F.C._season, http://dbpedia.org/resource/2006–07_Hibernian_F.C._season, http://dbpedia.org/resource/2012–13_Middlesbrough_F.C._season, http://dbpedia.org/resource/2002–03_Ipswich_Town_F.C._season, http://dbpedia.org/resource/2011–12_Middlesbrough_F.C._season, http://dbpedia.org/resource/2010–11_Middlesbrough_F.C._season, http://dbpedia.org/resource/Coventry_City_F.C., http://dbpedia.org/resource/2014–15_Coventry_City_F.C._season, http://dbpedia.org/resource/2015–16_Coventry_City_F.C._season, http://dbpedia.org/resource/2008–09_West_Bromwich_Albion_F.C._season, http://dbpedia.org/resource/2013–14_Middlesbrough_F.C._season, http://dbpedia.org/resource/2009–10_Celtic_F.C._season, http://dbpedia.org/resource/2007–08_West_Bromwich_Albion_F.C._season, http://dbpedia.org/resource/2012–13_Crystal_Palace_F.C._season, http://dbpedia.org/resource/2015–16_Gateshead_F.C._season, http://dbpedia.org/resource/2003–04_Blackburn_Rovers_F.C._season, http://dbpedia.org/resource/2004–05_Blackburn_Rovers_F.C._season, http://dbpedia.org/resource/1985–86_Rangers_F.C._season, http://dbpedia.org/resource/1987–88_Rangers_F.C._season, http://dbpedia.org/resource/1989–90_Rangers_F.C._season, http://dbpedia.org/resource/2004–05_Newcastle_United_F.C._season, http://dbpedia.org/resource/2005–06_Newcastle_United_F.C._season, http://dbpedia.org/resource/1996–97_Southampton_F.C._season, http://dbpedia.org/resource/1997–98_S.L._Benfica_season, http://dbpedia.org/resource/1998–99_S.L._Benfica_season, http://dbpedia.org/resource/2000–01_Blackburn_Rovers_F.C._season, http://dbpedia.org/resource/2001–02_Blackburn_Rovers_F.C._season, http://dbpedia.org/resource/1990–91_Liverpool_F.C._season, http://dbpedia.org/resource/1991–92_Liverpool_F.C._season, http://dbpedia.org/resource/1992–93_Liverpool_F.C._season, http://dbpedia.org/resource/1993–94_Liverpool_F.C._season, http://dbpedia.org/resource/1999–2000_Blackburn_Rovers_F.C._season, http://dbpedia.org/resource/2002–03_Blackburn_Rovers_F.C._season, http://dbpedia.org/resource/1986–87_Rangers_F.C._season, http://dbpedia.org/resource/1988–89_Rangers_F.C._season, http://dbpedia.org/resource/1990–91_Rangers_F.C._season, http://dbpedia.org/resource/1995–96_Galatasaray_S.K._season, http://dbpedia.org/resource/2014_Sarpsborg_08_FF_season, http://dbpedia.org/resource/2013_Sarpsborg_08_FF_season, http://dbpedia.org/resource/2014–15_Hartlepool_United_F.C._season, http://dbpedia.org/resource/2011–12_Bradford_City_A.F.C._season, http://dbpedia.org/resource/2013–14_Hartlepool_United_F.C._season, http://dbpedia.org/resource/1998–99_Barnsley_F.C._season, http://dbpedia.org/resource/Tottenham_Hotspur_F.C._Reserves_and_Academy, http://dbpedia.org/resource/Aston_Villa_Under-21s_and_Academy, http://dbpedia.org/resource/1993–94_Barnsley_F.C._season, http://dbpedia.org/resource/Glentoran_F.C., http://dbpedia.org/resource/Longford_Town_F.C., http://dbpedia.org/resource/2015–16_Middlesbrough_F.C._season, http://dbpedia.org/resource/2014–15_Middlesbrough_F.C._season, http://dbpedia.org/resource/Middlesbrough_F.C., http://dbpedia.org/resource/2007–08_Middlesbrough_F.C._season, http://dbpedia.org/resource/2006–07_Middlesbrough_F.C._season, http://dbpedia.org/resource/2009–10_Middlesbrough_F.C._season, http://dbpedia.org/resource/2008–09_Middlesbrough_F.C._season, http://dbpedia.org/resource/2015_Kelantan_FA_season, http://dbpedia.org/resource/2014_Kelantan_FA_season, http://dbpedia.org/resource/Bridport_F.C., http://dbpedia.org/resource/1995–96_Grimsby_Town_F.C._season, http://dbpedia.org/resource/2009–10_Burnley_F.C._season, http://dbpedia.org/resource/2010–11_Burnley_F.C._season, http://dbpedia.org/resource/2012_Shamrock_Rovers_F.C._season, http://dbpedia.org/resource/2008–09_Sheffield_Wednesday_F.C._season, http://dbpedia.org/resource/2006–07_Sheffield_Wednesday_F.C._season, http://dbpedia.org/resource/2007–08_Sheffield_Wednesday_F.C._season, http://dbpedia.org/resource/2013–14_Scunthorpe_United_F.C._season, http://dbpedia.org/resource/2012–13_Scunthorpe_United_F.C._season, http://dbpedia.org/resource/1999–2000_Scunthorpe_United_F.C._season, http://dbpedia.org/resource/2008–09_Port_Vale_F.C._season, http://dbpedia.org/resource/2007–08_Livingston_F.C._season, http://dbpedia.org/resource/2015–16_Barnsley_F.C._season, http://dbpedia.org/resource/Bryan_Robson, http://dbpedia.org/resource/Lennie_Lawrence, http://dbpedia.org/resource/Gareth_Southgate, http://dbpedia.org/resource/2012–13_Chelsea_F.C._season, Gordon Strachan@en] [INFO] P: 0.02444987775061125, R: 1.0, F: 0.04773269689737471 -[INFO] Cumulative metrics, sample: 280, P: 0.542, R: 0.605, macro F1: 0.551, macro F1*: 0.572 -[INFO] Current time: 5021 ms, average: 2893.02 ms/ques, entity detection: 389.96 ms/ques (linking tool: 298.61 ms/ques), relation detection: 956.24 ms/ques (similarity: 734.83 ms/ques), EDG generation: 65.74 ms/ques +[INFO] Cumulative metrics, sample: 280, P: 0.542, R: 0.605, macro F1: 0.572 +[INFO] Current question consumed time: 4579 ms, average: 2689.72 ms/ques, entity detection: 361.19 ms/ques (linking tool: 278.45 ms/ques), relation detection: 911.60 ms/ques (similarity: 722.37 ms/ques), EDG generation: 65.01 ms/ques [INFO] Entity Linking P: 0.75, R: 0.7107142857142857, micro F1: 0.7226190476190474, macro F1: 0.7298288508557457 [INFO] Relation Linking P: 0.5988095238095238, R: 0.5785714285714286, micro F1: 0.5810714285714286, macro F1: 0.5885165390726563 [INFO] Type Linking P: 0.5285714285714286, R: 0.5285714285714286, micro F1: 0.5285714285714286, macro F1: 0.5285714285714286 @@ -3929,8 +3929,8 @@ Question 280: To which uni. did sidney c patridge and cyril hume both once went [INFO] Golden Answer: [http://dbpedia.org/resource/Yale_University] [INFO] Predicted Answer: [] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 281, P: 0.540, R: 0.603, macro F1: 0.549, macro F1*: 0.570 -[INFO] Current time: 872 ms, average: 2885.83 ms/ques, entity detection: 388.78 ms/ques (linking tool: 297.55 ms/ques), relation detection: 952.84 ms/ques (similarity: 732.22 ms/ques), EDG generation: 65.63 ms/ques +[INFO] Cumulative metrics, sample: 281, P: 0.540, R: 0.603, macro F1: 0.570 +[INFO] Current question consumed time: 744 ms, average: 2682.80 ms/ques, entity detection: 360.11 ms/ques (linking tool: 277.46 ms/ques), relation detection: 908.36 ms/ques (similarity: 719.80 ms/ques), EDG generation: 64.92 ms/ques [INFO] Entity Linking P: 0.7473309608540926, R: 0.708185053380783, micro F1: 0.7200474495848158, macro F1: 0.7272315951587502 [INFO] Relation Linking P: 0.5966785290628707, R: 0.5765124555160143, micro F1: 0.5790035587188612, macro F1: 0.5864221741649245 [INFO] Type Linking P: 0.5302491103202847, R: 0.5302491103202847, micro F1: 0.5302491103202847, macro F1: 0.5302491103202847 @@ -3943,8 +3943,8 @@ Question 281: People in the royal Thai army follow which religion? [INFO] Golden Answer: [http://dbpedia.org/resource/Buddhism, http://dbpedia.org/resource/Theravada, http://dbpedia.org/resource/Santi_Asoke] [INFO] Predicted Answer: [http://dbpedia.org/resource/Phayap_Army, http://dbpedia.org/resource/Prayut_Chan-o-cha, http://dbpedia.org/resource/Praphas_Charusathien, http://dbpedia.org/resource/Sanan_Kachornprasart, http://dbpedia.org/resource/Royal_Thai_Survey_Department, http://dbpedia.org/resource/Surayud_Chulanont, http://dbpedia.org/resource/Sonthi_Boonyaratglin, http://dbpedia.org/resource/Phichit_Kriangsakphichit, http://dbpedia.org/resource/Thanom_Kittikachorn, http://dbpedia.org/resource/Plaek_Phibunsongkhram, http://dbpedia.org/resource/Sarit_Thanarat, http://dbpedia.org/resource/31st_Infantry_Regiment_(Thailand), http://dbpedia.org/resource/Charun_Rattanakun_Seriroengrit, http://dbpedia.org/resource/Khattiya_Sawasdipol, http://dbpedia.org/resource/Yuthasak_Sasiprapha, http://dbpedia.org/resource/Anupong_Paochinda, http://dbpedia.org/resource/Army_Reserve_Force_Students, http://dbpedia.org/resource/Chamlong_Srimuang, http://dbpedia.org/resource/Chavalit_Yongchaiyudh, http://dbpedia.org/resource/Prawit_Wongsuwan, http://dbpedia.org/resource/Prem_Tinsulanonda, http://dbpedia.org/resource/Royal_Guards_(Thailand), http://dbpedia.org/resource/Thahan_Phran, http://dbpedia.org/resource/Phin_Choonhavan, http://dbpedia.org/resource/Thammarak_Isarangkura_na_Ayudhaya, http://dbpedia.org/resource/Songkitti_Jaggabatara, http://dbpedia.org/resource/Kukrit_Pramoj, http://dbpedia.org/resource/Kriangsak_Chamanan, http://dbpedia.org/resource/21st_Infantry_Regiment_(Thailand), http://dbpedia.org/resource/Phraya_Phahonphonphayuhasena, http://dbpedia.org/resource/Suthida_Vajiralongkorn, http://dbpedia.org/resource/Kraichingrith_Phudvinichaikul] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 282, P: 0.539, R: 0.600, macro F1: 0.547, macro F1*: 0.568 -[INFO] Current time: 1626 ms, average: 2881.36 ms/ques, entity detection: 388.15 ms/ques (linking tool: 297.04 ms/ques), relation detection: 951.15 ms/ques (similarity: 731.37 ms/ques), EDG generation: 65.45 ms/ques +[INFO] Cumulative metrics, sample: 282, P: 0.539, R: 0.600, macro F1: 0.568 +[INFO] Current question consumed time: 1407 ms, average: 2678.28 ms/ques, entity detection: 359.44 ms/ques (linking tool: 276.92 ms/ques), relation detection: 906.67 ms/ques (similarity: 719.01 ms/ques), EDG generation: 64.73 ms/ques [INFO] Entity Linking P: 0.74822695035461, R: 0.7092198581560284, micro F1: 0.7210401891252952, macro F1: 0.7282014115373332 [INFO] Relation Linking P: 0.594562647754137, R: 0.574468085106383, micro F1: 0.576950354609929, macro F1: 0.5843426629090205 [INFO] Type Linking P: 0.5283687943262412, R: 0.5283687943262412, micro F1: 0.5283687943262412, macro F1: 0.5283687943262412 @@ -3957,8 +3957,8 @@ Question 282: Was Lucrezia Tornabuoni the mother of Lorenzo de' Medici? [INFO] Golden Answer: [true] [INFO] Predicted Answer: [false] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 283, P: 0.537, R: 0.598, macro F1: 0.545, macro F1*: 0.566 -[INFO] Current time: 1761 ms, average: 2877.40 ms/ques, entity detection: 388.18 ms/ques (linking tool: 296.78 ms/ques), relation detection: 949.36 ms/ques (similarity: 729.69 ms/ques), EDG generation: 65.29 ms/ques +[INFO] Cumulative metrics, sample: 283, P: 0.537, R: 0.598, macro F1: 0.566 +[INFO] Current question consumed time: 1561 ms, average: 2674.33 ms/ques, entity detection: 359.38 ms/ques (linking tool: 276.65 ms/ques), relation detection: 904.97 ms/ques (similarity: 717.39 ms/ques), EDG generation: 64.58 ms/ques [INFO] Entity Linking P: 0.7455830388692579, R: 0.7067137809187279, micro F1: 0.7184923439340398, macro F1: 0.7256282616732437 [INFO] Relation Linking P: 0.5924617196702002, R: 0.5724381625441696, micro F1: 0.5749116607773851, macro F1: 0.5822778478457378 [INFO] Type Linking P: 0.5300353356890459, R: 0.5300353356890459, micro F1: 0.5300353356890459, macro F1: 0.5300353356890459 @@ -3971,8 +3971,8 @@ Question 283: Who are the producers of the movies which has music by Gary Chang [INFO] Golden Answer: [http://dbpedia.org/resource/Steven_Seagal, http://dbpedia.org/resource/Gary_W._Goldstein, http://dbpedia.org/resource/Steven_Reuther, http://dbpedia.org/resource/Arnon_Milchan, *@en, http://dbpedia.org/resource/Frederick_Muller, http://dbpedia.org/resource/Gideon_Amir, http://dbpedia.org/resource/Ilene_Kahn, http://dbpedia.org/resource/Leo_Zisman, William S. Gilmore@en, Marianne Moloney@en, Yoram Globus and Menahem Golan@en, http://dbpedia.org/resource/Edward_R._Pressman, http://dbpedia.org/resource/John_Frankenheimer, http://dbpedia.org/resource/David_W._Rintels, http://dbpedia.org/resource/Lansing_L._Smith, http://dbpedia.org/resource/Ethel_Winant, Diane Batson-Smith@en, http://dbpedia.org/resource/Ned_Tanen, http://dbpedia.org/resource/Keith_Forsey, John Hughes@en, http://dbpedia.org/resource/Wendy_Hill-Tout, http://dbpedia.org/resource/Leslie_Greif, http://dbpedia.org/resource/Don_Johnson, http://dbpedia.org/resource/Lance_H._Robbins, http://dbpedia.org/resource/Howard_Braunstein, http://dbpedia.org/resource/Michael_Jaffe, http://dbpedia.org/resource/Anthony_Santa_Croce] [INFO] Predicted Answer: [http://dbpedia.org/resource/Steven_Seagal, http://dbpedia.org/resource/Gary_W._Goldstein, http://dbpedia.org/resource/Steven_Reuther, http://dbpedia.org/resource/Arnon_Milchan, http://dbpedia.org/resource/Frederick_Muller, http://dbpedia.org/resource/Gideon_Amir, http://dbpedia.org/resource/Ilene_Kahn, http://dbpedia.org/resource/Leo_Zisman, http://dbpedia.org/resource/Menahem_Golan, http://dbpedia.org/resource/Yoram_Globus, http://dbpedia.org/resource/Edward_R._Pressman, http://dbpedia.org/resource/John_Frankenheimer, http://dbpedia.org/resource/David_W._Rintels, http://dbpedia.org/resource/Lansing_L._Smith, http://dbpedia.org/resource/Ethel_Winant, http://dbpedia.org/resource/John_Hughes_(filmmaker), http://dbpedia.org/resource/Ned_Tanen, http://dbpedia.org/resource/Wendy_Hill-Tout, http://dbpedia.org/resource/Leslie_Greif, http://dbpedia.org/resource/Don_Johnson, http://dbpedia.org/resource/Lance_H._Robbins, http://dbpedia.org/resource/Howard_Braunstein, http://dbpedia.org/resource/Michael_Jaffe, http://dbpedia.org/resource/Anthony_Santa_Croce, *@en, William S. Gilmore@en, Marianne Moloney@en, Yoram Globus and Menahem Golan@en, Diane Batson-Smith@en, http://dbpedia.org/resource/Keith_Forsey, John Hughes@en] [INFO] P: 0.9032258064516129, R: 1.0, F: 0.9491525423728813 -[INFO] Cumulative metrics, sample: 284, P: 0.538, R: 0.600, macro F1: 0.547, macro F1*: 0.567 -[INFO] Current time: 1532 ms, average: 2872.67 ms/ques, entity detection: 387.33 ms/ques (linking tool: 296.02 ms/ques), relation detection: 948.29 ms/ques (similarity: 728.73 ms/ques), EDG generation: 65.16 ms/ques +[INFO] Cumulative metrics, sample: 284, P: 0.538, R: 0.600, macro F1: 0.567 +[INFO] Current question consumed time: 1426 ms, average: 2669.93 ms/ques, entity detection: 358.57 ms/ques (linking tool: 275.92 ms/ques), relation detection: 903.91 ms/ques (similarity: 716.46 ms/ques), EDG generation: 64.45 ms/ques [INFO] Entity Linking P: 0.7464788732394366, R: 0.7077464788732394, micro F1: 0.7194835680751172, macro F1: 0.7265968693517034 [INFO] Relation Linking P: 0.5938967136150234, R: 0.573943661971831, micro F1: 0.5764084507042253, macro F1: 0.583749734588435 [INFO] Type Linking P: 0.528169014084507, R: 0.528169014084507, micro F1: 0.528169014084507, macro F1: 0.528169014084507 @@ -3985,8 +3985,8 @@ Question 284: Where was Special 26 recorded ? [INFO] Golden Answer: [http://dbpedia.org/resource/India, http://dbpedia.org/resource/Mumbai, http://dbpedia.org/resource/Hyderabad, http://dbpedia.org/resource/Prasad_Studios, Spectral Harmony@en] [INFO] Predicted Answer: [http://dbpedia.org/resource/India, http://dbpedia.org/resource/Mumbai, http://dbpedia.org/resource/Hyderabad, http://dbpedia.org/resource/Prasad_Studios, Spectral Harmony@en] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 285, P: 0.540, R: 0.601, macro F1: 0.548, macro F1*: 0.569 -[INFO] Current time: 1099 ms, average: 2866.44 ms/ques, entity detection: 386.69 ms/ques (linking tool: 295.43 ms/ques), relation detection: 946.24 ms/ques (similarity: 726.93 ms/ques), EDG generation: 64.95 ms/ques +[INFO] Cumulative metrics, sample: 285, P: 0.540, R: 0.601, macro F1: 0.569 +[INFO] Current question consumed time: 988 ms, average: 2664.03 ms/ques, entity detection: 357.95 ms/ques (linking tool: 275.34 ms/ques), relation detection: 901.90 ms/ques (similarity: 714.71 ms/ques), EDG generation: 64.25 ms/ques [INFO] Entity Linking P: 0.7473684210526316, R: 0.7087719298245614, micro F1: 0.7204678362573097, macro F1: 0.7275586556753328 [INFO] Relation Linking P: 0.5953216374269006, R: 0.5754385964912281, micro F1: 0.5778947368421052, macro F1: 0.585211279948122 [INFO] Type Linking P: 0.5298245614035088, R: 0.5298245614035088, micro F1: 0.5298245614035088, macro F1: 0.5298245614035088 @@ -3999,8 +3999,8 @@ Question 285: How many movies music composers are associated with Demis Roussos [INFO] Golden Answer: [17] [INFO] Predicted Answer: [1] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 286, P: 0.538, R: 0.599, macro F1: 0.546, macro F1*: 0.567 -[INFO] Current time: 977 ms, average: 2859.84 ms/ques, entity detection: 385.91 ms/ques (linking tool: 294.76 ms/ques), relation detection: 943.83 ms/ques (similarity: 725.38 ms/ques), EDG generation: 64.80 ms/ques +[INFO] Cumulative metrics, sample: 286, P: 0.538, R: 0.599, macro F1: 0.567 +[INFO] Current question consumed time: 901 ms, average: 2657.87 ms/ques, entity detection: 357.15 ms/ques (linking tool: 274.66 ms/ques), relation detection: 899.65 ms/ques (similarity: 713.20 ms/ques), EDG generation: 64.10 ms/ques [INFO] Entity Linking P: 0.7482517482517482, R: 0.7097902097902098, micro F1: 0.7214452214452212, macro F1: 0.7285136925424696 [INFO] Relation Linking P: 0.5967365967365967, R: 0.5751748251748252, micro F1: 0.5782051282051281, macro F1: 0.5857573555236409 [INFO] Type Linking P: 0.527972027972028, R: 0.527972027972028, micro F1: 0.527972027972028, macro F1: 0.527972027972028 @@ -4013,8 +4013,8 @@ Question 286: To which nation did David Animle Hanses owe his allegiance? [INFO] Golden Answer: [http://dbpedia.org/resource/Ghana] [INFO] Predicted Answer: [Ghana, http://dbpedia.org/resource/Ghana] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 287, P: 0.539, R: 0.600, macro F1: 0.548, macro F1*: 0.568 -[INFO] Current time: 1309 ms, average: 2854.43 ms/ques, entity detection: 385.15 ms/ques (linking tool: 294.01 ms/ques), relation detection: 941.78 ms/ques (similarity: 724.13 ms/ques), EDG generation: 64.62 ms/ques +[INFO] Cumulative metrics, sample: 287, P: 0.539, R: 0.600, macro F1: 0.568 +[INFO] Current question consumed time: 1192 ms, average: 2652.76 ms/ques, entity detection: 356.41 ms/ques (linking tool: 273.95 ms/ques), relation detection: 897.71 ms/ques (similarity: 711.99 ms/ques), EDG generation: 63.91 ms/ques [INFO] Entity Linking P: 0.7491289198606271, R: 0.710801393728223, micro F1: 0.722415795586527, macro F1: 0.7294620508428065 [INFO] Relation Linking P: 0.5981416957026713, R: 0.5766550522648084, micro F1: 0.5796747967479674, macro F1: 0.5872018821875953 [INFO] Type Linking P: 0.5296167247386759, R: 0.5296167247386759, micro F1: 0.5296167247386759, macro F1: 0.5296167247386759 @@ -4027,8 +4027,8 @@ Question 287: What is the alma mater of the actors of The God Who Wasn't There ? [INFO] Golden Answer: [http://dbpedia.org/resource/Stanford_University, http://dbpedia.org/resource/University_of_California,_Los_Angeles, http://dbpedia.org/resource/Montclair_State_University, http://dbpedia.org/resource/Drew_University, http://dbpedia.org/resource/Gordon–Conwell_Theological_Seminary, http://dbpedia.org/resource/Columbia_University, http://dbpedia.org/resource/University_of_California,_Berkeley] [INFO] Predicted Answer: [http://dbpedia.org/resource/Lionsheart] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 288, P: 0.537, R: 0.598, macro F1: 0.546, macro F1*: 0.566 -[INFO] Current time: 4015 ms, average: 2858.46 ms/ques, entity detection: 384.71 ms/ques (linking tool: 293.58 ms/ques), relation detection: 944.61 ms/ques (similarity: 724.78 ms/ques), EDG generation: 64.52 ms/ques +[INFO] Cumulative metrics, sample: 288, P: 0.537, R: 0.598, macro F1: 0.566 +[INFO] Current question consumed time: 3659 ms, average: 2656.25 ms/ques, entity detection: 355.94 ms/ques (linking tool: 273.54 ms/ques), relation detection: 900.30 ms/ques (similarity: 712.69 ms/ques), EDG generation: 63.81 ms/ques [INFO] Entity Linking P: 0.7465277777777778, R: 0.7083333333333334, micro F1: 0.7199074074074071, macro F1: 0.7269291964996023 [INFO] Relation Linking P: 0.5978009259259259, R: 0.5763888888888888, micro F1: 0.579398148148148, macro F1: 0.5868996769070696 [INFO] Type Linking P: 0.53125, R: 0.53125, micro F1: 0.53125, macro F1: 0.53125 @@ -4041,8 +4041,8 @@ Question 288: List some musicians associated with famous guitar players? [INFO] Golden Answer: [http://dbpedia.org/resource/Barry_White, http://dbpedia.org/resource/Janet_Jackson, http://dbpedia.org/resource/Bette_Midler, http://dbpedia.org/resource/Lalah_Hathaway, http://dbpedia.org/resource/Gerald_Levert, http://dbpedia.org/resource/Troop_(band), http://dbpedia.org/resource/Angela_Winbush, http://dbpedia.org/resource/Baltimora, http://dbpedia.org/resource/Level_42, http://dbpedia.org/resource/Yellowbirds, http://dbpedia.org/resource/Apollo_Sunshine, http://dbpedia.org/resource/The_Mallett_Brothers_Band, http://dbpedia.org/resource/Bana_(singer), http://dbpedia.org/resource/Val_Xalino, http://dbpedia.org/resource/Voz_de_Cabo_Verde, http://dbpedia.org/resource/Veronica_Ballestrini, http://dbpedia.org/resource/Juliet_Simms, http://dbpedia.org/resource/Jeff_Buckley, http://dbpedia.org/resource/Keith_Morris, http://dbpedia.org/resource/Inger_Lorre, http://dbpedia.org/resource/The_Weavers, http://dbpedia.org/resource/Pete_Seeger, http://dbpedia.org/resource/Big_Bill_Broonzy, http://dbpedia.org/resource/Win_Stracke, http://dbpedia.org/resource/The_Pierces, http://dbpedia.org/resource/Rufus_Wainwright, http://dbpedia.org/resource/Ben_Kweller, http://dbpedia.org/resource/Sector7seven, http://dbpedia.org/resource/The_Salads, http://dbpedia.org/resource/Woods_of_Ypres, http://dbpedia.org/resource/Baptized_in_Blood, http://dbpedia.org/resource/Kittie, http://dbpedia.org/resource/Thine_Eyes_Bleed, http://dbpedia.org/resource/Dayna_Manning, http://dbpedia.org/resource/Hong_Faux, http://dbpedia.org/resource/Nutmeg_(band), http://dbpedia.org/resource/The_Genuine_Fakes, http://dbpedia.org/resource/Friska_Viljor, http://dbpedia.org/resource/Karl_Martindahl, http://dbpedia.org/resource/Colbie_Caillat, http://dbpedia.org/resource/Ekhymosis, http://dbpedia.org/resource/Nelly_Furtado, http://dbpedia.org/resource/Laura_Pausini, http://dbpedia.org/resource/Juan_Luis_Guerra, http://dbpedia.org/resource/The_Music_Tapes, http://dbpedia.org/resource/Circulatory_System_(band), http://dbpedia.org/resource/The_Ludlows, http://dbpedia.org/resource/The_Dubliners, http://dbpedia.org/resource/Pete_Kipley, http://dbpedia.org/resource/Evan_Wickham, http://dbpedia.org/resource/The_Zombies, http://dbpedia.org/resource/Soul_Syndicate, http://dbpedia.org/resource/Charley_Patton, http://dbpedia.org/resource/Bassnectar, http://dbpedia.org/resource/Talib_Kweli, http://dbpedia.org/resource/Michal_Menert, http://dbpedia.org/resource/Emancipator_(musician), http://dbpedia.org/resource/Jordin_Sparks, http://dbpedia.org/resource/Kevin_Gilbert_(musician), http://dbpedia.org/resource/Glen_Campbell, http://dbpedia.org/resource/LeAnn_Rimes, http://dbpedia.org/resource/Avril_Lavigne, http://dbpedia.org/resource/Meat_Loaf, http://dbpedia.org/resource/Demi_Lovato, http://dbpedia.org/resource/Mark_Isham, http://dbpedia.org/resource/Henry_Jackman, http://dbpedia.org/resource/Sean_Altman, http://dbpedia.org/resource/David_Wilcox_(American_musician), http://dbpedia.org/resource/Dar_Williams, http://dbpedia.org/resource/Jonatha_Brooke, http://dbpedia.org/resource/Patty_Larkin, http://dbpedia.org/resource/Ellis_Paul, http://dbpedia.org/resource/Christine_Lavin, http://dbpedia.org/resource/Vance_Gilbert, http://dbpedia.org/resource/Jim_Infantino, http://dbpedia.org/resource/Free_Kitten, http://dbpedia.org/resource/Velvet_Monkeys, http://dbpedia.org/resource/Action_Swingers, http://dbpedia.org/resource/Pussy_Galore_(band), http://dbpedia.org/resource/Little_Sister_(band), http://dbpedia.org/resource/Sister_7, http://dbpedia.org/resource/Hector_Zazou, http://dbpedia.org/resource/The_Golden_Palominos, http://dbpedia.org/resource/Bill_Laswell, http://dbpedia.org/resource/Graeme_Revell, http://dbpedia.org/resource/Nazia_Hassan, http://dbpedia.org/resource/Frost_(rapper), http://dbpedia.org/resource/South_Central_Cartel, http://dbpedia.org/resource/UTFO, http://dbpedia.org/resource/Bloods_&_Crips, http://dbpedia.org/resource/World_Class_Wreckin'_Cru, http://dbpedia.org/resource/Darryl_Strawberry, http://dbpedia.org/resource/DJ_U-Neek, http://dbpedia.org/resource/Battlecat_(record_producer), http://dbpedia.org/resource/Bone_Thugs-n-Harmony, http://dbpedia.org/resource/DJ_Quik, http://dbpedia.org/resource/LA_Dream_Team, http://dbpedia.org/resource/Mike_Elliott_(saxophonist), http://dbpedia.org/resource/The_Foundations, http://dbpedia.org/resource/Eric_Allandale, http://dbpedia.org/resource/Clem_Curtis, http://dbpedia.org/resource/Edwin_Starr, http://dbpedia.org/resource/The_Ramong_Sound, http://dbpedia.org/resource/Hue_Montgomery, http://dbpedia.org/resource/Steve_Bingham_(guitarist), http://dbpedia.org/resource/Pluto_(UK_band), http://dbpedia.org/resource/Colin_Young, http://dbpedia.org/resource/Watch_me_Fall, http://dbpedia.org/resource/Insomnium, http://dbpedia.org/resource/Chicago_(band), http://dbpedia.org/resource/Soko_Friedhof, http://dbpedia.org/resource/Untoten_vs._Soko_Friedhof, http://dbpedia.org/resource/Jaime_Guardia, http://dbpedia.org/resource/They_Might_Be_Giants, http://dbpedia.org/resource/Violent_Femmes, http://dbpedia.org/resource/Bexa_Lala_(band), http://dbpedia.org/resource/HagenBaden_235_(band), http://dbpedia.org/resource/Leo_Patett_(band), http://dbpedia.org/resource/Mechaniczna_Pomarańcza_(band), http://dbpedia.org/resource/Caroline's_Spine, http://dbpedia.org/resource/Doug_Cox_(musician), http://dbpedia.org/resource/Don_Johnson, http://dbpedia.org/resource/Rita_Chiarelli, http://dbpedia.org/resource/Suzie_Vinnick, http://dbpedia.org/resource/BT_(musician), http://dbpedia.org/resource/Paul_Oakenfold, http://dbpedia.org/resource/Sonny_Landreth, http://dbpedia.org/resource/Garrison_Starr, http://dbpedia.org/resource/Stephen_Kellogg, http://dbpedia.org/resource/Stephen_Kellogg_and_the_Sixers, http://dbpedia.org/resource/The_Alternate_Routes, http://dbpedia.org/resource/Dave_O'Donnell, http://dbpedia.org/resource/Avial_(band), http://dbpedia.org/resource/Neuronium, http://dbpedia.org/resource/Cherish_Band, http://dbpedia.org/resource/Gato_viejo, http://dbpedia.org/resource/Programa, http://dbpedia.org/resource/Berberism, http://dbpedia.org/resource/Serebro, http://dbpedia.org/resource/Stan_Rogers, http://dbpedia.org/resource/Catherine_MacLellan, http://dbpedia.org/resource/Dry_Bones_(band), http://dbpedia.org/resource/Barney_Bentall, http://dbpedia.org/resource/Mason_Jennings, http://dbpedia.org/resource/Charlotte_Gainsbourg, http://dbpedia.org/resource/The_Strokes, http://dbpedia.org/resource/Bert_Jansch, http://dbpedia.org/resource/Joanna_Newsom, http://dbpedia.org/resource/Devendra_Banhart, http://dbpedia.org/resource/Little_Joy, http://dbpedia.org/resource/Supercluster, http://dbpedia.org/resource/Pylon_(band), http://dbpedia.org/resource/Soundhouses, http://dbpedia.org/resource/Neil_Diamond, http://dbpedia.org/resource/The_Notorious_Cherry_Bombs, http://dbpedia.org/resource/Mark_Knopfler, http://dbpedia.org/resource/Carter_Family, http://dbpedia.org/resource/Loaded_(band), http://dbpedia.org/resource/The_Gentlemen_(band), http://dbpedia.org/resource/The_Presidents_of_the_United_States_of_America_(band), http://dbpedia.org/resource/Elvis_Presley, http://dbpedia.org/resource/Roy_Orbison, http://dbpedia.org/resource/JJ_Cale, http://dbpedia.org/resource/Shakira, http://dbpedia.org/resource/Paloma_Faith, http://dbpedia.org/resource/Draco_Rosa, http://dbpedia.org/resource/Ricky_Martin, http://dbpedia.org/resource/Diego_Bertie, http://dbpedia.org/resource/Anna_Carina, http://dbpedia.org/resource/Stone_Temple_Pilots, http://dbpedia.org/resource/Scott_Weiland, http://dbpedia.org/resource/The_Crystal_Method, http://dbpedia.org/resource/Velvet_Revolver, http://dbpedia.org/resource/Sheryl_Crow, http://dbpedia.org/resource/John_Taylor_(bass_guitarist), http://dbpedia.org/resource/Pete_Tong, http://dbpedia.org/resource/Perry_Farrell, http://dbpedia.org/resource/Thunderpuss, http://dbpedia.org/resource/Pusaka_(DJ_collaboration), http://dbpedia.org/resource/E_Street_Band, http://dbpedia.org/resource/Bruce_Springsteen, http://dbpedia.org/resource/The_Swell_Season, http://dbpedia.org/resource/Mae, http://dbpedia.org/resource/Anjan_Dutt, http://dbpedia.org/resource/Kabir_Suman, http://dbpedia.org/resource/AB_de_Villiers, http://dbpedia.org/resource/The_Last_Shadow_Puppets, http://dbpedia.org/resource/Ipso_Facto_(English_band), http://dbpedia.org/resource/Inti-Illimani, http://dbpedia.org/resource/Quilapayún, http://dbpedia.org/resource/Violeta_Parra, http://dbpedia.org/resource/Víctor_Jara, http://dbpedia.org/resource/Mercedes_Sosa, http://dbpedia.org/resource/Pablo_Neruda, http://dbpedia.org/resource/Nano_Stern, http://dbpedia.org/resource/Pueblo_Nuevo_(band), http://dbpedia.org/resource/Singapore_Sling_(band), http://dbpedia.org/resource/Lady_and_Bird, http://dbpedia.org/resource/Elvin_Jones, http://dbpedia.org/resource/Zbigniew_Seifert, http://dbpedia.org/resource/Paul_Winter_Consort, http://dbpedia.org/resource/Tchaikovsky_Symphony_Orchestra, http://dbpedia.org/resource/Secret_Machines, http://dbpedia.org/resource/Mars_Williams, http://dbpedia.org/resource/The_Drovers, http://dbpedia.org/resource/Tom_Morello, http://dbpedia.org/resource/Ike_Reilly, http://dbpedia.org/resource/Mickey_Petralia, http://dbpedia.org/resource/Motormark, http://dbpedia.org/resource/Pillar_(band), http://dbpedia.org/resource/Jamie_Anderson_(singer), http://dbpedia.org/resource/Kara_Barnard, http://dbpedia.org/resource/Stealin_Horses, http://dbpedia.org/resource/Dread_Zeppelin, http://dbpedia.org/resource/Talha_Bora_Öge, http://dbpedia.org/resource/Ufuk_Akyıldız, http://dbpedia.org/resource/Özcan_Deniz, http://dbpedia.org/resource/Selim_Gülgören, http://dbpedia.org/resource/Joy_Zipper, http://dbpedia.org/resource/Levellers, http://dbpedia.org/resource/Oysterband, http://dbpedia.org/resource/The_Wonder_Stuff, http://dbpedia.org/resource/Roy_Harter, http://dbpedia.org/resource/Janus_(musical_project), http://dbpedia.org/resource/Lacrimosa_(band), http://dbpedia.org/resource/Eurovision_Song_Contest, http://dbpedia.org/resource/Cătălin_Măruță, http://dbpedia.org/resource/Marius_Moga, http://dbpedia.org/resource/Faimoşii, http://dbpedia.org/resource/Ploaia_de_stele, http://dbpedia.org/resource/Divertis, http://dbpedia.org/resource/The_Fray, http://dbpedia.org/resource/Primetime_Show_Band, http://dbpedia.org/resource/Dervish_(band), http://dbpedia.org/resource/Muddy_Waters, http://dbpedia.org/resource/Hubert_Sumlin, http://dbpedia.org/resource/News_(disambiguation), http://dbpedia.org/resource/KAT-TUN, http://dbpedia.org/resource/Arashi, http://dbpedia.org/resource/Super_Junior, http://dbpedia.org/resource/Bobby_Farrell, http://dbpedia.org/resource/Günther_(singer), http://dbpedia.org/resource/Arash, http://dbpedia.org/resource/Tri-Continental, http://dbpedia.org/resource/Tanya_Tagaq, http://dbpedia.org/resource/Great_Uncles_of_the_Revolution, http://dbpedia.org/resource/Zubot_and_Dawson, http://dbpedia.org/resource/Dan_Mangan, http://dbpedia.org/resource/Fond_of_Tigers, http://dbpedia.org/resource/Major_Maker, http://dbpedia.org/resource/Andrew_Howie, http://dbpedia.org/resource/Steve_Lawson_(musician), http://dbpedia.org/resource/Yvonne_Lyon, http://dbpedia.org/resource/Jo_Mango, http://dbpedia.org/resource/Oldsolar, http://dbpedia.org/resource/Rhiannon_Giddens, http://dbpedia.org/resource/Soundgarden, http://dbpedia.org/resource/John_Dragonetti, http://dbpedia.org/resource/Cleopatra_Stratan, http://dbpedia.org/resource/Plastic_Band, http://dbpedia.org/resource/Melvin_Seals, http://dbpedia.org/resource/The_String_Cheese_Incident, http://dbpedia.org/resource/Hanuman, http://dbpedia.org/resource/Nicki_Bluhm, http://dbpedia.org/resource/Tim_Bluhm, http://dbpedia.org/resource/Peter_Rowan, http://dbpedia.org/resource/Bill_Nershi, http://dbpedia.org/resource/Darol_Anger, http://dbpedia.org/resource/Leftover_Salmon, http://dbpedia.org/resource/Lowercase_(band), http://dbpedia.org/resource/The_Folk_Implosion, http://dbpedia.org/resource/Yeah_Yeah_Yeahs, http://dbpedia.org/resource/Grim_Tower, http://dbpedia.org/resource/Imaad_Wasif_(album), http://dbpedia.org/resource/Alaska!, http://dbpedia.org/resource/Ernst_Reijseger, http://dbpedia.org/resource/Sean_Bergin, http://dbpedia.org/resource/Fame_(musical), http://dbpedia.org/resource/Doxy_(band), http://dbpedia.org/resource/Uke_til_U_Puke, http://dbpedia.org/resource/Meek_Mill, http://dbpedia.org/resource/Yo_Gotti, http://dbpedia.org/resource/Gym_Class_Heroes, http://dbpedia.org/resource/Fat_Joe, http://dbpedia.org/resource/Jay_Z, http://dbpedia.org/resource/Charlie_Puth, http://dbpedia.org/resource/The_Game_(rapper), http://dbpedia.org/resource/Lil_Wayne, http://dbpedia.org/resource/Nelly, http://dbpedia.org/resource/Mephisto_Walz, http://dbpedia.org/resource/Christian_Death, http://dbpedia.org/resource/Diva_Destruction, http://dbpedia.org/resource/Ragheb_Alama] [INFO] Predicted Answer: [http://dbpedia.org/resource/John_Bailey_(businessman)] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 289, P: 0.536, R: 0.596, macro F1: 0.544, macro F1*: 0.564 -[INFO] Current time: 1598 ms, average: 2854.10 ms/ques, entity detection: 383.91 ms/ques (linking tool: 292.94 ms/ques), relation detection: 942.56 ms/ques (similarity: 722.97 ms/ques), EDG generation: 64.33 ms/ques +[INFO] Cumulative metrics, sample: 289, P: 0.536, R: 0.596, macro F1: 0.564 +[INFO] Current question consumed time: 1435 ms, average: 2652.03 ms/ques, entity detection: 355.16 ms/ques (linking tool: 272.91 ms/ques), relation detection: 898.30 ms/ques (similarity: 710.92 ms/ques), EDG generation: 63.62 ms/ques [INFO] Entity Linking P: 0.7439446366782007, R: 0.7058823529411765, micro F1: 0.717416378316032, macro F1: 0.7244138705601573 [INFO] Relation Linking P: 0.5957324106113033, R: 0.5743944636678201, micro F1: 0.5773933102652825, macro F1: 0.5848688821772874 [INFO] Type Linking P: 0.532871972318339, R: 0.532871972318339, micro F1: 0.532871972318339, macro F1: 0.532871972318339 @@ -4055,8 +4055,8 @@ Question 289: What are the airlines whose hub airport is operated by the Los Ang [INFO] Golden Answer: [http://dbpedia.org/resource/United_Express, http://dbpedia.org/resource/Alaska_Airlines, http://dbpedia.org/resource/American_Airlines, http://dbpedia.org/resource/Atlas_Air, http://dbpedia.org/resource/Delta_Air_Lines, http://dbpedia.org/resource/United_Airlines, http://dbpedia.org/resource/MasAir, http://dbpedia.org/resource/Pacific_Southwest_Airlines, http://dbpedia.org/resource/SkyWest_Airlines, http://dbpedia.org/resource/Western_Airlines, http://dbpedia.org/resource/American_Eagle_(airline_brand), http://dbpedia.org/resource/Braniff_International_Airways, http://dbpedia.org/resource/Compass_Airlines_(North_America), http://dbpedia.org/resource/Delta_Connection, http://dbpedia.org/resource/Envoy_Air, http://dbpedia.org/resource/UPS_Airlines, http://dbpedia.org/resource/Virgin_America, http://dbpedia.org/resource/West_Air_(United_States), http://dbpedia.org/resource/Wings_West_Airlines, http://dbpedia.org/resource/Ted_(airline)] [INFO] Predicted Answer: [http://dbpedia.org/resource/Los_Angeles_International_Airport, http://dbpedia.org/resource/Van_Nuys_Airport, http://dbpedia.org/resource/Ontario_International_Airport] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 290, P: 0.534, R: 0.594, macro F1: 0.542, macro F1*: 0.562 -[INFO] Current time: 4011 ms, average: 2858.09 ms/ques, entity detection: 390.24 ms/ques (linking tool: 299.27 ms/ques), relation detection: 940.05 ms/ques (similarity: 721.17 ms/ques), EDG generation: 64.20 ms/ques +[INFO] Cumulative metrics, sample: 290, P: 0.534, R: 0.594, macro F1: 0.562 +[INFO] Current question consumed time: 3721 ms, average: 2655.71 ms/ques, entity detection: 361.02 ms/ques (linking tool: 278.80 ms/ques), relation detection: 896.12 ms/ques (similarity: 709.62 ms/ques), EDG generation: 63.49 ms/ques [INFO] Entity Linking P: 0.7448275862068966, R: 0.7068965517241379, micro F1: 0.7183908045977009, macro F1: 0.7253665328855762 [INFO] Relation Linking P: 0.5971264367816091, R: 0.5741379310344827, micro F1: 0.5777011494252873, macro F1: 0.5854065852255422 [INFO] Type Linking P: 0.5310344827586206, R: 0.5310344827586206, micro F1: 0.5310344827586206, macro F1: 0.5310344827586206 @@ -4069,8 +4069,8 @@ Question 290: Which company's cars are assembled in Colombia? [INFO] Golden Answer: [http://dbpedia.org/resource/General_Motors] [INFO] Predicted Answer: [http://dbpedia.org/resource/Synergy_Group] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 291, P: 0.532, R: 0.592, macro F1: 0.540, macro F1*: 0.560 -[INFO] Current time: 6784 ms, average: 2871.58 ms/ques, entity detection: 393.38 ms/ques (linking tool: 302.41 ms/ques), relation detection: 945.05 ms/ques (similarity: 727.02 ms/ques), EDG generation: 64.01 ms/ques +[INFO] Cumulative metrics, sample: 291, P: 0.532, R: 0.592, macro F1: 0.560 +[INFO] Current question consumed time: 6357 ms, average: 2668.43 ms/ques, entity detection: 363.91 ms/ques (linking tool: 281.67 ms/ques), relation detection: 900.87 ms/ques (similarity: 715.49 ms/ques), EDG generation: 63.31 ms/ques [INFO] Entity Linking P: 0.7422680412371134, R: 0.7044673539518901, micro F1: 0.7159221076746848, macro F1: 0.7228738643876875 [INFO] Relation Linking P: 0.5985108820160366, R: 0.5738831615120275, micro F1: 0.5780068728522335, macro F1: 0.5859383524964032 [INFO] Type Linking P: 0.5292096219931272, R: 0.5292096219931272, micro F1: 0.5292096219931272, macro F1: 0.5292096219931272 @@ -4083,8 +4083,8 @@ Question 291: Who directed The Haunted House ? [INFO] Golden Answer: [http://dbpedia.org/resource/Walt_Disney] [INFO] Predicted Answer: [http://dbpedia.org/resource/Walt_Disney] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 292, P: 0.533, R: 0.594, macro F1: 0.542, macro F1*: 0.562 -[INFO] Current time: 691 ms, average: 2864.11 ms/ques, entity detection: 392.49 ms/ques (linking tool: 301.67 ms/ques), relation detection: 942.52 ms/ques (similarity: 724.98 ms/ques), EDG generation: 63.82 ms/ques +[INFO] Cumulative metrics, sample: 292, P: 0.533, R: 0.594, macro F1: 0.562 +[INFO] Current question consumed time: 672 ms, average: 2661.60 ms/ques, entity detection: 363.11 ms/ques (linking tool: 280.98 ms/ques), relation detection: 898.46 ms/ques (similarity: 713.47 ms/ques), EDG generation: 63.12 ms/ques [INFO] Entity Linking P: 0.7431506849315068, R: 0.7054794520547946, micro F1: 0.7168949771689496, macro F1: 0.7238252534084653 [INFO] Relation Linking P: 0.5998858447488584, R: 0.5753424657534246, micro F1: 0.5794520547945204, macro F1: 0.5873578742174349 [INFO] Type Linking P: 0.5308219178082192, R: 0.5308219178082192, micro F1: 0.5308219178082192, macro F1: 0.5308219178082192 @@ -4097,8 +4097,8 @@ Question 292: Count all the different purposes followed by the different NGOs. [INFO] Golden Answer: [344] [INFO] Predicted Answer: [] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 293, P: 0.532, R: 0.592, macro F1: 0.540, macro F1*: 0.560 -[INFO] Current time: 475 ms, average: 2855.96 ms/ques, entity detection: 391.15 ms/ques (linking tool: 300.64 ms/ques), relation detection: 940.21 ms/ques (similarity: 723.10 ms/ques), EDG generation: 63.66 ms/ques +[INFO] Cumulative metrics, sample: 293, P: 0.532, R: 0.592, macro F1: 0.560 +[INFO] Current question consumed time: 460 ms, average: 2654.08 ms/ques, entity detection: 361.87 ms/ques (linking tool: 280.02 ms/ques), relation detection: 896.28 ms/ques (similarity: 711.62 ms/ques), EDG generation: 62.96 ms/ques [INFO] Entity Linking P: 0.7406143344709898, R: 0.7030716723549488, micro F1: 0.7144482366325368, macro F1: 0.7213548600521225 [INFO] Relation Linking P: 0.5978384527872582, R: 0.5733788395904437, micro F1: 0.5774744027303753, macro F1: 0.5853532398344402 [INFO] Type Linking P: 0.5324232081911263, R: 0.5324232081911263, micro F1: 0.5324232081911263, macro F1: 0.5324232081911263 @@ -4111,8 +4111,8 @@ Question 293: The players born in Bellaire, Ohio have played for which team? [INFO] Golden Answer: [http://dbpedia.org/resource/Pittsburgh_Steelers, http://dbpedia.org/resource/Arizona_Cardinals, http://dbpedia.org/resource/Cleveland_Gladiators, http://dbpedia.org/resource/Winnipeg_Blue_Bombers, http://dbpedia.org/resource/Colorado_Crush, http://dbpedia.org/resource/Indiana_Firebirds, http://dbpedia.org/resource/Kansas_City_Command, http://dbpedia.org/resource/Grand_Rapids_Rampage, http://dbpedia.org/resource/Chicago_Bears, http://dbpedia.org/resource/Green_Bay_Packers] [INFO] Predicted Answer: [] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 294, P: 0.530, R: 0.590, macro F1: 0.538, macro F1*: 0.558 -[INFO] Current time: 2235 ms, average: 2853.85 ms/ques, entity detection: 389.82 ms/ques (linking tool: 299.62 ms/ques), relation detection: 937.91 ms/ques (similarity: 721.24 ms/ques), EDG generation: 63.57 ms/ques +[INFO] Cumulative metrics, sample: 294, P: 0.530, R: 0.590, macro F1: 0.558 +[INFO] Current question consumed time: 2093 ms, average: 2652.17 ms/ques, entity detection: 360.64 ms/ques (linking tool: 279.07 ms/ques), relation detection: 894.08 ms/ques (similarity: 709.81 ms/ques), EDG generation: 62.87 ms/ques [INFO] Entity Linking P: 0.7380952380952381, R: 0.7006802721088435, micro F1: 0.7120181405895689, macro F1: 0.7189012720927614 [INFO] Relation Linking P: 0.5958049886621315, R: 0.5714285714285714, micro F1: 0.5755102040816326, macro F1: 0.5833622424200374 [INFO] Type Linking P: 0.5340136054421769, R: 0.5340136054421769, micro F1: 0.5340136054421769, macro F1: 0.5340136054421769 @@ -4125,8 +4125,8 @@ Question 294: What is the common ideology followed by Katyayana and Palang Dharm [INFO] Golden Answer: [http://dbpedia.org/resource/Buddhism] [INFO] Predicted Answer: [http://dbpedia.org/resource/Buddhism] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 295, P: 0.531, R: 0.591, macro F1: 0.540, macro F1*: 0.560 -[INFO] Current time: 6322 ms, average: 2865.60 ms/ques, entity detection: 398.41 ms/ques (linking tool: 308.13 ms/ques), relation detection: 935.52 ms/ques (similarity: 719.30 ms/ques), EDG generation: 63.51 ms/ques +[INFO] Cumulative metrics, sample: 295, P: 0.531, R: 0.591, macro F1: 0.560 +[INFO] Current question consumed time: 5891 ms, average: 2663.15 ms/ques, entity detection: 368.58 ms/ques (linking tool: 286.96 ms/ques), relation detection: 891.81 ms/ques (similarity: 707.92 ms/ques), EDG generation: 62.82 ms/ques [INFO] Entity Linking P: 0.7389830508474576, R: 0.7016949152542373, micro F1: 0.7129943502824856, macro F1: 0.7198564307078764 [INFO] Relation Linking P: 0.5971751412429378, R: 0.5728813559322034, micro F1: 0.5769491525423728, macro F1: 0.584776043670052 [INFO] Type Linking P: 0.535593220338983, R: 0.535593220338983, micro F1: 0.535593220338983, macro F1: 0.535593220338983 @@ -4139,8 +4139,8 @@ Question 295: Count the tenants of MasterCard Centre? [INFO] Golden Answer: [11] [INFO] Predicted Answer: [7] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 296, P: 0.530, R: 0.589, macro F1: 0.538, macro F1*: 0.558 -[INFO] Current time: 624 ms, average: 2858.03 ms/ques, entity detection: 397.64 ms/ques (linking tool: 307.43 ms/ques), relation detection: 932.80 ms/ques (similarity: 717.16 ms/ques), EDG generation: 63.32 ms/ques +[INFO] Cumulative metrics, sample: 296, P: 0.530, R: 0.589, macro F1: 0.558 +[INFO] Current question consumed time: 577 ms, average: 2656.10 ms/ques, entity detection: 367.85 ms/ques (linking tool: 286.32 ms/ques), relation detection: 889.24 ms/ques (similarity: 705.82 ms/ques), EDG generation: 62.64 ms/ques [INFO] Entity Linking P: 0.7398648648648649, R: 0.7027027027027027, micro F1: 0.7139639639639637, macro F1: 0.7208051142477373 [INFO] Relation Linking P: 0.5951576576576576, R: 0.5709459459459459, micro F1: 0.5749999999999998, macro F1: 0.5828004489279235 [INFO] Type Linking P: 0.5371621621621622, R: 0.5371621621621622, micro F1: 0.5371621621621622, macro F1: 0.5371621621621622 @@ -4153,8 +4153,8 @@ Question 296: who are starring in the movies written by Mark Sandrich ? [INFO] Golden Answer: [http://dbpedia.org/resource/Bobby_Clark_(comedian), http://dbpedia.org/resource/Paul_McCullough, http://dbpedia.org/resource/Walter_Catlett, http://dbpedia.org/resource/Phil_Harris, http://dbpedia.org/resource/Edgar_Kennedy, http://dbpedia.org/resource/Edna_May_Oliver, http://dbpedia.org/resource/Betty_Grable, http://dbpedia.org/resource/Wheeler_&_Woolsey] [INFO] Predicted Answer: [http://dbpedia.org/resource/Bobby_Clark_(comedian), http://dbpedia.org/resource/Paul_McCullough, http://dbpedia.org/resource/Walter_Catlett, http://dbpedia.org/resource/Phil_Harris, http://dbpedia.org/resource/Edgar_Kennedy, http://dbpedia.org/resource/Edna_May_Oliver, http://dbpedia.org/resource/Betty_Grable, http://dbpedia.org/resource/Wheeler_&_Woolsey, See below@en, Bobby Clark@en] [INFO] P: 0.8, R: 1.0, F: 0.888888888888889 -[INFO] Cumulative metrics, sample: 297, P: 0.531, R: 0.590, macro F1: 0.539, macro F1*: 0.559 -[INFO] Current time: 1475 ms, average: 2853.37 ms/ques, entity detection: 396.95 ms/ques (linking tool: 306.77 ms/ques), relation detection: 931.47 ms/ques (similarity: 716.25 ms/ques), EDG generation: 63.17 ms/ques +[INFO] Cumulative metrics, sample: 297, P: 0.531, R: 0.590, macro F1: 0.559 +[INFO] Current question consumed time: 1364 ms, average: 2651.75 ms/ques, entity detection: 367.16 ms/ques (linking tool: 285.68 ms/ques), relation detection: 887.93 ms/ques (similarity: 705.14 ms/ques), EDG generation: 62.48 ms/ques [INFO] Entity Linking P: 0.7407407407407407, R: 0.7037037037037037, micro F1: 0.7149270482603813, macro F1: 0.7217473884140551 [INFO] Relation Linking P: 0.5965207631874299, R: 0.5723905723905723, micro F1: 0.5764309764309763, macro F1: 0.5842066043698304 [INFO] Type Linking P: 0.5353535353535354, R: 0.5353535353535354, micro F1: 0.5353535353535354, macro F1: 0.5353535353535354 @@ -4167,8 +4167,8 @@ Question 297: Count the number of religions followed by BJP members. [INFO] Golden Answer: [13] [INFO] Predicted Answer: [3] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 298, P: 0.529, R: 0.588, macro F1: 0.537, macro F1*: 0.557 -[INFO] Current time: 1248 ms, average: 2847.99 ms/ques, entity detection: 395.89 ms/ques (linking tool: 305.87 ms/ques), relation detection: 929.50 ms/ques (similarity: 714.62 ms/ques), EDG generation: 62.99 ms/ques +[INFO] Cumulative metrics, sample: 298, P: 0.529, R: 0.588, macro F1: 0.557 +[INFO] Current question consumed time: 1171 ms, average: 2646.79 ms/ques, entity detection: 366.17 ms/ques (linking tool: 284.83 ms/ques), relation detection: 886.08 ms/ques (similarity: 703.55 ms/ques), EDG generation: 62.32 ms/ques [INFO] Entity Linking P: 0.738255033557047, R: 0.7013422818791947, micro F1: 0.7125279642058163, macro F1: 0.7193254173119945 [INFO] Relation Linking P: 0.5945190156599552, R: 0.5704697986577181, micro F1: 0.5744966442953019, macro F1: 0.5822461795229518 [INFO] Type Linking P: 0.5369127516778524, R: 0.5369127516778524, micro F1: 0.5369127516778524, macro F1: 0.5369127516778524 @@ -4181,8 +4181,8 @@ Question 298: What is the mascot of the rival school of Sullivan Central High? [INFO] Golden Answer: [Patriots@en, http://dbpedia.org/resource/Oakland_Raiders, http://dbpedia.org/resource/Confederate_States_of_America, http://dbpedia.org/resource/Native_Americans_in_the_United_States] [INFO] Predicted Answer: [Patriots, Raiders, Rebels, Indian, Patriots@en, http://dbpedia.org/resource/Oakland_Raiders, http://dbpedia.org/resource/Confederate_States_of_America, http://dbpedia.org/resource/Native_Americans_in_the_United_States] [INFO] P: 0.5714285714285714, R: 1.0, F: 0.7272727272727273 -[INFO] Cumulative metrics, sample: 299, P: 0.529, R: 0.590, macro F1: 0.538, macro F1*: 0.558 -[INFO] Current time: 1680 ms, average: 2844.08 ms/ques, entity detection: 395.38 ms/ques (linking tool: 305.25 ms/ques), relation detection: 928.46 ms/ques (similarity: 713.46 ms/ques), EDG generation: 62.85 ms/ques +[INFO] Cumulative metrics, sample: 299, P: 0.529, R: 0.590, macro F1: 0.558 +[INFO] Current question consumed time: 1596 ms, average: 2643.27 ms/ques, entity detection: 365.73 ms/ques (linking tool: 284.27 ms/ques), relation detection: 885.10 ms/ques (similarity: 702.41 ms/ques), EDG generation: 62.17 ms/ques [INFO] Entity Linking P: 0.7391304347826086, R: 0.7023411371237458, micro F1: 0.7134894091415828, macro F1: 0.7202663169575305 [INFO] Relation Linking P: 0.5958751393534002, R: 0.5719063545150501, micro F1: 0.5759197324414714, macro F1: 0.5836447665647624 [INFO] Type Linking P: 0.5384615384615384, R: 0.5384615384615384, micro F1: 0.5384615384615384, macro F1: 0.5384615384615384 @@ -4195,8 +4195,8 @@ Question 299: What award has been awarded to both Dion Beebe and Charles LeMaire [INFO] Golden Answer: [http://dbpedia.org/resource/Academy_Awards] [INFO] Predicted Answer: [http://dbpedia.org/resource/Academy_Awards] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 300, P: 0.530, R: 0.591, macro F1: 0.539, macro F1*: 0.559 -[INFO] Current time: 1529 ms, average: 2839.70 ms/ques, entity detection: 395.21 ms/ques (linking tool: 304.85 ms/ques), relation detection: 926.43 ms/ques (similarity: 712.24 ms/ques), EDG generation: 62.82 ms/ques +[INFO] Cumulative metrics, sample: 300, P: 0.530, R: 0.591, macro F1: 0.559 +[INFO] Current question consumed time: 1452 ms, average: 2639.30 ms/ques, entity detection: 365.55 ms/ques (linking tool: 283.88 ms/ques), relation detection: 883.18 ms/ques (similarity: 701.22 ms/ques), EDG generation: 62.14 ms/ques [INFO] Entity Linking P: 0.74, R: 0.7033333333333334, micro F1: 0.7144444444444442, macro F1: 0.7212009237875288 [INFO] Relation Linking P: 0.5972222222222222, R: 0.5716666666666667, micro F1: 0.5762222222222221, macro F1: 0.584165082382763 [INFO] Type Linking P: 0.5366666666666666, R: 0.5366666666666666, micro F1: 0.5366666666666666, macro F1: 0.5366666666666666 @@ -4209,8 +4209,8 @@ Question 300: What is the birth name of Putri Raemawasti ? [INFO] Golden Answer: [http://dbpedia.org/resource/Indonesia, http://dbpedia.org/resource/East_Java, Gracia Putri Raemawasti@en] [INFO] Predicted Answer: [Gracia Putri Raemawasti@en, East Java,Indonesia@en, http://dbpedia.org/resource/Indonesia, http://dbpedia.org/resource/East_Java] [INFO] P: 0.75, R: 1.0, F: 0.8571428571428571 -[INFO] Cumulative metrics, sample: 301, P: 0.531, R: 0.592, macro F1: 0.540, macro F1*: 0.560 -[INFO] Current time: 631 ms, average: 2832.36 ms/ques, entity detection: 394.48 ms/ques (linking tool: 304.15 ms/ques), relation detection: 923.86 ms/ques (similarity: 710.16 ms/ques), EDG generation: 62.66 ms/ques +[INFO] Cumulative metrics, sample: 301, P: 0.531, R: 0.592, macro F1: 0.560 +[INFO] Current question consumed time: 594 ms, average: 2632.50 ms/ques, entity detection: 364.83 ms/ques (linking tool: 283.19 ms/ques), relation detection: 880.76 ms/ques (similarity: 699.19 ms/ques), EDG generation: 61.99 ms/ques [INFO] Entity Linking P: 0.7408637873754153, R: 0.7043189368770764, micro F1: 0.7153931339977849, macro F1: 0.7221293007981058 [INFO] Relation Linking P: 0.5985603543743079, R: 0.5730897009966778, micro F1: 0.5776301218161681, macro F1: 0.5855481727574752 [INFO] Type Linking P: 0.5382059800664452, R: 0.5382059800664452, micro F1: 0.5382059800664452, macro F1: 0.5382059800664452 @@ -4223,8 +4223,8 @@ Question 301: What is the party of the politicians whose children is Sanjay Gand [INFO] Golden Answer: [http://dbpedia.org/resource/Indian_National_Congress] [INFO] Predicted Answer: [http://dbpedia.org/resource/Indian_National_Congress, http://dbpedia.org/resource/Bharatiya_Janata_Party] [INFO] P: 0.5, R: 1.0, F: 0.6666666666666666 -[INFO] Cumulative metrics, sample: 302, P: 0.531, R: 0.594, macro F1: 0.541, macro F1*: 0.561 -[INFO] Current time: 2229 ms, average: 2830.36 ms/ques, entity detection: 393.97 ms/ques (linking tool: 303.62 ms/ques), relation detection: 924.26 ms/ques (similarity: 710.37 ms/ques), EDG generation: 62.54 ms/ques +[INFO] Cumulative metrics, sample: 302, P: 0.531, R: 0.594, macro F1: 0.561 +[INFO] Current question consumed time: 2018 ms, average: 2630.47 ms/ques, entity detection: 364.38 ms/ques (linking tool: 282.70 ms/ques), relation detection: 880.95 ms/ques (similarity: 699.42 ms/ques), EDG generation: 61.86 ms/ques [INFO] Entity Linking P: 0.7417218543046358, R: 0.7052980132450332, micro F1: 0.7163355408388519, macro F1: 0.7230515101459378 [INFO] Relation Linking P: 0.5982339955849889, R: 0.5728476821192053, micro F1: 0.5773730684326709, macro F1: 0.5852656808833351 [INFO] Type Linking P: 0.5364238410596026, R: 0.5364238410596026, micro F1: 0.5364238410596026, macro F1: 0.5364238410596026 @@ -4237,8 +4237,8 @@ Question 302: Name the tenants of MAsterCard center wich is also the draft team [INFO] Golden Answer: [http://dbpedia.org/resource/Toronto_Maple_Leafs] [INFO] Predicted Answer: [http://dbpedia.org/resource/Toronto_Maple_Leafs] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 303, P: 0.533, R: 0.595, macro F1: 0.542, macro F1*: 0.562 -[INFO] Current time: 2296 ms, average: 2828.60 ms/ques, entity detection: 394.47 ms/ques (linking tool: 303.47 ms/ques), relation detection: 923.60 ms/ques (similarity: 709.50 ms/ques), EDG generation: 62.96 ms/ques +[INFO] Cumulative metrics, sample: 303, P: 0.533, R: 0.595, macro F1: 0.562 +[INFO] Current question consumed time: 2174 ms, average: 2628.96 ms/ques, entity detection: 364.74 ms/ques (linking tool: 282.48 ms/ques), relation detection: 880.45 ms/ques (similarity: 698.58 ms/ques), EDG generation: 62.29 ms/ques [INFO] Entity Linking P: 0.7425742574257426, R: 0.7046204620462047, micro F1: 0.7161716171617158, macro F1: 0.7230996759881233 [INFO] Relation Linking P: 0.5995599559955995, R: 0.5726072607260726, micro F1: 0.5776677667766775, macro F1: 0.5857737345766486 [INFO] Type Linking P: 0.5379537953795379, R: 0.5379537953795379, micro F1: 0.5379537953795379, macro F1: 0.5379537953795379 @@ -4251,8 +4251,8 @@ Question 303: What is the number of occupation of the people who died in tampa b [INFO] Golden Answer: [41] [INFO] Predicted Answer: [2, 1] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 304, P: 0.531, R: 0.593, macro F1: 0.541, macro F1*: 0.560 -[INFO] Current time: 5580 ms, average: 2837.65 ms/ques, entity detection: 396.34 ms/ques (linking tool: 305.32 ms/ques), relation detection: 924.91 ms/ques (similarity: 711.05 ms/ques), EDG generation: 62.99 ms/ques +[INFO] Cumulative metrics, sample: 304, P: 0.531, R: 0.593, macro F1: 0.560 +[INFO] Current question consumed time: 5180 ms, average: 2637.36 ms/ques, entity detection: 366.45 ms/ques (linking tool: 284.11 ms/ques), relation detection: 881.83 ms/ques (similarity: 700.17 ms/ques), EDG generation: 62.31 ms/ques [INFO] Entity Linking P: 0.7401315789473685, R: 0.7023026315789473, micro F1: 0.713815789473684, macro F1: 0.7207210586328991 [INFO] Relation Linking P: 0.6008771929824561, R: 0.5740131578947368, micro F1: 0.5790570175438594, macro F1: 0.5871380504457596 [INFO] Type Linking P: 0.5394736842105263, R: 0.5394736842105263, micro F1: 0.5394736842105263, macro F1: 0.5394736842105263 @@ -4265,8 +4265,8 @@ Question 304: Who was the United States president who also served as a governor [INFO] Golden Answer: [http://dbpedia.org/resource/William_Henry_Harrison] [INFO] Predicted Answer: [http://dbpedia.org/resource/Vincennes,_Indiana, http://dbpedia.org/resource/Corydon,_Indiana] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 305, P: 0.529, R: 0.591, macro F1: 0.539, macro F1*: 0.558 -[INFO] Current time: 2089 ms, average: 2835.20 ms/ques, entity detection: 396.88 ms/ques (linking tool: 305.73 ms/ques), relation detection: 922.74 ms/ques (similarity: 709.27 ms/ques), EDG generation: 62.88 ms/ques +[INFO] Cumulative metrics, sample: 305, P: 0.529, R: 0.591, macro F1: 0.558 +[INFO] Current question consumed time: 2059 ms, average: 2635.46 ms/ques, entity detection: 367.11 ms/ques (linking tool: 284.63 ms/ques), relation detection: 879.80 ms/ques (similarity: 698.43 ms/ques), EDG generation: 62.20 ms/ques [INFO] Entity Linking P: 0.740983606557377, R: 0.7032786885245902, micro F1: 0.7147540983606555, macro F1: 0.7216389721069574 [INFO] Relation Linking P: 0.5989071038251366, R: 0.5721311475409836, micro F1: 0.577158469945355, macro F1: 0.585213007657413 [INFO] Type Linking P: 0.5409836065573771, R: 0.5409836065573771, micro F1: 0.5409836065573771, macro F1: 0.5409836065573771 @@ -4279,8 +4279,8 @@ Question 305: List the people who were born in Gelnhausen and graduated from Pep [INFO] Golden Answer: [http://dbpedia.org/resource/Tamera_Mowry, http://dbpedia.org/resource/Tia_Mowry] [INFO] Predicted Answer: [http://dbpedia.org/resource/Tamera_Mowry, http://dbpedia.org/resource/Tia_Mowry] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 306, P: 0.531, R: 0.593, macro F1: 0.540, macro F1*: 0.560 -[INFO] Current time: 4775 ms, average: 2841.54 ms/ques, entity detection: 396.87 ms/ques (linking tool: 305.58 ms/ques), relation detection: 924.58 ms/ques (similarity: 712.59 ms/ques), EDG generation: 62.77 ms/ques +[INFO] Cumulative metrics, sample: 306, P: 0.531, R: 0.593, macro F1: 0.560 +[INFO] Current question consumed time: 4388 ms, average: 2641.19 ms/ques, entity detection: 367.04 ms/ques (linking tool: 284.46 ms/ques), relation detection: 881.41 ms/ques (similarity: 701.82 ms/ques), EDG generation: 62.09 ms/ques [INFO] Entity Linking P: 0.7418300653594772, R: 0.704248366013072, micro F1: 0.7156862745098036, macro F1: 0.7225508659207562 [INFO] Relation Linking P: 0.6002178649237473, R: 0.5735294117647058, micro F1: 0.578540305010893, macro F1: 0.5865702197352258 [INFO] Type Linking P: 0.5392156862745098, R: 0.5392156862745098, micro F1: 0.5392156862745098, macro F1: 0.5392156862745098 @@ -4293,8 +4293,8 @@ Question 306: Musical artists who are associated with Ren & Angela were born whe [INFO] Golden Answer: [Philadelphia, Pennsylvania, United States@en, http://dbpedia.org/resource/Missouri, http://dbpedia.org/resource/United_States, http://dbpedia.org/resource/St._Louis] [INFO] Predicted Answer: [Norman Dugas@en] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 307, P: 0.529, R: 0.591, macro F1: 0.539, macro F1*: 0.558 -[INFO] Current time: 740 ms, average: 2834.69 ms/ques, entity detection: 396.13 ms/ques (linking tool: 304.88 ms/ques), relation detection: 922.24 ms/ques (similarity: 710.72 ms/ques), EDG generation: 62.63 ms/ques +[INFO] Cumulative metrics, sample: 307, P: 0.529, R: 0.591, macro F1: 0.558 +[INFO] Current question consumed time: 703 ms, average: 2634.87 ms/ques, entity detection: 366.35 ms/ques (linking tool: 283.78 ms/ques), relation detection: 879.18 ms/ques (similarity: 699.98 ms/ques), EDG generation: 61.95 ms/ques [INFO] Entity Linking P: 0.739413680781759, R: 0.7019543973941368, micro F1: 0.7133550488599346, macro F1: 0.7201972800382781 [INFO] Relation Linking P: 0.5982627578718783, R: 0.5716612377850163, micro F1: 0.5766558089033658, macro F1: 0.584659567553678 [INFO] Type Linking P: 0.5374592833876222, R: 0.5374592833876222, micro F1: 0.5374592833876222, macro F1: 0.5374592833876222 @@ -4307,8 +4307,8 @@ Question 307: What religion do the Wickramabahu Centeral College and Neungin Hig [INFO] Golden Answer: [http://dbpedia.org/resource/Buddhism] [INFO] Predicted Answer: [http://dbpedia.org/resource/Buddhism] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 308, P: 0.531, R: 0.592, macro F1: 0.540, macro F1*: 0.560 -[INFO] Current time: 2102 ms, average: 2832.31 ms/ques, entity detection: 396.57 ms/ques (linking tool: 305.39 ms/ques), relation detection: 920.79 ms/ques (similarity: 710.17 ms/ques), EDG generation: 62.52 ms/ques +[INFO] Cumulative metrics, sample: 308, P: 0.531, R: 0.592, macro F1: 0.560 +[INFO] Current question consumed time: 2018 ms, average: 2632.87 ms/ques, entity detection: 366.72 ms/ques (linking tool: 284.23 ms/ques), relation detection: 877.85 ms/ques (similarity: 699.44 ms/ques), EDG generation: 62.03 ms/ques [INFO] Entity Linking P: 0.7402597402597403, R: 0.702922077922078, micro F1: 0.714285714285714, macro F1: 0.7211079134588695 [INFO] Relation Linking P: 0.5963203463203463, R: 0.5698051948051948, micro F1: 0.5747835497835496, macro F1: 0.5827613222044776 [INFO] Type Linking P: 0.538961038961039, R: 0.538961038961039, micro F1: 0.538961038961039, macro F1: 0.538961038961039 @@ -4321,8 +4321,8 @@ Question 308: Who designed the language integrated query? [INFO] Golden Answer: [http://dbpedia.org/resource/Microsoft] [INFO] Predicted Answer: [http://dbpedia.org/resource/Microsoft] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 309, P: 0.532, R: 0.593, macro F1: 0.542, macro F1*: 0.561 -[INFO] Current time: 463 ms, average: 2824.65 ms/ques, entity detection: 395.52 ms/ques (linking tool: 304.55 ms/ques), relation detection: 918.11 ms/ques (similarity: 708.04 ms/ques), EDG generation: 62.34 ms/ques +[INFO] Cumulative metrics, sample: 309, P: 0.532, R: 0.593, macro F1: 0.561 +[INFO] Current question consumed time: 436 ms, average: 2625.76 ms/ques, entity detection: 365.76 ms/ques (linking tool: 283.45 ms/ques), relation detection: 875.29 ms/ques (similarity: 697.34 ms/ques), EDG generation: 61.85 ms/ques [INFO] Entity Linking P: 0.7411003236245954, R: 0.7038834951456311, micro F1: 0.7152103559870547, macro F1: 0.7220126333184749 [INFO] Relation Linking P: 0.5976267529665588, R: 0.5711974110032363, micro F1: 0.5761596548004313, macro F1: 0.584113273088681 [INFO] Type Linking P: 0.540453074433657, R: 0.540453074433657, micro F1: 0.540453074433657, macro F1: 0.540453074433657 @@ -4335,8 +4335,8 @@ Question 309: Name the debut team of Dan Otero ? [INFO] Golden Answer: [http://dbpedia.org/resource/San_Francisco_Giants] [INFO] Predicted Answer: [http://dbpedia.org/resource/San_Francisco_Giants] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 310, P: 0.534, R: 0.595, macro F1: 0.543, macro F1*: 0.562 -[INFO] Current time: 792 ms, average: 2818.09 ms/ques, entity detection: 394.79 ms/ques (linking tool: 303.84 ms/ques), relation detection: 916.02 ms/ques (similarity: 706.27 ms/ques), EDG generation: 62.18 ms/ques +[INFO] Cumulative metrics, sample: 310, P: 0.534, R: 0.595, macro F1: 0.562 +[INFO] Current question consumed time: 736 ms, average: 2619.66 ms/ques, entity detection: 365.09 ms/ques (linking tool: 282.81 ms/ques), relation detection: 873.28 ms/ques (similarity: 695.60 ms/ques), EDG generation: 61.68 ms/ques [INFO] Entity Linking P: 0.7419354838709677, R: 0.7048387096774194, micro F1: 0.7161290322580642, macro F1: 0.7229114971050454 [INFO] Relation Linking P: 0.5989247311827957, R: 0.5725806451612904, micro F1: 0.5775268817204299, macro F1: 0.5854564834416498 [INFO] Type Linking P: 0.5419354838709678, R: 0.5419354838709678, micro F1: 0.5419354838709678, macro F1: 0.5419354838709678 @@ -4349,8 +4349,8 @@ Question 310: Who started at the pole position in both 1997 canadian grand prix [INFO] Golden Answer: [http://dbpedia.org/resource/Michael_Schumacher] [INFO] Predicted Answer: [http://dbpedia.org/resource/Michael_Schumacher] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 311, P: 0.535, R: 0.596, macro F1: 0.545, macro F1*: 0.564 -[INFO] Current time: 2553 ms, average: 2817.24 ms/ques, entity detection: 394.46 ms/ques (linking tool: 303.41 ms/ques), relation detection: 916.43 ms/ques (similarity: 706.03 ms/ques), EDG generation: 62.23 ms/ques +[INFO] Cumulative metrics, sample: 311, P: 0.535, R: 0.596, macro F1: 0.564 +[INFO] Current question consumed time: 2333 ms, average: 2618.74 ms/ques, entity detection: 364.73 ms/ques (linking tool: 282.39 ms/ques), relation detection: 873.67 ms/ques (similarity: 695.40 ms/ques), EDG generation: 61.73 ms/ques [INFO] Entity Linking P: 0.7427652733118971, R: 0.7041800643086816, micro F1: 0.715969989281886, macro F1: 0.7229581993569132 [INFO] Relation Linking P: 0.60021436227224, R: 0.5739549839228296, micro F1: 0.5788853161843514, macro F1: 0.5867910378763753 [INFO] Type Linking P: 0.5434083601286174, R: 0.5434083601286174, micro F1: 0.5434083601286174, macro F1: 0.5434083601286174 @@ -4363,8 +4363,8 @@ Question 311: Who founded the Not on our Watch NGO? [INFO] Golden Answer: [http://dbpedia.org/resource/Brad_Pitt, http://dbpedia.org/resource/George_Clooney, http://dbpedia.org/resource/David_Pressman, http://dbpedia.org/resource/Don_Cheadle, http://dbpedia.org/resource/Matt_Damon, http://dbpedia.org/resource/Jerry_Weintraub] [INFO] Predicted Answer: [Don Cheadle, George Clooney, Matt Damon, Brad Pitt, David Pressman, and Jerry Weintraub@en] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 312, P: 0.533, R: 0.594, macro F1: 0.543, macro F1*: 0.562 -[INFO] Current time: 802 ms, average: 2810.78 ms/ques, entity detection: 393.63 ms/ques (linking tool: 302.72 ms/ques), relation detection: 914.31 ms/ques (similarity: 704.26 ms/ques), EDG generation: 62.07 ms/ques +[INFO] Cumulative metrics, sample: 312, P: 0.533, R: 0.594, macro F1: 0.562 +[INFO] Current question consumed time: 716 ms, average: 2612.64 ms/ques, entity detection: 363.96 ms/ques (linking tool: 281.75 ms/ques), relation detection: 871.65 ms/ques (similarity: 693.65 ms/ques), EDG generation: 61.56 ms/ques [INFO] Entity Linking P: 0.7435897435897436, R: 0.7051282051282052, micro F1: 0.7168803418803416, macro F1: 0.7238484229634672 [INFO] Relation Linking P: 0.5982905982905983, R: 0.5721153846153846, micro F1: 0.5770299145299144, macro F1: 0.5849102973703612 [INFO] Type Linking P: 0.5448717948717948, R: 0.5448717948717948, micro F1: 0.5448717948717948, macro F1: 0.5448717948717948 @@ -4377,8 +4377,8 @@ Question 312: What is the name of the artist which worked on the subsequent work [INFO] Golden Answer: [http://dbpedia.org/resource/Ella_Fitzgerald] [INFO] Predicted Answer: [http://dbpedia.org/resource/Ella_Fitzgerald] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 313, P: 0.535, R: 0.595, macro F1: 0.544, macro F1*: 0.563 -[INFO] Current time: 1869 ms, average: 2807.77 ms/ques, entity detection: 393.28 ms/ques (linking tool: 302.64 ms/ques), relation detection: 912.75 ms/ques (similarity: 703.26 ms/ques), EDG generation: 62.23 ms/ques +[INFO] Cumulative metrics, sample: 313, P: 0.535, R: 0.595, macro F1: 0.563 +[INFO] Current question consumed time: 1743 ms, average: 2609.87 ms/ques, entity detection: 363.60 ms/ques (linking tool: 281.63 ms/ques), relation detection: 870.17 ms/ques (similarity: 692.66 ms/ques), EDG generation: 61.72 ms/ques [INFO] Entity Linking P: 0.744408945686901, R: 0.7060702875399361, micro F1: 0.7177848775292862, macro F1: 0.7247329383119169 [INFO] Relation Linking P: 0.5995740149094781, R: 0.5734824281150159, micro F1: 0.5783812566560168, macro F1: 0.5862380517998257 [INFO] Type Linking P: 0.5463258785942492, R: 0.5463258785942492, micro F1: 0.5463258785942492, macro F1: 0.5463258785942492 @@ -4391,8 +4391,8 @@ Question 313: Which company owns Sony bank ? [INFO] Golden Answer: [http://dbpedia.org/resource/Sony] [INFO] Predicted Answer: [http://dbpedia.org/resource/Sony] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 314, P: 0.536, R: 0.597, macro F1: 0.546, macro F1*: 0.565 -[INFO] Current time: 1074 ms, average: 2802.25 ms/ques, entity detection: 392.98 ms/ques (linking tool: 302.42 ms/ques), relation detection: 910.58 ms/ques (similarity: 701.81 ms/ques), EDG generation: 62.05 ms/ques +[INFO] Cumulative metrics, sample: 314, P: 0.536, R: 0.597, macro F1: 0.565 +[INFO] Current question consumed time: 1017 ms, average: 2604.79 ms/ques, entity detection: 363.36 ms/ques (linking tool: 281.44 ms/ques), relation detection: 868.11 ms/ques (similarity: 691.21 ms/ques), EDG generation: 61.55 ms/ques [INFO] Entity Linking P: 0.7452229299363057, R: 0.7070063694267515, micro F1: 0.7186836518046706, macro F1: 0.7256118002011397 [INFO] Relation Linking P: 0.5976645435244161, R: 0.571656050955414, micro F1: 0.5765392781316346, macro F1: 0.584371051634858 [INFO] Type Linking P: 0.5445859872611465, R: 0.5445859872611465, micro F1: 0.5445859872611465, macro F1: 0.5445859872611465 @@ -4405,8 +4405,8 @@ Question 314: What state was Cyrus Mann born in, and has the famous Downriver ? [INFO] Golden Answer: [http://dbpedia.org/resource/Michigan] [INFO] Predicted Answer: [http://dbpedia.org/resource/Michigan, Michigan@en] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 315, P: 0.538, R: 0.598, macro F1: 0.547, macro F1*: 0.566 -[INFO] Current time: 1345 ms, average: 2797.62 ms/ques, entity detection: 392.21 ms/ques (linking tool: 301.73 ms/ques), relation detection: 908.65 ms/ques (similarity: 700.71 ms/ques), EDG generation: 61.99 ms/ques +[INFO] Cumulative metrics, sample: 315, P: 0.538, R: 0.598, macro F1: 0.566 +[INFO] Current question consumed time: 1227 ms, average: 2600.42 ms/ques, entity detection: 362.61 ms/ques (linking tool: 280.77 ms/ques), relation detection: 866.25 ms/ques (similarity: 690.14 ms/ques), EDG generation: 61.48 ms/ques [INFO] Entity Linking P: 0.746031746031746, R: 0.7063492063492064, micro F1: 0.7185185185185182, macro F1: 0.7256483649926273 [INFO] Relation Linking P: 0.5989417989417989, R: 0.5714285714285714, micro F1: 0.5768253968253966, macro F1: 0.5848617928183931 [INFO] Type Linking P: 0.546031746031746, R: 0.546031746031746, micro F1: 0.546031746031746, macro F1: 0.546031746031746 @@ -4419,8 +4419,8 @@ Question 315: Who is a religious figure titled superior general? [INFO] Golden Answer: [http://dbpedia.org/resource/Mother_Teresa] [INFO] Predicted Answer: [] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 316, P: 0.536, R: 0.596, macro F1: 0.545, macro F1*: 0.564 -[INFO] Current time: 107 ms, average: 2789.11 ms/ques, entity detection: 390.97 ms/ques (linking tool: 300.78 ms/ques), relation detection: 905.96 ms/ques (similarity: 698.63 ms/ques), EDG generation: 61.83 ms/ques +[INFO] Cumulative metrics, sample: 316, P: 0.536, R: 0.596, macro F1: 0.564 +[INFO] Current question consumed time: 105 ms, average: 2592.52 ms/ques, entity detection: 361.47 ms/ques (linking tool: 279.88 ms/ques), relation detection: 863.69 ms/ques (similarity: 688.09 ms/ques), EDG generation: 61.33 ms/ques [INFO] Entity Linking P: 0.7436708860759493, R: 0.7041139240506329, micro F1: 0.7162447257383963, macro F1: 0.7233520094072076 [INFO] Relation Linking P: 0.5970464135021096, R: 0.569620253164557, micro F1: 0.5749999999999997, macro F1: 0.5830109643601071 [INFO] Type Linking P: 0.5443037974683544, R: 0.5443037974683544, micro F1: 0.5443037974683544, macro F1: 0.5443037974683544 @@ -4436,8 +4436,8 @@ Question 316: List the employments of people who died in Tampa, Florida. * television personality@en, TV presenter/Game show host@en, Intellectual, author, editor, radio and television personality@en, Musician, songwriter@en, Writer, editor, musician@en, musician@en, Musician, bandleader, composer, educator@en, http://dbpedia.org/resource/Clerk, Telecommunications, Horse breeding@en, naval engineering, aircraft executive, corporate troubleshooter@en, Negotiator and educator@en, http://dbpedia.org/resource/Mackenzie's_Raiders, Businessman: Coral Sands Hotel@en, Businessman, citrus farmer, lawyer@en, pilot, aircraft preservationist, founder of the Old Rhinebeck Aerodrome aviation museum in upstate New York@en, Slave and slaveholder@en, journalist@en, President of Milam Dairy Farm, Inc.@en, real estate broker@en, farmer, shoemaker@en, farmer@en, Theater director@en, attorney@en, Fashion designer@en, Radio Engineer, Author and Publisher@en, Violinist, Recording artist, Record company executive@en, Director of Mars, Inc.@en, Founder of Ethel M Chocolates@en, Hairdresser@en, biographer, historian, short story writer, novelist@en, Actor: Sky King@en, Philanthropist@en, Real estate dealer@en, Theatre owner, entrepreneur@en, Businessman, philanthropist@en, Financier@en, Racehorse owner/breeder@en, Businessman:@en, Photographer@en, Real estate developer@en, Journalist, novelist, essayist, poet@en, Missionary Priest@en, well drilling@en, broadcaster, musician@en, businessman@en, auctioneer, teacher, postmaster, banker@en, merchant and banker@en, hardware merchant, undertaker, banker@en, logistics manager@en, pharmacist@en, rancher, realtor, promoter@en, municipal bonds business executive@en, civil engineer@en, Author, screenwriter@en, Railroad executive@en, Gambler, bartender, boxing promoter@en, Political leader, writer@en, Spanish Conquistador and Explorer@en, Poet, critic, essayist, novelist@en, Radio broadcaster and personality, inventor, entrepreneur@en, First Lady of Nicaragua@en, Inventor, yachtsman@en, bassist, songwriter@en, Chrysler Senior Vice President of Design@en, Sporting goods merchant@en, Film director@en, landscape architect@en, lumber merchant@en, citrus farmer@en, American psychologist, politician and radio show host@en, Confectioner@en, http://dbpedia.org/resource/Activism, Editor Baptist Standard@en, Pastor, First Baptist Church of Fort Worth@en, Pastor, Temple Baptist Church in Detroit, Michigan@en, Publisher, The Searchlight Anti-communist lecturer@en, Editor, author, sculptor@en, Documentarian, director, producer@en, Dean of students, Assistant professor for education@en, pharmacist, hardware merchant, banker@en, journalist, poet@en, Heiress@en, Writer, Historian, Philosopher, Professor@en, Pilot@en, Live performer@en, http://dbpedia.org/resource/News_presenter, Actress@en, Real Estate and construction@en, Metallurgist@en, Writer, educator@en, Guitarist@en, Jazz singer@en, general physician, anesthesiologist, dermatologist@en, Restaurateur@en, Engineer and Author@en, cattle rancher; boat captain for passenger, mail and freight@en, painter@en, Script writer, theater designer, theater director@en, Banker, agricultural engineer, lawyer, writer@en, Sailor, boat builder and yacht broker@en, http://dbpedia.org/resource/Virginia_Military_Institute, http://dbpedia.org/resource/Bricklayer, http://dbpedia.org/resource/Secretary_to_the_President_of_the_United_States, Criminal figure, mafia boss, drug trafficker, racketeer@en] [INFO] P: 0.020179372197309416, R: 0.21951219512195122, F: 0.03696098562628337 -[INFO] Cumulative metrics, sample: 317, P: 0.534, R: 0.595, macro F1: 0.544, macro F1*: 0.563 -[INFO] Current time: 10712 ms, average: 2814.10 ms/ques, entity detection: 392.45 ms/ques (linking tool: 302.33 ms/ques), relation detection: 919.08 ms/ques (similarity: 709.79 ms/ques), EDG generation: 61.71 ms/ques +[INFO] Cumulative metrics, sample: 317, P: 0.534, R: 0.595, macro F1: 0.563 +[INFO] Current question consumed time: 9007 ms, average: 2612.76 ms/ques, entity detection: 362.69 ms/ques (linking tool: 281.22 ms/ques), relation detection: 873.07 ms/ques (similarity: 696.15 ms/ques), EDG generation: 61.21 ms/ques [INFO] Entity Linking P: 0.7413249211356467, R: 0.7018927444794952, micro F1: 0.713985278654048, macro F1: 0.7210701418696456 [INFO] Relation Linking P: 0.5983175604626708, R: 0.5709779179810726, micro F1: 0.5763406940063089, macro F1: 0.5843281210993351 [INFO] Type Linking P: 0.5457413249211357, R: 0.5457413249211357, micro F1: 0.5457413249211357, macro F1: 0.5457413249211357 @@ -4450,8 +4450,8 @@ Question 317: What are some famous veterinary physicians known for? [INFO] Golden Answer: [http://dbpedia.org/resource/Bacillus_anthracis, http://dbpedia.org/resource/Blood] [INFO] Predicted Answer: [http://dbpedia.org/resource/Robert_C_Shapcott] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 318, P: 0.533, R: 0.593, macro F1: 0.542, macro F1*: 0.561 -[INFO] Current time: 1074 ms, average: 2808.63 ms/ques, entity detection: 391.83 ms/ques (linking tool: 301.80 ms/ques), relation detection: 916.96 ms/ques (similarity: 707.97 ms/ques), EDG generation: 61.54 ms/ques +[INFO] Cumulative metrics, sample: 318, P: 0.533, R: 0.593, macro F1: 0.561 +[INFO] Current question consumed time: 931 ms, average: 2607.47 ms/ques, entity detection: 362.09 ms/ques (linking tool: 280.69 ms/ques), relation detection: 871.02 ms/ques (similarity: 694.37 ms/ques), EDG generation: 61.04 ms/ques [INFO] Entity Linking P: 0.7421383647798742, R: 0.7028301886792453, micro F1: 0.7148846960167712, macro F1: 0.7219496170981583 [INFO] Relation Linking P: 0.59958071278826, R: 0.5707547169811321, micro F1: 0.5766247379454924, macro F1: 0.5848127149363324 [INFO] Type Linking P: 0.5471698113207547, R: 0.5471698113207547, micro F1: 0.5471698113207547, macro F1: 0.5471698113207547 @@ -4464,8 +4464,8 @@ Question 318: List the movies edited by Julian Doyle in which music was composed [INFO] Golden Answer: [http://dbpedia.org/resource/Monty_Python's_Life_of_Brian] [INFO] Predicted Answer: [http://dbpedia.org/resource/Geoffrey_Barkas] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 319, P: 0.531, R: 0.591, macro F1: 0.540, macro F1*: 0.559 -[INFO] Current time: 2543 ms, average: 2807.80 ms/ques, entity detection: 391.59 ms/ques (linking tool: 301.41 ms/ques), relation detection: 916.82 ms/ques (similarity: 707.63 ms/ques), EDG generation: 61.47 ms/ques +[INFO] Cumulative metrics, sample: 319, P: 0.531, R: 0.591, macro F1: 0.559 +[INFO] Current question consumed time: 2245 ms, average: 2606.33 ms/ques, entity detection: 361.81 ms/ques (linking tool: 280.28 ms/ques), relation detection: 870.84 ms/ques (similarity: 694.11 ms/ques), EDG generation: 60.97 ms/ques [INFO] Entity Linking P: 0.7398119122257053, R: 0.700626959247649, micro F1: 0.7126436781609192, macro F1: 0.7196864521542771 [INFO] Relation Linking P: 0.5992685475444096, R: 0.5705329153605015, micro F1: 0.5763845350052245, macro F1: 0.5845477926918224 [INFO] Type Linking P: 0.5454545454545454, R: 0.5454545454545454, micro F1: 0.5454545454545454, macro F1: 0.5454545454545454 @@ -4478,8 +4478,8 @@ Question 319: Name the children of the person who is the star of The Weathered U [INFO] Golden Answer: [http://dbpedia.org/resource/James_Haven, http://dbpedia.org/resource/Angelina_Jolie, 1, 2] [INFO] Predicted Answer: [http://dbpedia.org/resource/James_Haven, http://dbpedia.org/resource/Angelina_Jolie] [INFO] P: 1.0, R: 0.5, F: 0.6666666666666666 -[INFO] Cumulative metrics, sample: 320, P: 0.533, R: 0.591, macro F1: 0.541, macro F1*: 0.560 -[INFO] Current time: 2102 ms, average: 2805.59 ms/ques, entity detection: 390.85 ms/ques (linking tool: 300.77 ms/ques), relation detection: 916.98 ms/ques (similarity: 707.11 ms/ques), EDG generation: 61.58 ms/ques +[INFO] Cumulative metrics, sample: 320, P: 0.533, R: 0.591, macro F1: 0.560 +[INFO] Current question consumed time: 1947 ms, average: 2604.27 ms/ques, entity detection: 361.05 ms/ques (linking tool: 279.63 ms/ques), relation detection: 870.98 ms/ques (similarity: 693.64 ms/ques), EDG generation: 61.10 ms/ques [INFO] Entity Linking P: 0.740625, R: 0.7015625, micro F1: 0.7135416666666663, macro F1: 0.7205647345612134 [INFO] Relation Linking P: 0.5989583333333333, R: 0.5703125, micro F1: 0.5761458333333331, macro F1: 0.5842845211581291 [INFO] Type Linking P: 0.54375, R: 0.54375, micro F1: 0.54375, macro F1: 0.54375 @@ -4492,8 +4492,8 @@ Question 320: Which owner of Chelsea F.C also owns Evraz ? [INFO] Golden Answer: [http://dbpedia.org/resource/Roman_Abramovich] [INFO] Predicted Answer: [http://dbpedia.org/resource/Roman_Abramovich] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 321, P: 0.534, R: 0.592, macro F1: 0.542, macro F1*: 0.562 -[INFO] Current time: 3331 ms, average: 2807.23 ms/ques, entity detection: 391.10 ms/ques (linking tool: 301.01 ms/ques), relation detection: 918.43 ms/ques (similarity: 707.20 ms/ques), EDG generation: 61.43 ms/ques +[INFO] Cumulative metrics, sample: 321, P: 0.534, R: 0.592, macro F1: 0.562 +[INFO] Current question consumed time: 3060 ms, average: 2605.69 ms/ques, entity detection: 361.21 ms/ques (linking tool: 279.83 ms/ques), relation detection: 872.23 ms/ques (similarity: 693.76 ms/ques), EDG generation: 60.95 ms/ques [INFO] Entity Linking P: 0.7414330218068536, R: 0.7024922118380063, micro F1: 0.7144340602284525, macro F1: 0.7214375249943711 [INFO] Relation Linking P: 0.6002076843198338, R: 0.5700934579439252, micro F1: 0.5764278296988574, macro F1: 0.5847631210143545 [INFO] Type Linking P: 0.5451713395638629, R: 0.5451713395638629, micro F1: 0.5451713395638629, macro F1: 0.5451713395638629 @@ -4506,8 +4506,8 @@ Question 321: Did Umkhonto we Sizwe had Nelson Mandela as a commander? [INFO] Golden Answer: [true] [INFO] Predicted Answer: [true] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 322, P: 0.535, R: 0.593, macro F1: 0.544, macro F1*: 0.563 -[INFO] Current time: 2195 ms, average: 2805.33 ms/ques, entity detection: 391.15 ms/ques (linking tool: 301.04 ms/ques), relation detection: 919.53 ms/ques (similarity: 707.24 ms/ques), EDG generation: 61.28 ms/ques +[INFO] Cumulative metrics, sample: 322, P: 0.535, R: 0.593, macro F1: 0.563 +[INFO] Current question consumed time: 1969 ms, average: 2603.71 ms/ques, entity detection: 361.21 ms/ques (linking tool: 279.82 ms/ques), relation detection: 873.17 ms/ques (similarity: 693.83 ms/ques), EDG generation: 60.80 ms/ques [INFO] Entity Linking P: 0.7422360248447205, R: 0.703416149068323, micro F1: 0.7153209109730846, macro F1: 0.722304874875743 [INFO] Relation Linking P: 0.598343685300207, R: 0.5683229813664596, micro F1: 0.57463768115942, macro F1: 0.5829470864770429 [INFO] Type Linking P: 0.546583850931677, R: 0.546583850931677, micro F1: 0.546583850931677, macro F1: 0.546583850931677 @@ -4520,8 +4520,8 @@ Question 322: Name the office holder with associate as Marcus Calpurnius Bibulus [INFO] Golden Answer: [http://dbpedia.org/resource/Julius_Caesar] [INFO] Predicted Answer: [http://dbpedia.org/resource/Julius_Caesar] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 323, P: 0.537, R: 0.595, macro F1: 0.545, macro F1*: 0.564 -[INFO] Current time: 2505 ms, average: 2804.40 ms/ques, entity detection: 391.32 ms/ques (linking tool: 300.93 ms/ques), relation detection: 919.20 ms/ques (similarity: 707.56 ms/ques), EDG generation: 61.21 ms/ques +[INFO] Cumulative metrics, sample: 323, P: 0.537, R: 0.595, macro F1: 0.564 +[INFO] Current question consumed time: 2308 ms, average: 2602.80 ms/ques, entity detection: 361.34 ms/ques (linking tool: 279.72 ms/ques), relation detection: 872.83 ms/ques (similarity: 694.21 ms/ques), EDG generation: 60.73 ms/ques [INFO] Entity Linking P: 0.7430340557275542, R: 0.7043343653250774, micro F1: 0.7162022703818366, macro F1: 0.7231668349861757 [INFO] Relation Linking P: 0.5995872033023736, R: 0.5696594427244582, micro F1: 0.5759545923632609, macro F1: 0.5842403110730984 [INFO] Type Linking P: 0.544891640866873, R: 0.544891640866873, micro F1: 0.544891640866873, macro F1: 0.544891640866873 @@ -4534,8 +4534,8 @@ Question 323: What are the movies whose screenplay is done by Eric Roth? [INFO] Golden Answer: [http://dbpedia.org/resource/The_Insider_(film), http://dbpedia.org/resource/Forrest_Gump, http://dbpedia.org/resource/The_Curious_Case_of_Benjamin_Button_(film), http://dbpedia.org/resource/Lucky_You_(film), http://dbpedia.org/resource/Ali_(film), http://dbpedia.org/resource/Wolfen_(film), http://dbpedia.org/resource/Munich_(film), http://dbpedia.org/resource/The_Horse_Whisperer_(film), http://dbpedia.org/resource/The_Postman_(film), http://dbpedia.org/resource/Extremely_Loud_and_Incredibly_Close_(film)] [INFO] Predicted Answer: [http://dbpedia.org/resource/The_Insider_(film), http://dbpedia.org/resource/Forrest_Gump, http://dbpedia.org/resource/The_Curious_Case_of_Benjamin_Button_(film), http://dbpedia.org/resource/Lucky_You_(film), http://dbpedia.org/resource/Ali_(film), http://dbpedia.org/resource/Wolfen_(film), http://dbpedia.org/resource/Munich_(film), http://dbpedia.org/resource/The_Horse_Whisperer_(film), http://dbpedia.org/resource/The_Postman_(film), http://dbpedia.org/resource/Extremely_Loud_and_Incredibly_Close_(film)] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 324, P: 0.538, R: 0.596, macro F1: 0.546, macro F1*: 0.566 -[INFO] Current time: 1023 ms, average: 2798.90 ms/ques, entity detection: 390.68 ms/ques (linking tool: 300.28 ms/ques), relation detection: 917.26 ms/ques (similarity: 706.30 ms/ques), EDG generation: 61.08 ms/ques +[INFO] Cumulative metrics, sample: 324, P: 0.538, R: 0.596, macro F1: 0.566 +[INFO] Current question consumed time: 914 ms, average: 2597.59 ms/ques, entity detection: 360.70 ms/ques (linking tool: 279.10 ms/ques), relation detection: 870.97 ms/ques (similarity: 692.98 ms/ques), EDG generation: 60.61 ms/ques [INFO] Entity Linking P: 0.7438271604938271, R: 0.7052469135802469, micro F1: 0.7170781893004112, macro F1: 0.7240234554753547 [INFO] Relation Linking P: 0.6008230452674896, R: 0.5709876543209876, micro F1: 0.5772633744855964, macro F1: 0.585525531384472 [INFO] Type Linking P: 0.5432098765432098, R: 0.5432098765432098, micro F1: 0.5432098765432098, macro F1: 0.5432098765432098 @@ -4548,8 +4548,8 @@ Question 324: What is the total number of writers whose singles are recorded in [INFO] Golden Answer: [754] [INFO] Predicted Answer: [1] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 325, P: 0.537, R: 0.594, macro F1: 0.545, macro F1*: 0.564 -[INFO] Current time: 5082 ms, average: 2805.93 ms/ques, entity detection: 391.62 ms/ques (linking tool: 301.20 ms/ques), relation detection: 919.99 ms/ques (similarity: 709.80 ms/ques), EDG generation: 61.01 ms/ques +[INFO] Cumulative metrics, sample: 325, P: 0.537, R: 0.594, macro F1: 0.564 +[INFO] Current question consumed time: 4734 ms, average: 2604.16 ms/ques, entity detection: 361.51 ms/ques (linking tool: 279.90 ms/ques), relation detection: 873.48 ms/ques (similarity: 696.50 ms/ques), EDG generation: 60.54 ms/ques [INFO] Entity Linking P: 0.7415384615384616, R: 0.703076923076923, micro F1: 0.7148717948717945, macro F1: 0.7217956909969689 [INFO] Relation Linking P: 0.5989743589743589, R: 0.5692307692307692, micro F1: 0.5754871794871792, macro F1: 0.5837239143648274 [INFO] Type Linking P: 0.5415384615384615, R: 0.5415384615384615, micro F1: 0.5415384615384615, macro F1: 0.5415384615384615 @@ -4562,8 +4562,8 @@ Question 325: What religions are followed by members of the societies whose arch [INFO] Golden Answer: [http://dbpedia.org/resource/Balinese_Hinduism, http://dbpedia.org/resource/Kebatinan, http://dbpedia.org/resource/Buddhism, http://dbpedia.org/resource/Catholic_Church, http://dbpedia.org/resource/Confucianism, http://dbpedia.org/resource/Hinduism, http://dbpedia.org/resource/Islam, http://dbpedia.org/resource/Theravada, http://dbpedia.org/resource/Christianity, http://dbpedia.org/resource/Taoism] [INFO] Predicted Answer: [http://dbpedia.org/resource/Buddhism_in_Malaysia, http://dbpedia.org/resource/Balinese_Hinduism, http://dbpedia.org/resource/Kebatinan, http://dbpedia.org/resource/Buddhism, http://dbpedia.org/resource/Catholic_Church, http://dbpedia.org/resource/Confucianism, http://dbpedia.org/resource/Hinduism, http://dbpedia.org/resource/Islam, http://dbpedia.org/resource/Theravada, http://dbpedia.org/resource/Christianity, http://dbpedia.org/resource/Taoism] [INFO] P: 0.9090909090909091, R: 1.0, F: 0.9523809523809523 -[INFO] Cumulative metrics, sample: 326, P: 0.538, R: 0.595, macro F1: 0.546, macro F1*: 0.565 -[INFO] Current time: 2250 ms, average: 2804.22 ms/ques, entity detection: 390.81 ms/ques (linking tool: 300.49 ms/ques), relation detection: 919.75 ms/ques (similarity: 710.24 ms/ques), EDG generation: 60.99 ms/ques +[INFO] Cumulative metrics, sample: 326, P: 0.538, R: 0.595, macro F1: 0.565 +[INFO] Current question consumed time: 2191 ms, average: 2602.89 ms/ques, entity detection: 360.81 ms/ques (linking tool: 279.27 ms/ques), relation detection: 873.32 ms/ques (similarity: 696.98 ms/ques), EDG generation: 60.52 ms/ques [INFO] Entity Linking P: 0.7423312883435583, R: 0.7039877300613497, micro F1: 0.7157464212678933, macro F1: 0.7226512435836548 [INFO] Relation Linking P: 0.600204498977505, R: 0.5705521472392638, micro F1: 0.5767893660531694, macro F1: 0.5850028129772015 [INFO] Type Linking P: 0.5429447852760736, R: 0.5429447852760736, micro F1: 0.5429447852760736, macro F1: 0.5429447852760736 @@ -4576,8 +4576,8 @@ Question 326: Who is the producer of album which has the song Money? [INFO] Golden Answer: [http://dbpedia.org/resource/Pink_Floyd] [INFO] Predicted Answer: [] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 327, P: 0.536, R: 0.593, macro F1: 0.544, macro F1*: 0.563 -[INFO] Current time: 1230 ms, average: 2799.41 ms/ques, entity detection: 390.16 ms/ques (linking tool: 299.88 ms/ques), relation detection: 918.02 ms/ques (similarity: 709.12 ms/ques), EDG generation: 60.87 ms/ques +[INFO] Cumulative metrics, sample: 327, P: 0.536, R: 0.593, macro F1: 0.563 +[INFO] Current question consumed time: 1109 ms, average: 2598.32 ms/ques, entity detection: 360.15 ms/ques (linking tool: 278.68 ms/ques), relation detection: 871.62 ms/ques (similarity: 695.91 ms/ques), EDG generation: 60.40 ms/ques [INFO] Entity Linking P: 0.7400611620795107, R: 0.7018348623853211, micro F1: 0.713557594291539, macro F1: 0.7204413009427262 [INFO] Relation Linking P: 0.5983690112130479, R: 0.5688073394495413, micro F1: 0.5750254841997958, macro F1: 0.5832138135491367 [INFO] Type Linking P: 0.5443425076452599, R: 0.5443425076452599, micro F1: 0.5443425076452599, macro F1: 0.5443425076452599 @@ -4590,8 +4590,8 @@ Question 327: List the Sci-fi TV shows with theme music given by Ron Grainer? [INFO] Golden Answer: [http://dbpedia.org/resource/Doctor_Who] [INFO] Predicted Answer: [http://dbpedia.org/resource/Bob_Hund] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 328, P: 0.535, R: 0.592, macro F1: 0.543, macro F1*: 0.562 -[INFO] Current time: 2798 ms, average: 2799.40 ms/ques, entity detection: 391.99 ms/ques (linking tool: 301.61 ms/ques), relation detection: 916.97 ms/ques (similarity: 708.00 ms/ques), EDG generation: 60.75 ms/ques +[INFO] Cumulative metrics, sample: 328, P: 0.535, R: 0.592, macro F1: 0.562 +[INFO] Current question consumed time: 2565 ms, average: 2598.22 ms/ques, entity detection: 361.84 ms/ques (linking tool: 280.30 ms/ques), relation detection: 870.60 ms/ques (similarity: 694.81 ms/ques), EDG generation: 60.27 ms/ques [INFO] Entity Linking P: 0.7378048780487805, R: 0.6996951219512195, micro F1: 0.7113821138211379, macro F1: 0.7182448335618032 [INFO] Relation Linking P: 0.5965447154471545, R: 0.5670731707317073, micro F1: 0.573272357723577, macro F1: 0.5814357226541697 [INFO] Type Linking P: 0.5426829268292683, R: 0.5426829268292683, micro F1: 0.5426829268292683, macro F1: 0.5426829268292683 @@ -4604,8 +4604,8 @@ Question 328: What is the company to which Fusajiro Yamauchi proprietor to ? [INFO] Golden Answer: [http://dbpedia.org/resource/Nintendo] [INFO] Predicted Answer: [http://dbpedia.org/resource/Kyoto, Kyoto@en] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 329, P: 0.533, R: 0.590, macro F1: 0.541, macro F1*: 0.560 -[INFO] Current time: 1805 ms, average: 2796.38 ms/ques, entity detection: 391.10 ms/ques (linking tool: 300.97 ms/ques), relation detection: 916.78 ms/ques (similarity: 707.42 ms/ques), EDG generation: 60.63 ms/ques +[INFO] Cumulative metrics, sample: 329, P: 0.533, R: 0.590, macro F1: 0.560 +[INFO] Current question consumed time: 1662 ms, average: 2595.38 ms/ques, entity detection: 361.01 ms/ques (linking tool: 279.69 ms/ques), relation detection: 870.36 ms/ques (similarity: 694.29 ms/ques), EDG generation: 60.16 ms/ques [INFO] Entity Linking P: 0.7386018237082067, R: 0.7006079027355623, micro F1: 0.7122593718338396, macro F1: 0.7191033595131642 [INFO] Relation Linking P: 0.5962512664640324, R: 0.5683890577507599, micro F1: 0.5735562310030392, macro F1: 0.5819868820988642 [INFO] Type Linking P: 0.541033434650456, R: 0.541033434650456, micro F1: 0.541033434650456, macro F1: 0.541033434650456 @@ -4618,8 +4618,8 @@ Question 329: Who is the writer of mark twain Tonight ? [INFO] Golden Answer: [http://dbpedia.org/resource/Mark_Twain, http://dbpedia.org/resource/Hal_Holbrook] [INFO] Predicted Answer: [http://dbpedia.org/resource/Mark_Twain, http://dbpedia.org/resource/Hal_Holbrook] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 330, P: 0.534, R: 0.591, macro F1: 0.542, macro F1*: 0.561 -[INFO] Current time: 679 ms, average: 2789.96 ms/ques, entity detection: 390.56 ms/ques (linking tool: 300.52 ms/ques), relation detection: 914.40 ms/ques (similarity: 705.54 ms/ques), EDG generation: 60.47 ms/ques +[INFO] Cumulative metrics, sample: 330, P: 0.534, R: 0.591, macro F1: 0.561 +[INFO] Current question consumed time: 619 ms, average: 2589.39 ms/ques, entity detection: 360.48 ms/ques (linking tool: 279.26 ms/ques), relation detection: 868.13 ms/ques (similarity: 692.45 ms/ques), EDG generation: 60.01 ms/ques [INFO] Entity Linking P: 0.7393939393939394, R: 0.7015151515151515, micro F1: 0.7131313131313128, macro F1: 0.7199566644361598 [INFO] Relation Linking P: 0.5974747474747475, R: 0.5696969696969697, micro F1: 0.5748484848484846, macro F1: 0.5832553138481308 [INFO] Type Linking P: 0.5424242424242425, R: 0.5424242424242425, micro F1: 0.5424242424242425, macro F1: 0.5424242424242425 @@ -4632,8 +4632,8 @@ Question 330: How many members are there in the European Go Federation? [INFO] Golden Answer: [37] [INFO] Predicted Answer: [1, 37] [INFO] P: 0.5, R: 1.0, F: 0.6666666666666666 -[INFO] Cumulative metrics, sample: 331, P: 0.534, R: 0.592, macro F1: 0.543, macro F1*: 0.562 -[INFO] Current time: 828 ms, average: 2784.04 ms/ques, entity detection: 389.52 ms/ques (linking tool: 299.73 ms/ques), relation detection: 912.37 ms/ques (similarity: 704.10 ms/ques), EDG generation: 60.33 ms/ques +[INFO] Cumulative metrics, sample: 331, P: 0.534, R: 0.592, macro F1: 0.562 +[INFO] Current question consumed time: 839 ms, average: 2584.10 ms/ques, entity detection: 359.56 ms/ques (linking tool: 278.56 ms/ques), relation detection: 866.27 ms/ques (similarity: 691.07 ms/ques), EDG generation: 59.86 ms/ques [INFO] Entity Linking P: 0.7401812688821753, R: 0.702416918429003, micro F1: 0.7139979859013088, macro F1: 0.7208047958747884 [INFO] Relation Linking P: 0.5986908358509566, R: 0.5709969788519638, micro F1: 0.5761329305135948, macro F1: 0.5845160635858673 [INFO] Type Linking P: 0.5438066465256798, R: 0.5438066465256798, micro F1: 0.5438066465256798, macro F1: 0.5438066465256798 @@ -4646,8 +4646,8 @@ Question 331: Was Kevin Rudd the PM of Julia Gillard? [INFO] Golden Answer: [true] [INFO] Predicted Answer: [true] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 332, P: 0.536, R: 0.594, macro F1: 0.544, macro F1*: 0.563 -[INFO] Current time: 1375 ms, average: 2779.79 ms/ques, entity detection: 389.53 ms/ques (linking tool: 299.70 ms/ques), relation detection: 911.08 ms/ques (similarity: 702.83 ms/ques), EDG generation: 60.19 ms/ques +[INFO] Cumulative metrics, sample: 332, P: 0.536, R: 0.594, macro F1: 0.563 +[INFO] Current question consumed time: 1280 ms, average: 2580.17 ms/ques, entity detection: 359.53 ms/ques (linking tool: 278.51 ms/ques), relation detection: 865.10 ms/ques (similarity: 689.83 ms/ques), EDG generation: 59.72 ms/ques [INFO] Entity Linking P: 0.7409638554216867, R: 0.7033132530120482, micro F1: 0.7148594377510037, macro F1: 0.7216478007965124 [INFO] Relation Linking P: 0.5998995983935743, R: 0.572289156626506, micro F1: 0.5774096385542166, macro F1: 0.5857692010010062 [INFO] Type Linking P: 0.5451807228915663, R: 0.5451807228915663, micro F1: 0.5451807228915663, macro F1: 0.5451807228915663 @@ -4660,8 +4660,8 @@ Question 332: Daniel L D Granger and Job Durfee have based their political caree [INFO] Golden Answer: [http://dbpedia.org/resource/Rhode_Island, Rhode Island@en] [INFO] Predicted Answer: [http://dbpedia.org/resource/Rhode_Island, Rhode Island@en] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 333, P: 0.537, R: 0.595, macro F1: 0.546, macro F1*: 0.564 -[INFO] Current time: 13333 ms, average: 2811.48 ms/ques, entity detection: 420.79 ms/ques (linking tool: 330.79 ms/ques), relation detection: 910.92 ms/ques (similarity: 703.71 ms/ques), EDG generation: 60.22 ms/ques +[INFO] Cumulative metrics, sample: 333, P: 0.537, R: 0.595, macro F1: 0.564 +[INFO] Current question consumed time: 11719 ms, average: 2607.62 ms/ques, entity detection: 386.52 ms/ques (linking tool: 305.39 ms/ques), relation detection: 864.83 ms/ques (similarity: 690.67 ms/ques), EDG generation: 59.94 ms/ques [INFO] Entity Linking P: 0.7417417417417418, R: 0.7042042042042042, micro F1: 0.7157157157157154, macro F1: 0.7224857256009903 [INFO] Relation Linking P: 0.601101101101101, R: 0.5735735735735735, micro F1: 0.5786786786786784, macro F1: 0.5870147949398056 [INFO] Type Linking P: 0.5465465465465466, R: 0.5465465465465466, micro F1: 0.5465465465465466, macro F1: 0.5465465465465466 @@ -4674,8 +4674,8 @@ Question 333: What is the mascot of the handball teams in the Turkish Handball S [INFO] Golden Answer: [http://dbpedia.org/resource/Lion] [INFO] Predicted Answer: [] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 334, P: 0.535, R: 0.593, macro F1: 0.544, macro F1*: 0.563 -[INFO] Current time: 1555 ms, average: 2807.72 ms/ques, entity detection: 420.45 ms/ques (linking tool: 330.39 ms/ques), relation detection: 909.05 ms/ques (similarity: 701.96 ms/ques), EDG generation: 60.32 ms/ques +[INFO] Cumulative metrics, sample: 334, P: 0.535, R: 0.593, macro F1: 0.563 +[INFO] Current question consumed time: 1417 ms, average: 2604.05 ms/ques, entity detection: 386.19 ms/ques (linking tool: 305.02 ms/ques), relation detection: 863.05 ms/ques (similarity: 688.96 ms/ques), EDG generation: 60.03 ms/ques [INFO] Entity Linking P: 0.7395209580838323, R: 0.7020958083832335, micro F1: 0.7135728542914168, macro F1: 0.7203225946860174 [INFO] Relation Linking P: 0.5993013972055888, R: 0.5718562874251497, micro F1: 0.5769461077844309, macro F1: 0.5852572656136386 [INFO] Type Linking P: 0.5449101796407185, R: 0.5449101796407185, micro F1: 0.5449101796407185, macro F1: 0.5449101796407185 @@ -4688,8 +4688,8 @@ Question 334: What city was recorded in Live in detroit which is also the offici [INFO] Golden Answer: [http://dbpedia.org/resource/Detroit, http://dbpedia.org/resource/Michigan] [INFO] Predicted Answer: [http://dbpedia.org/resource/Detroit] [INFO] P: 1.0, R: 0.5, F: 0.6666666666666666 -[INFO] Cumulative metrics, sample: 335, P: 0.537, R: 0.593, macro F1: 0.544, macro F1*: 0.563 -[INFO] Current time: 11055 ms, average: 2832.34 ms/ques, entity detection: 424.81 ms/ques (linking tool: 334.28 ms/ques), relation detection: 916.66 ms/ques (similarity: 710.58 ms/ques), EDG generation: 60.54 ms/ques +[INFO] Cumulative metrics, sample: 335, P: 0.537, R: 0.593, macro F1: 0.563 +[INFO] Current question consumed time: 10110 ms, average: 2626.46 ms/ques, entity detection: 390.15 ms/ques (linking tool: 308.60 ms/ques), relation detection: 870.19 ms/ques (similarity: 697.67 ms/ques), EDG generation: 60.28 ms/ques [INFO] Entity Linking P: 0.7388059701492538, R: 0.7014925373134329, micro F1: 0.7129353233830843, macro F1: 0.7196659191091177 [INFO] Relation Linking P: 0.5990049751243781, R: 0.5716417910447761, micro F1: 0.5767164179104475, macro F1: 0.5850035838656272 [INFO] Type Linking P: 0.5432835820895522, R: 0.5432835820895522, micro F1: 0.5432835820895522, macro F1: 0.5432835820895522 @@ -4702,8 +4702,8 @@ Question 335: Marika Gombitov makes songs in which languages? [INFO] Golden Answer: [http://dbpedia.org/resource/Slovakia] [INFO] Predicted Answer: [http://dbpedia.org/resource/Paradiso_(Marika_Gombitová_song), http://dbpedia.org/resource/Vyznanie, http://dbpedia.org/resource/Cirkusový_kôň, http://dbpedia.org/resource/Domy_na_zbúranie, http://dbpedia.org/resource/Kufor_a_šál, http://dbpedia.org/resource/Srdcia_dievčat, http://dbpedia.org/resource/Svet_stromov, http://dbpedia.org/resource/Záhradná_kaviareň_(song), http://dbpedia.org/resource/Nenápadná, http://dbpedia.org/resource/Tváre_pred_zrkadlom, http://dbpedia.org/resource/Úsmev, http://dbpedia.org/resource/Boľavé_námestie, http://dbpedia.org/resource/Chcem_sa_s_tebou_deliť, http://dbpedia.org/resource/Nespáľme_to_krásne_v_nás, http://dbpedia.org/resource/Študentská_láska] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 336, P: 0.535, R: 0.591, macro F1: 0.543, macro F1*: 0.562 -[INFO] Current time: 1524 ms, average: 2828.45 ms/ques, entity detection: 424.18 ms/ques (linking tool: 333.57 ms/ques), relation detection: 915.38 ms/ques (similarity: 710.05 ms/ques), EDG generation: 60.40 ms/ques +[INFO] Cumulative metrics, sample: 336, P: 0.535, R: 0.591, macro F1: 0.562 +[INFO] Current question consumed time: 1399 ms, average: 2622.81 ms/ques, entity detection: 389.52 ms/ques (linking tool: 307.93 ms/ques), relation detection: 868.99 ms/ques (similarity: 697.19 ms/ques), EDG generation: 60.14 ms/ques [INFO] Entity Linking P: 0.7395833333333334, R: 0.7023809523809523, micro F1: 0.7137896825396822, macro F1: 0.7205022359821123 [INFO] Relation Linking P: 0.5972222222222222, R: 0.5699404761904762, micro F1: 0.5749999999999997, macro F1: 0.583262501770789 [INFO] Type Linking P: 0.5446428571428571, R: 0.5446428571428571, micro F1: 0.5446428571428571, macro F1: 0.5446428571428571 @@ -4716,8 +4716,8 @@ Question 336: What are some musical artists associated with the ones signed up w [INFO] Golden Answer: [http://dbpedia.org/resource/Hankus_Netsky, http://dbpedia.org/resource/M-Base_Collective, http://dbpedia.org/resource/Us3, http://dbpedia.org/resource/Coldcut, http://dbpedia.org/resource/Bugz_in_the_Attic, http://dbpedia.org/resource/Cabaret_Voltaire_(band), http://dbpedia.org/resource/Hey,_Rube!_(band), http://dbpedia.org/resource/William_Orbit, http://dbpedia.org/resource/The_Chemical_Brothers, http://dbpedia.org/resource/David_Bowie, http://dbpedia.org/resource/Placebo_(band), http://dbpedia.org/resource/Michael_Stipe, http://dbpedia.org/resource/Herbs_(band), http://dbpedia.org/resource/Mississippi_(band), http://dbpedia.org/resource/Little_River_Band, http://dbpedia.org/resource/Be-Bop_Deluxe, http://dbpedia.org/resource/Chain_(band), http://dbpedia.org/resource/Healing_Force_(band), http://dbpedia.org/resource/Friends_(Australian_band), http://dbpedia.org/resource/Tandoori_Cassette, http://dbpedia.org/resource/Eddie_Hardin, http://dbpedia.org/resource/Colin_Blunstone, http://dbpedia.org/resource/Linda_McCartney, http://dbpedia.org/resource/Paul_McCartney_and_Wings, http://dbpedia.org/resource/The_Moody_Blues, http://dbpedia.org/resource/Paul_McCartney, http://dbpedia.org/resource/Mike_Pinder, http://dbpedia.org/resource/Maggie_Bell, http://dbpedia.org/resource/Ginger_Baker's_Air_Force, http://dbpedia.org/resource/World_Classic_Rockers, http://dbpedia.org/resource/Baltimora, http://dbpedia.org/resource/The_Darkness_(band), http://dbpedia.org/resource/Queen_(band), http://dbpedia.org/resource/The_Cars, http://dbpedia.org/resource/The_Smashing_Pumpkins, http://dbpedia.org/resource/The_Moors_Murderers, http://dbpedia.org/resource/Strange_Cruise, http://dbpedia.org/resource/Visage_(band), http://dbpedia.org/resource/Blitz_Kids, http://dbpedia.org/resource/The_Photons, http://dbpedia.org/resource/Mystikal, http://dbpedia.org/resource/C-Murder, http://dbpedia.org/resource/504_Boyz, http://dbpedia.org/resource/Master_P, http://dbpedia.org/resource/Romeo_Miller, http://dbpedia.org/resource/Snoop_Dogg, http://dbpedia.org/resource/Montell_Jordan, http://dbpedia.org/resource/TRU_(band), http://dbpedia.org/resource/Mýa, http://shockerworld.net/, http://dbpedia.org/resource/Iron_Maiden, http://dbpedia.org/resource/Lauren_Harris, http://dbpedia.org/resource/Shahrdad_Rohani, http://dbpedia.org/resource/Chameleon_(American_band), http://dbpedia.org/resource/Hudson_Ford, http://dbpedia.org/resource/Strawbs, http://dbpedia.org/resource/The_Monks_(UK_band), http://dbpedia.org/resource/Elmer_Gantry's_Velvet_Opera, http://dbpedia.org/resource/Vanda_&_Young, http://dbpedia.org/resource/Fieldy's_Dreams, http://dbpedia.org/resource/Korn, http://dbpedia.org/resource/L.A.P.D._(band), http://dbpedia.org/resource/StillWell, http://dbpedia.org/resource/Sean_Paul, http://dbpedia.org/resource/Faydee, http://dbpedia.org/resource/Tamer_Hosny, http://dbpedia.org/resource/Mohombi, http://dbpedia.org/resource/Major_Lazer, http://dbpedia.org/resource/Rayvon, http://dbpedia.org/resource/Lewis_Nash, http://dbpedia.org/resource/Out_of_the_Blue_(American_band), http://dbpedia.org/resource/Yoko_Kanno, http://dbpedia.org/resource/The_Blue_Note_7, http://dbpedia.org/resource/Chick_Corea, http://dbpedia.org/resource/Al_Bano_and_Romina_Power, http://dbpedia.org/resource/Eurovision_Song_Contest, http://dbpedia.org/resource/Sanremo_Music_Festival, http://dbpedia.org/resource/Martyn_Bates, http://dbpedia.org/resource/Angel_Haze, http://dbpedia.org/resource/Depeche_Mode, http://dbpedia.org/resource/John_Foxx, http://dbpedia.org/resource/Benny_Benassi, http://dbpedia.org/resource/Eyeless_in_Gaza_(band), http://dbpedia.org/resource/Good_Charlotte, http://dbpedia.org/resource/Rad_Omen, http://dbpedia.org/resource/The_Madden_Brothers, http://dbpedia.org/resource/Birtles_&_Goble, http://dbpedia.org/resource/Travis_Wellington_Hedge, http://dbpedia.org/resource/Birtles_Shorrock_Goble, http://dbpedia.org/resource/Luxuria_(band), http://dbpedia.org/resource/ShelleyDevoto, http://dbpedia.org/resource/Buzzcocks, http://dbpedia.org/resource/Magazine_(band), http://dbpedia.org/resource/Mansun, http://dbpedia.org/resource/Mike_Oldfield, http://dbpedia.org/resource/Cado_Belle, http://dbpedia.org/resource/Malú, http://dbpedia.org/resource/Estrella_Morente, http://dbpedia.org/resource/Jesse_&_Joy, http://dbpedia.org/resource/Miguel_Bosé, http://dbpedia.org/resource/Carminho, http://dbpedia.org/resource/Dwight_Twilley, http://dbpedia.org/resource/Tom_Petty_and_the_Heartbreakers, http://dbpedia.org/resource/20/20_(band), http://dbpedia.org/resource/Cyndi_Lauper, http://dbpedia.org/resource/Lykke_Li, http://dbpedia.org/resource/Teddybears, http://dbpedia.org/resource/Robyn, http://dbpedia.org/resource/Junior_Reid, http://dbpedia.org/resource/Ninjaman, http://dbpedia.org/resource/Cool_&_Dre, http://dbpedia.org/resource/Fat_Joe, http://dbpedia.org/resource/Yami_Bolo, http://dbpedia.org/resource/Buju_Banton, http://dbpedia.org/resource/MC_Shan, http://dbpedia.org/resource/Michael_Rose_(singer), http://dbpedia.org/resource/Dick_Curless, http://dbpedia.org/resource/Nick_Harper, http://dbpedia.org/resource/New_Model_Army_(band), http://dbpedia.org/resource/The_Damned_(band), http://dbpedia.org/resource/Young_Tuxedo_Brass_Band, http://dbpedia.org/resource/Flag_of_Convenience_(band), http://dbpedia.org/resource/Basement_Birds, http://dbpedia.org/resource/Jebediah, http://dbpedia.org/resource/Electric_Sun, http://dbpedia.org/resource/Uli_Jon_Roth, http://dbpedia.org/resource/Rick_Rubin, http://dbpedia.org/resource/Jeanie_Lambe, http://dbpedia.org/resource/Dizzy_Mizz_Lizzy, http://dbpedia.org/resource/Tim_Christensen, http://dbpedia.org/resource/Footlights, http://dbpedia.org/resource/Clive_James, http://dbpedia.org/resource/Jethro_Tull_(band), http://dbpedia.org/resource/Gary_Moore, http://dbpedia.org/resource/Rick_Wakeman, http://dbpedia.org/resource/Martin_Barre, http://dbpedia.org/resource/Archive_(band), http://dbpedia.org/resource/Tomasz_Stańko, http://dbpedia.org/resource/Hans_Koller, http://dbpedia.org/resource/De_Toppers, http://dbpedia.org/resource/Toni_Willé, http://dbpedia.org/resource/Pussycat_(band), http://dbpedia.org/resource/Sweet_Reaction, http://dbpedia.org/resource/The_BGs_from_Holland, http://dbpedia.org/resource/Zingende_Zusjes_(Singing_Sisters), http://dbpedia.org/resource/Frankie_Knuckles, http://dbpedia.org/resource/Tony_Humphries, http://dbpedia.org/resource/Aerosmith, http://dbpedia.org/resource/Jordin_Sparks, http://dbpedia.org/resource/T.I., http://dbpedia.org/resource/Terence_Trent_D'Arby, http://dbpedia.org/resource/Hardware_(band), http://dbpedia.org/resource/T._M._Stevens, http://dbpedia.org/resource/Mick_Jagger, http://dbpedia.org/resource/The_Rolling_Stones, http://dbpedia.org/resource/B'z, http://dbpedia.org/resource/Bernard_Fowler, http://dbpedia.org/resource/Richie_Kotzen, http://dbpedia.org/resource/Colorcode, http://dbpedia.org/resource/Heavy_Glow, http://dbpedia.org/resource/Glenn_Hughes, http://dbpedia.org/resource/Rod_Stewart, http://dbpedia.org/resource/Was_(Not_Was), http://dbpedia.org/resource/Sass_Jordan, http://dbpedia.org/resource/George_Clinton_(musician), http://dbpedia.org/resource/Ronnie_Wood, http://dbpedia.org/resource/Daughtry_(band), http://dbpedia.org/resource/Mass_Mental, http://dbpedia.org/resource/Bootsy_Collins, http://dbpedia.org/resource/Justin_Timberlake, http://dbpedia.org/resource/Matt_Sorum, http://dbpedia.org/resource/Jerry_Cantrell, http://dbpedia.org/resource/Koshi_Inaba, http://dbpedia.org/resource/Haiku_D'Etat, http://dbpedia.org/resource/Freestyle_Fellowship, http://dbpedia.org/resource/Geto_Boys, http://dbpedia.org/resource/Pimp_C, http://dbpedia.org/resource/3-2, http://dbpedia.org/resource/Jim_Jones, http://dbpedia.org/resource/Waka_Flocka_Flame, http://dbpedia.org/resource/Joe_Budden, http://dbpedia.org/resource/French_Montana, http://dbpedia.org/resource/Beyoncé, http://dbpedia.org/resource/Drake_(rapper), http://dbpedia.org/resource/Jay_Z, http://dbpedia.org/resource/Kanye_West, http://dbpedia.org/resource/Hot_Dollar, http://dbpedia.org/resource/Beenie_Man, http://dbpedia.org/resource/Silkk_the_Shocker, http://dbpedia.org/resource/Fiend_(rapper), http://dbpedia.org/resource/Magic_(rapper), http://dbpedia.org/resource/UGK, http://dbpedia.org/resource/Charlie_Wilson_(singer), http://dbpedia.org/resource/Foxy_Brown_(rapper), http://dbpedia.org/resource/Monica_(singer), http://dbpedia.org/resource/Donald_Robertson_(producer), http://dbpedia.org/resource/DJ_King_Assassin, http://dbpedia.org/resource/Too_Short, http://dbpedia.org/resource/Tupac_Shakur, http://dbpedia.org/resource/MC_Breed, http://dbpedia.org/resource/Timos, http://dbpedia.org/resource/LiL'_Gang$ta, http://dbpedia.org/resource/Mac_Dre, http://dbpedia.org/resource/Ras_Kass, http://dbpedia.org/resource/Digital_Underground, http://dbpedia.org/resource/Chino_XL, http://dbpedia.org/resource/Xzibit, http://dbpedia.org/resource/Deepspace5, http://dbpedia.org/resource/Tunnel_Rats_(music_group), http://dbpedia.org/resource/Roberta_Flack, http://dbpedia.org/resource/Dave_Grusin, http://dbpedia.org/resource/Doug_E._Fresh, http://dbpedia.org/resource/Marcus_Miller, http://dbpedia.org/resource/Lenny_White, http://dbpedia.org/resource/Gåte, http://dbpedia.org/resource/The_Anointed_Pace_Sisters, http://dbpedia.org/resource/Lisa_Marie_Presley, http://dbpedia.org/resource/Carly_Simon, http://dbpedia.org/resource/Bijou_Phillips, http://dbpedia.org/resource/Fiona_Apple, http://dbpedia.org/resource/João_Gordo, http://dbpedia.org/resource/Angra_(band), http://dbpedia.org/resource/Igor_Cavalera, http://dbpedia.org/resource/Recoil_(band), http://dbpedia.org/resource/Olivia_Louvel, http://dbpedia.org/resource/Eleven_(band), http://dbpedia.org/resource/Kate_Mirron, http://dbpedia.org/resource/Lovely_Baby_Music, http://dbpedia.org/resource/Lovely_Family, http://dbpedia.org/resource/Music_4_Brains, http://dbpedia.org/resource/Sleepy_Seaside_Piano, http://dbpedia.org/resource/Living_in_a_Box, http://dbpedia.org/resource/Etro_Anime, http://dbpedia.org/resource/Yasmine_Hamdan, http://dbpedia.org/resource/Nouvelle_Vague_(band), http://dbpedia.org/resource/Tiny_Morrie, http://dbpedia.org/resource/Al_Hurricane, http://dbpedia.org/resource/Baby_Gaby, http://dbpedia.org/resource/Vital_signs, http://dbpedia.org/resource/Pipedown, http://dbpedia.org/resource/Silage, http://dbpedia.org/resource/Worlds_Apart_(band), http://dbpedia.org/resource/Sebastian_Hardie, http://dbpedia.org/resource/Jon_English, http://dbpedia.org/resource/Das_Pop, http://dbpedia.org/resource/Camané, http://dbpedia.org/resource/Clã, http://dbpedia.org/resource/David_Fonseca, http://dbpedia.org/resource/Tír_na_nÓg_(band), http://dbpedia.org/resource/Anne_Byrne_(singer), http://dbpedia.org/resource/Lasgo, http://dbpedia.org/resource/Astroline, http://dbpedia.org/resource/AnnaGrace, http://dbpedia.org/resource/Leo_O'Kelly, http://dbpedia.org/resource/Dónal_Lunny, http://dbpedia.org/resource/Atabake, http://dbpedia.org/resource/Sin_Ánimo_De_Lucro, http://dbpedia.org/resource/Juanes, http://dbpedia.org/resource/Aterciopelados, http://dbpedia.org/resource/Bloque_de_Búsqueda_(banda), http://dbpedia.org/resource/Hora_Local, http://dbpedia.org/resource/Michel_Teló, http://dbpedia.org/resource/Rafael_Escalona, http://dbpedia.org/resource/Sidestepper, http://dbpedia.org/resource/Olivia_Newton-John, http://dbpedia.org/resource/The_Drifters, http://dbpedia.org/resource/The_Shadows, http://dbpedia.org/resource/Fistful_of_Mercy, http://dbpedia.org/resource/George_Harrison, http://dbpedia.org/resource/Thenewno2, http://dbpedia.org/resource/Traveling_Wilburys, http://dbpedia.org/resource/Antônio_Carlos_Jobim, http://dbpedia.org/resource/João_Gilberto, http://dbpedia.org/resource/Nana_Caymmi, http://dbpedia.org/resource/Dori_Caymmi, http://dbpedia.org/resource/Danilo_Caymmi, http://dbpedia.org/resource/Fela_Kuti, http://dbpedia.org/resource/Hugh_Masekela, http://dbpedia.org/resource/Africa_'70_(band), http://dbpedia.org/resource/Ginger_Baker, http://dbpedia.org/resource/Femi_Kuti, http://dbpedia.org/resource/Roy_Ayers, http://dbpedia.org/resource/Seun_Kuti, http://dbpedia.org/resource/Tony_Allen_(musician), http://dbpedia.org/resource/Lester_Bowie, http://dbpedia.org/resource/Faces_(band), http://dbpedia.org/resource/Rich_Kids, http://dbpedia.org/resource/Sex_Pistols, http://dbpedia.org/resource/The_International_Swingers, http://dbpedia.org/resource/Keanan_Duffty, http://dbpedia.org/resource/Bette_Bright, http://dbpedia.org/resource/Johnny_Hates_Jazz, http://dbpedia.org/resource/The_Philistines, http://dbpedia.org/resource/Vicious_White_Kids, http://dbpedia.org/resource/The_Flying_Padovanis, http://dbpedia.org/resource/The_London_Cowboys, http://dbpedia.org/resource/The_Graham_Bond_Organisation, http://dbpedia.org/resource/West,_Bruce_and_Laing, http://dbpedia.org/resource/Cream_(band), http://dbpedia.org/resource/The_Tony_Williams_Lifetime, http://dbpedia.org/resource/Kip_Hanrahan, http://dbpedia.org/resource/Blues_Incorporated, http://dbpedia.org/resource/Manfred_Mann, http://dbpedia.org/resource/Ringo_Starr_&_His_All-Starr_Band, http://dbpedia.org/resource/Rocket_88_(band), http://dbpedia.org/resource/John_Mayall_&_the_Bluesbreakers, http://dbpedia.org/resource/Bruce-Baker-Moore, http://dbpedia.org/resource/Deep_Purple, http://dbpedia.org/resource/Whitesnake, http://dbpedia.org/resource/The_Artwoods, http://dbpedia.org/resource/The_Flower_Pot_Men, http://dbpedia.org/resource/Paice_Ashton_Lord, http://dbpedia.org/resource/Diddy_–_Dirty_Money, http://dbpedia.org/resource/Trina, http://dbpedia.org/resource/Lady_Gaga, http://dbpedia.org/resource/Fabolous, http://dbpedia.org/resource/Vado_(rapper), http://dbpedia.org/resource/G-Unit, http://dbpedia.org/resource/Nate_Dogg, http://dbpedia.org/resource/Mobb_Deep, http://dbpedia.org/resource/Obie_Trice, http://dbpedia.org/resource/Raekwon, http://dbpedia.org/resource/Busta_Rhymes, http://dbpedia.org/resource/Eminem, http://dbpedia.org/resource/Jadakiss, http://dbpedia.org/resource/Styles_P, http://dbpedia.org/resource/Juelz_Santana, http://dbpedia.org/resource/Cypress_Hill, http://dbpedia.org/resource/The_Reyes_Brothers, http://dbpedia.org/resource/The_Zzyzzx, http://dbpedia.org/resource/Easy_Mo_Bee, http://dbpedia.org/resource/Wu-Tang_Clan, http://dbpedia.org/resource/John_Frusciante, http://dbpedia.org/resource/Gravediggaz, http://dbpedia.org/resource/James_Blake_(musician), http://dbpedia.org/resource/Jónsi, http://dbpedia.org/resource/Jónsi_&_Alex, http://dbpedia.org/resource/The_Album_Leaf, http://dbpedia.org/resource/Amiina, http://dbpedia.org/resource/Skunk_Anansie, http://dbpedia.org/resource/Clearlight_(French_band), http://dbpedia.org/resource/Gong_(band), http://dbpedia.org/resource/Khan_(band), http://dbpedia.org/resource/System_7_(band), http://dbpedia.org/resource/Uriel_(band), http://dbpedia.org/resource/The_Orb, http://dbpedia.org/resource/Bel_Canto_(band), http://dbpedia.org/resource/Röyksopp, http://dbpedia.org/resource/Red_Hot_Chili_Peppers, http://dbpedia.org/resource/The_Weirdos, http://dbpedia.org/resource/The_Dickies, http://dbpedia.org/resource/Captain_Beefheart, http://dbpedia.org/resource/Lydia_Lunch, http://dbpedia.org/resource/Thomas_Lang, http://dbpedia.org/resource/Talking_Heads, http://dbpedia.org/resource/The_Modern_Lovers, http://dbpedia.org/resource/Whitney_Houston, http://dbpedia.org/resource/Madonna_(entertainer), http://dbpedia.org/resource/Soft_Machine, http://dbpedia.org/resource/Adiemus, http://dbpedia.org/resource/Jeff_Lynne, http://dbpedia.org/resource/Electric_Light_Orchestra, http://dbpedia.org/resource/Alias_(band), http://dbpedia.org/resource/Heart_(band), http://dbpedia.org/resource/The_Beach_Boys, http://dbpedia.org/resource/Wizzard, http://dbpedia.org/resource/The_Move, http://dbpedia.org/resource/Roy_Wood's_Helicopters, http://dbpedia.org/resource/Wizzo_Band, http://dbpedia.org/resource/Alter_Bridge, http://dbpedia.org/resource/Projected, http://dbpedia.org/resource/Creed_(band), http://dbpedia.org/resource/Il_Divo, http://dbpedia.org/resource/Alessandra_Amoroso, http://dbpedia.org/resource/Elisa_(Italian_singer), http://dbpedia.org/resource/Laura_Pausini, http://dbpedia.org/resource/Fiorella_Mannoia, http://dbpedia.org/resource/Giusy_Ferreri, http://dbpedia.org/resource/Emile_Ford, http://dbpedia.org/resource/AC/DC, http://dbpedia.org/resource/Marcus_Hook_Roll_Band, http://dbpedia.org/resource/Paul_Kelly_(Australian_musician), http://dbpedia.org/resource/Professor_Ratbaggy, http://dbpedia.org/resource/Stardust_Five, http://dbpedia.org/resource/Carson_(band), http://dbpedia.org/resource/The_Dingoes, http://dbpedia.org/resource/MaJiKer, http://dbpedia.org/resource/Shaggy_(musician), http://dbpedia.org/resource/Colby_O'Donis, http://dbpedia.org/resource/Cham_(singer), http://dbpedia.org/resource/Lecca, http://dbpedia.org/resource/QED_(band), http://dbpedia.org/resource/INXS, http://dbpedia.org/resource/The_Crocodiles, http://dbpedia.org/resource/Eric_B._&_Rakim, http://dbpedia.org/resource/Kool_G_Rap, http://dbpedia.org/resource/A_Tribe_Called_Quest, http://dbpedia.org/resource/Main_Source, http://dbpedia.org/resource/Common_(rapper), http://dbpedia.org/resource/Akinyele_(rapper), http://dbpedia.org/resource/Nas, http://dbpedia.org/resource/Pete_Rock, http://dbpedia.org/resource/Skillz, http://dbpedia.org/resource/Cormega, http://dbpedia.org/resource/Tragedy_Khadafi, http://dbpedia.org/resource/Monrose, http://dbpedia.org/resource/Jean_Carn, http://dbpedia.org/resource/Marvin_Gaye, http://dbpedia.org/resource/Norman_Connors, http://dbpedia.org/resource/Miles_Davis, http://dbpedia.org/resource/Stevie_Wonder, http://dbpedia.org/resource/Ray_Parker,_Jr., http://dbpedia.org/resource/Legião_Urbana, http://dbpedia.org/resource/Lantana_(singer), http://dbpedia.org/resource/Uncle_Murda, http://dbpedia.org/resource/Joe_(singer), http://dbpedia.org/resource/Maino, http://dbpedia.org/resource/Spider_Loc, http://dbpedia.org/resource/K-Slick, http://dbpedia.org/resource/Danny_Brown_(rapper), http://dbpedia.org/resource/Mr._Porter, http://dbpedia.org/resource/Jagged_Edge_(group), http://dbpedia.org/resource/Tommy_&_the_Tom_Toms, http://dbpedia.org/resource/The_Rebels_(rockabilly_band), http://dbpedia.org/resource/Jürgen_Drews, http://dbpedia.org/resource/Bernhard_Brink, http://dbpedia.org/resource/Roberto_Blanco, http://dbpedia.org/resource/Christoph_Brüx, http://dbpedia.org/resource/Die_So_Fluid, http://dbpedia.org/resource/Jonathan_Davis_and_the_SFA, http://dbpedia.org/resource/Everlast_(musician), http://dbpedia.org/resource/Whitey_(band), http://dbpedia.org/resource/Evanescence, http://dbpedia.org/resource/Fear_and_the_Nervous_System, http://dbpedia.org/resource/Hairy_Apes_BMX, http://dbpedia.org/resource/Adam_Lambert, http://dbpedia.org/resource/Cottonmouth,TX, http://dbpedia.org/resource/Maimou, http://dbpedia.org/resource/Billy_Goat_(band), http://dbpedia.org/resource/Billy_Thorpe_and_the_Aztecs, http://dbpedia.org/resource/Max_Merritt, http://dbpedia.org/resource/Per_Gessle, http://dbpedia.org/resource/Sandy_Mouche, http://dbpedia.org/resource/Kimbra, http://dbpedia.org/resource/Bee_Gees, http://dbpedia.org/resource/Circus_Diablo, http://dbpedia.org/resource/Filter_(band), http://dbpedia.org/resource/Otep, http://dbpedia.org/resource/Smile_Empty_Soul, http://dbpedia.org/resource/World_Fire_Brigade, http://dbpedia.org/resource/He_Is_Legend, http://dbpedia.org/resource/The_Dandy_Warhols, http://dbpedia.org/resource/Boy_in_Static, http://dbpedia.org/resource/Animotion, http://dbpedia.org/resource/Peaches_(musician), http://dbpedia.org/resource/The_Gentry_(band), http://dbpedia.org/resource/Styrofoam_(musician), http://dbpedia.org/resource/Peter_Gabriel, http://dbpedia.org/resource/Bill_Nelson's_Red_Noise, http://dbpedia.org/resource/Darlin'_(French_band), http://dbpedia.org/resource/Phoenix_(band), http://dbpedia.org/resource/The_Wurzels, http://dbpedia.org/resource/Lennon_&_Maisy, http://dbpedia.org/resource/Johnny_Reid, http://dbpedia.org/resource/Humanos, http://dbpedia.org/resource/Donnie_Klang, http://dbpedia.org/resource/Danity_Kane, http://dbpedia.org/resource/Dawn_Richard_(singer), http://dbpedia.org/resource/Sean_Combs, http://dbpedia.org/resource/Day26, http://dbpedia.org/resource/Beryl_Marsden, http://dbpedia.org/resource/Adam_Rickitt, http://dbpedia.org/resource/Blake_(band), http://dbpedia.org/resource/Lizzie_Marvelly, http://dbpedia.org/resource/Julie_Atherton, http://dbpedia.org/resource/The_Soldiers, http://dbpedia.org/resource/Uwe_Schmidt, http://dbpedia.org/resource/Los_Prisioneros, http://dbpedia.org/resource/Los_Updates, http://dbpedia.org/resource/Dandy_Jack, http://dbpedia.org/resource/Damage_(British_band), http://dbpedia.org/resource/Big_Dirty_Band, http://dbpedia.org/resource/The_Tea_Party, http://dbpedia.org/resource/Crash_Karma, http://dbpedia.org/resource/The_Art_Decay, http://dbpedia.org/resource/Empire_of_the_Sun_(band), http://dbpedia.org/resource/Pnau, http://dbpedia.org/resource/Teenager_(band), http://dbpedia.org/resource/Mantronix, http://dbpedia.org/resource/Os_Paralamas_do_Sucesso, http://dbpedia.org/resource/Paula_Toller, http://dbpedia.org/resource/Dado_e_o_Reino_Animal, http://dbpedia.org/resource/John_Gosling_(Psychic_TV_musician), http://dbpedia.org/resource/Goldie, http://dbpedia.org/resource/Metalheadz, http://dbpedia.org/resource/Khao_(musician), http://dbpedia.org/resource/Jam_Nation, http://dbpedia.org/resource/Cyssero, http://dbpedia.org/resource/Backstreet_Boys, http://dbpedia.org/resource/Kaila_Yu, http://dbpedia.org/resource/J._Holiday, http://dbpedia.org/resource/Jasmine_Trias, http://dbpedia.org/resource/Natalise, http://dbpedia.org/resource/Lord_Large, http://dbpedia.org/resource/Mike_Elliott_(saxophonist), http://dbpedia.org/resource/The_Foundations, http://dbpedia.org/resource/Arthur_Brown_(musician), http://dbpedia.org/resource/The_Ramong_Sound, http://dbpedia.org/resource/Johnny_Johnson_and_the_Bandwagon, http://dbpedia.org/resource/Donnie_Elbert, http://dbpedia.org/resource/Alan_Warner_(musician), http://dbpedia.org/resource/The_Corporation_(band), http://dbpedia.org/resource/Colin_Young, http://dbpedia.org/resource/Lucky_Dube, http://dbpedia.org/resource/Massive_Attack, http://dbpedia.org/resource/Christine_Anu, http://dbpedia.org/resource/Thundamentals, http://dbpedia.org/resource/Cher, http://dbpedia.org/resource/Elton_John, http://dbpedia.org/resource/Sparks_(band), http://dbpedia.org/resource/Poperetta, http://dbpedia.org/resource/Twin_Trip, http://dbpedia.org/resource/K.d._lang, http://dbpedia.org/resource/Paul_Young, http://dbpedia.org/resource/Rickie_Lee_Jones, http://dbpedia.org/resource/Martini_Ranch, http://dbpedia.org/resource/David_Shankle, http://dbpedia.org/resource/Civilization_One_(band), http://dbpedia.org/resource/Firewind, http://dbpedia.org/resource/Avalon_(metal_band), http://dbpedia.org/resource/Faro_(band), http://dbpedia.org/resource/Rekuiem, http://dbpedia.org/resource/Domain_(band), http://dbpedia.org/resource/Power_Quest, http://dbpedia.org/resource/Red_Circuit, http://dbpedia.org/resource/Elis_Regina, http://dbpedia.org/resource/Rádio_Táxi, http://dbpedia.org/resource/Bob_Dylan, http://dbpedia.org/resource/Maria_McKee, http://dbpedia.org/resource/Joe_Cocker, http://dbpedia.org/resource/Beth_Hart, http://dbpedia.org/resource/Joe_Bonamassa, http://dbpedia.org/resource/Bison_B.C., http://dbpedia.org/resource/3_Inches_of_Blood, http://dbpedia.org/resource/Bloods, http://dbpedia.org/resource/Tha_Alkaholiks, http://dbpedia.org/resource/Ol'_Dirty_Bastard, http://dbpedia.org/resource/DJ_Pooh, http://dbpedia.org/resource/Skee-Lo, http://dbpedia.org/resource/Crips, http://dbpedia.org/resource/Ice-T, http://dbpedia.org/resource/Kurtis_Blow, http://dbpedia.org/resource/List_of_Wu-Tang_Clan_affiliates, http://dbpedia.org/resource/DJ_Tomekk, http://dbpedia.org/resource/Prodigal_Sunn, http://dbpedia.org/resource/Flavor_Flav, http://dbpedia.org/resource/LL_Cool_J, http://dbpedia.org/resource/Battlecat_(record_producer), http://dbpedia.org/resource/DJ_Quik, http://dbpedia.org/resource/60_Second_Assassin_(emcee), http://dbpedia.org/resource/Raphael_Saadiq, http://dbpedia.org/resource/Solange, http://dbpedia.org/resource/Justice_(band), http://dbpedia.org/resource/Lenny_Kravitz, http://dbpedia.org/resource/A-ha, http://dbpedia.org/resource/Montée, http://dbpedia.org/resource/Dev_Hynes, http://dbpedia.org/resource/Dâm-funk, http://dbpedia.org/resource/Funkin’_Matt, http://dbpedia.org/resource/Jimmy_Douglas_(producer), http://dbpedia.org/resource/Hans-Peter_Lindstrøm, http://dbpedia.org/resource/On_An_On, http://dbpedia.org/resource/Royal_Scottish_National_Orchestra, http://dbpedia.org/resource/Royal_Philharmonic_Orchestra, http://dbpedia.org/resource/Edmonton_Symphony_Orchestra, http://dbpedia.org/resource/London_Symphony_Orchestra, http://dbpedia.org/resource/Philharmonia_Orchestra, http://dbpedia.org/resource/National_Arts_Centre_Orchestra, http://dbpedia.org/resource/Toronto_Symphony_Orchestra, http://dbpedia.org/resource/London_Philharmonic_Orchestra, http://dbpedia.org/resource/The_Hallé, http://dbpedia.org/resource/English_Chamber_Orchestra, http://dbpedia.org/resource/John_Arpin, http://dbpedia.org/resource/Quarteto_Novo, http://dbpedia.org/resource/Très.b, http://olivierheim.com/, http://dbpedia.org/resource/Xmal_Deutschland, http://dbpedia.org/resource/Glen_Matlock, http://dbpedia.org/resource/R.E.M., http://dbpedia.org/resource/Method_Actors, http://dbpedia.org/resource/Blaine_L._Reininger, http://dbpedia.org/resource/39_Clocks, http://dbpedia.org/resource/Tanzplagen, http://dbpedia.org/resource/Abwärts, http://dbpedia.org/resource/Tuxedomoon, http://dbpedia.org/resource/Pylon_(band), http://dbpedia.org/resource/Montanablue, http://dbpedia.org/resource/Oh_Ok, http://dbpedia.org/resource/The_Upper_Crust_(band), http://dbpedia.org/resource/Kastrierte_Philosophen, http://dbpedia.org/resource/Lionel_Hampton, http://dbpedia.org/resource/Maria_Schneider_(musician), http://dbpedia.org/resource/Baker_Gurvitz_Army, http://dbpedia.org/resource/Three_Man_Army, http://dbpedia.org/resource/The_Gun_(band), http://dbpedia.org/resource/The_Entire_Population_of_Hackney, http://dbpedia.org/resource/Bruce_Dickinson, http://dbpedia.org/resource/Michael_Kiske, http://dbpedia.org/resource/ASAP_(band), http://dbpedia.org/resource/Psycho_Motel, http://dbpedia.org/resource/Urchin_(band), http://dbpedia.org/resource/Primal_Rock_Rebellion, http://dbpedia.org/resource/Samson_(band), http://dbpedia.org/resource/Sack_Trick, http://dbpedia.org/resource/Tribe_of_Gypsies, http://dbpedia.org/resource/Interpol_(band), http://dbpedia.org/resource/O_Rappa, http://dbpedia.org/resource/Nação_Zumbi, http://dbpedia.org/resource/CPM_22, http://dbpedia.org/resource/Marcelo_Nova, http://dbpedia.org/resource/Raimundos, http://dbpedia.org/resource/Marcelo_D2, http://dbpedia.org/resource/Planet_Hemp, http://dbpedia.org/resource/A_Banca, http://dbpedia.org/resource/Carl_Michael_Bellman, http://dbpedia.org/resource/Ann-Louise_Hanson, http://dbpedia.org/resource/Fred_Åkerström, http://dbpedia.org/resource/Evert_Taube, http://dbpedia.org/resource/Marmalade, http://dbpedia.org/resource/Marmalade_(band), http://dbpedia.org/resource/Teen_Angels, http://dbpedia.org/resource/Cris_Morena, http://dbpedia.org/resource/Harry_Nilsson, http://dbpedia.org/resource/The_Beatles, http://dbpedia.org/resource/Plastic_Ono_Band, http://dbpedia.org/resource/Badfinger, http://dbpedia.org/resource/Ringo_Starr, http://dbpedia.org/resource/Alan_Parsons, http://dbpedia.org/resource/Dusty_Springfield, http://dbpedia.org/resource/Tremonti_(band), http://dbpedia.org/resource/Clark_Terry, http://dbpedia.org/resource/Charlie_Parker, http://dbpedia.org/resource/Gerry_Mulligan, http://dbpedia.org/resource/Miles_Davis_Quintet, http://dbpedia.org/resource/Gil_Evans, http://dbpedia.org/resource/John_Coltrane, http://dbpedia.org/resource/Billy_Eckstine, http://dbpedia.org/resource/Paulo_Coelho, http://dbpedia.org/resource/Zé_Ramalho, http://dbpedia.org/resource/Mutya_Keisha_Siobhan, http://dbpedia.org/resource/Sugababes, http://dbpedia.org/resource/Shania_Twain, http://dbpedia.org/resource/Dean_Brody, http://dbpedia.org/resource/Reba_McEntire, http://dbpedia.org/resource/Tom_Shapiro, http://dbpedia.org/resource/Dierks_Bentley, http://dbpedia.org/resource/King_Crimson, http://dbpedia.org/resource/Uriah_Heep_(band), http://dbpedia.org/resource/Australian_Crawl, http://dbpedia.org/resource/The_Choirboys_(band), http://dbpedia.org/resource/Rainbow_(rock_band), http://dbpedia.org/resource/The_Angels_(Australian_band), http://dbpedia.org/resource/The_Animals, http://dbpedia.org/resource/Models_(band), http://dbpedia.org/resource/Mondo_Rock, http://dbpedia.org/resource/Noiseworks, http://dbpedia.org/resource/Rose_Tattoo, http://dbpedia.org/resource/Skyhooks_(band), http://dbpedia.org/resource/Sherbet_(band), http://dbpedia.org/resource/Swanee_(singer), http://dbpedia.org/resource/Divinyls, http://dbpedia.org/resource/Dragon_(band), http://dbpedia.org/resource/Alcatrazz, http://dbpedia.org/resource/Gang_Gajang, http://dbpedia.org/resource/Eagles_(band), http://dbpedia.org/resource/Status_Quo_(band), http://dbpedia.org/resource/The_La_De_Das, http://dbpedia.org/resource/Them_(band), http://dbpedia.org/resource/The_Ferrets_(band), http://dbpedia.org/resource/Rick_Springfield, http://dbpedia.org/resource/Cotton_Keays_&_Morris, http://dbpedia.org/resource/Carmen_Maki, http://dbpedia.org/resource/Will_Oldham, http://dbpedia.org/resource/Seeker_Lover_Keeper, http://dbpedia.org/resource/Darren_Hanlon, http://dbpedia.org/resource/Tiny_Ruins, http://dbpedia.org/resource/Jim_White_(drummer), http://dbpedia.org/resource/Machine_Translations, http://dbpedia.org/resource/Tony_Dupe, http://dbpedia.org/resource/Hayden_(musician), http://dbpedia.org/resource/Jack_Ladder, http://dbpedia.org/resource/Daddy_Cool_(band), http://dbpedia.org/resource/The_Black_Sorrows, http://dbpedia.org/resource/Company_Caine, http://dbpedia.org/resource/Ol'_55_(band), http://dbpedia.org/resource/The_Pelaco_Brothers, http://dbpedia.org/resource/The_Seekers, http://dbpedia.org/resource/Jimmy_Barnes, http://dbpedia.org/resource/Jon_Stevens, http://dbpedia.org/resource/Allison_Durbin, http://dbpedia.org/resource/Venetta_Fields, http://dbpedia.org/resource/Kate_Ceberano, http://dbpedia.org/resource/Tommy_Emmanuel, http://dbpedia.org/resource/Lionel_Richie, http://dbpedia.org/resource/Richard_Marx, http://dbpedia.org/resource/Tom_Jones_(singer), http://dbpedia.org/resource/Anthony_Warlow, http://dbpedia.org/resource/Colleen_Hewett, http://dbpedia.org/resource/Lisa_Edwards, http://dbpedia.org/resource/Human_Nature_(band), http://dbpedia.org/resource/David_Hirschfelder, http://dbpedia.org/resource/An_Empty_Flight, http://dbpedia.org/resource/Something_for_Kate, http://dbpedia.org/resource/Cold_Chisel, http://dbpedia.org/resource/Spectrum_(band), http://dbpedia.org/resource/Cheetah_(band), http://dbpedia.org/resource/Mighty_Kong, http://dbpedia.org/resource/Cam-Pact, http://dbpedia.org/resource/Flash_and_the_Pan, http://dbpedia.org/resource/Billy_Dean, http://dbpedia.org/resource/Kathy_Mattea, http://dbpedia.org/resource/Patty_Loveless, http://dbpedia.org/resource/Pam_Tillis, http://dbpedia.org/resource/Lee_Greenwood, http://dbpedia.org/resource/Gretchen_Peters, http://dbpedia.org/resource/Matraca_Berg, http://dbpedia.org/resource/Bobby_Caldwell, http://dbpedia.org/resource/Jacques_Dutronc, http://dbpedia.org/resource/Rick_Ross, http://dbpedia.org/resource/Big_Daddy_Kane, http://dbpedia.org/resource/Deion_Sanders, http://dbpedia.org/resource/Jon_Gibson_(Christian_musician), http://dbpedia.org/resource/B_Angie_B, http://dbpedia.org/resource/Insane_Clown_Posse, http://dbpedia.org/resource/BeBe_&_CeCe_Winans, http://dbpedia.org/resource/Teddy_Riley, http://dbpedia.org/resource/Keyshia_Cole, http://dbpedia.org/resource/Vanilla_Ice, http://dbpedia.org/resource/Psy, http://dbpedia.org/resource/Tha_Dogg_Pound, http://dbpedia.org/resource/D'Angelo, http://dbpedia.org/resource/Johnny_%22J%22, http://dbpedia.org/resource/Paperboy_(rapper), http://dbpedia.org/resource/Giorgio_Gaber, http://dbpedia.org/resource/Lucio_Battisti, http://dbpedia.org/resource/Pino_Presti, http://dbpedia.org/resource/Adriano_Celentano, http://dbpedia.org/resource/Bruno_Canfora, http://dbpedia.org/resource/Augusto_Martelli, http://dbpedia.org/resource/Gianni_Ferrio, http://dbpedia.org/resource/Currensy, http://dbpedia.org/resource/Kendrick_Lamar, http://dbpedia.org/resource/MF_Doom, http://dbpedia.org/resource/DeStorm_Power, http://dbpedia.org/resource/Mary_J._Blige, http://dbpedia.org/resource/Mos_Def, http://dbpedia.org/resource/Pharrell_Williams, http://dbpedia.org/resource/Black_Star_(rap_duo), http://dbpedia.org/resource/The_Procussions, http://dbpedia.org/resource/The_Roots, http://dbpedia.org/resource/Cassper_Nyovest, http://dbpedia.org/resource/Lupe_Fiasco, http://dbpedia.org/resource/Madlib, http://dbpedia.org/resource/Black_Thought, http://dbpedia.org/resource/Reflection_Eternal, http://dbpedia.org/resource/Res_(singer), http://dbpedia.org/resource/4th_Avenue_Jones, http://dbpedia.org/resource/Beck, http://dbpedia.org/resource/Sting_(musician), http://dbpedia.org/resource/Geordie_(band), http://dbpedia.org/resource/The_Hollywood_Vampires, http://dbpedia.org/resource/Hoodoo_Gurus, http://dbpedia.org/resource/The_Manikins, http://dbpedia.org/resource/The_Victims_(Australian_band), http://dbpedia.org/resource/Blondie_(band), http://dbpedia.org/resource/Iggy_Pop, http://dbpedia.org/resource/Marc_Bolan, http://dbpedia.org/resource/T._Rex_(band), http://dbpedia.org/resource/Andy_McCoy, http://dbpedia.org/resource/The_69_Cats, http://dbpedia.org/resource/Bam_Margera, http://dbpedia.org/resource/The_69_Eyes, http://dbpedia.org/resource/Ville_Valo, http://dbpedia.org/resource/Maryslim, http://dbpedia.org/resource/Clube_da_Esquina, http://dbpedia.org/resource/Giorgio_Moroder, http://dbpedia.org/resource/The_Human_League, http://dbpedia.org/resource/Blind_Melon, http://dbpedia.org/resource/J-Son, http://dbpedia.org/resource/Sebastian_Ingrosso, http://dbpedia.org/resource/Axwell, http://dbpedia.org/resource/Axwell_and_Ingrosso, http://dbpedia.org/resource/Avicii, http://dbpedia.org/resource/Goodie_Mob, http://dbpedia.org/resource/TLC_(group), http://dbpedia.org/resource/Outkast, http://dbpedia.org/resource/Kelis, http://dbpedia.org/resource/The_D.O.C., http://dbpedia.org/resource/Fishbone, http://dbpedia.org/resource/Pink_(singer), http://dbpedia.org/resource/Dallas_Austin, http://dbpedia.org/resource/Organized_Noize, http://dbpedia.org/resource/Truth_Hurts, http://dbpedia.org/resource/Sheriff_(band), http://dbpedia.org/resource/The_Cranberries, http://dbpedia.org/resource/Dolores_O'Riordan, http://dbpedia.org/resource/George_Michael, http://dbpedia.org/resource/Andrew_Ridgeley, http://dbpedia.org/resource/Wham!, http://dbpedia.org/resource/Boogie_Box_High, http://dbpedia.org/resource/Fancy_Me_Yet, http://dbpedia.org/resource/Üebermutter, http://dbpedia.org/resource/Andy_García, http://dbpedia.org/resource/Orestes_López, http://dbpedia.org/resource/Arcaño_y_sus_Maravillas, http://dbpedia.org/resource/Scooch, http://dbpedia.org/resource/The_Irish_Tenors, http://dbpedia.org/resource/St._Michael's_Choir_School, http://dbpedia.org/resource/Frente!, http://dbpedia.org/resource/Merril_Bainbridge, http://dbpedia.org/resource/The_Harp_Consort, http://dbpedia.org/resource/Sequence_(musical_form), http://dbpedia.org/resource/Chris_Elliott, http://dbpedia.org/resource/Brasco, http://dbpedia.org/resource/Ekaterina_Vilkova, http://dbpedia.org/resource/Rihanna, http://dbpedia.org/resource/Vera_Brezhneva, http://dbpedia.org/resource/Bob_Clearmountain, http://dbpedia.org/resource/Jesse_Dylan, http://dbpedia.org/resource/Tony_Maserati, http://dbpedia.org/resource/O-Zone, http://dbpedia.org/resource/Crazy_Loop, http://dbpedia.org/resource/Jack_Joseph_Puig, http://dbpedia.org/resource/Chris_Lord-Alge, http://dbpedia.org/resource/Spanish_Harlem_Orchestra, http://dbpedia.org/resource/The_Detroit_Experiment, http://dbpedia.org/resource/Bio_Ritmo, http://dbpedia.org/resource/The_Philadelphia_Experiment_(album), http://dbpedia.org/resource/The_Harlem_Experiment, http://dbpedia.org/resource/Jimmy_Bosch, http://dbpedia.org/resource/Junoon_(band), http://dbpedia.org/resource/Atish_Raj, http://dbpedia.org/resource/Mizraab, http://dbpedia.org/resource/Ali_Azmat, http://dbpedia.org/resource/Coke_Studio_(Pakistan), http://dbpedia.org/resource/Bondax, http://dbpedia.org/resource/Tully_(band), http://dbpedia.org/resource/Ray_Brown_&_the_Whispers, http://dbpedia.org/resource/Tim_Ries, http://dbpedia.org/resource/Lionel_Loueke, http://dbpedia.org/resource/Aaron_Goldberg, http://dbpedia.org/resource/Alex_Kautz, http://dbpedia.org/resource/Los_Súper_Ratones, http://dbpedia.org/resource/The_Tropical_Showband, http://dbpedia.org/resource/The_Word_(Irish_band), http://dbpedia.org/resource/Carrier_Frequency, http://dbpedia.org/resource/Naima_(band), http://dbpedia.org/resource/The_Food_Zebras, http://dbpedia.org/resource/Emmet_Spiceland, http://dbpedia.org/resource/Garvan_Gallagher, http://dbpedia.org/resource/Lough_Gill_Quartet, http://dbpedia.org/resource/Ceoltóirí_Chualann, http://dbpedia.org/resource/Gun_(band), http://dbpedia.org/resource/David_Foster, http://dbpedia.org/resource/Jon_Secada, http://dbpedia.org/resource/Lara_Fabian, http://dbpedia.org/resource/Jim_Brickman, http://dbpedia.org/resource/Amy_Sky, http://dbpedia.org/resource/Tin_Machine, http://dbpedia.org/resource/Utopia_(American_band), http://dbpedia.org/resource/Chequered_Past, http://dbpedia.org/resource/FTG_(band), http://dbpedia.org/resource/Kids_Top_20, http://dbpedia.org/resource/Hurriganes, http://dbpedia.org/resource/Jane_(Finnish_band), http://dbpedia.org/resource/U2, http://dbpedia.org/resource/Eric_Clapton, http://dbpedia.org/resource/Bobby_Bland, http://dbpedia.org/resource/Garbage_(band), http://dbpedia.org/resource/Jane's_Addiction, http://dbpedia.org/resource/Deconstruction_(band), http://dbpedia.org/resource/Nine_Inch_Nails, http://dbpedia.org/resource/Polar_Bear_(American_band), http://dbpedia.org/resource/Alanis_Morissette, http://dbpedia.org/resource/Cilla_Black, http://dbpedia.org/resource/Jeff_Beck, http://dbpedia.org/resource/America_(band), http://dbpedia.org/resource/Mahavishnu_Orchestra, http://dbpedia.org/resource/Jeru_the_Damaja, http://dbpedia.org/resource/Big_L, http://dbpedia.org/resource/Nujabes, http://dbpedia.org/resource/Gang_Starr_Foundation, http://dbpedia.org/resource/Krumbsnatcha, http://dbpedia.org/resource/Gang_Starr, http://dbpedia.org/resource/Group_Home, http://dbpedia.org/resource/M.O.P., http://dbpedia.org/resource/DJ_Premier, http://dbpedia.org/resource/Gemelli_Diversi, http://dbpedia.org/resource/Club_Dogo, http://dbpedia.org/resource/Max_Pezzali, http://dbpedia.org/resource/Marracash, http://dbpedia.org/resource/The_Jeff_Beck_Group, http://dbpedia.org/resource/Beck,_Bogert_&_Appice, http://dbpedia.org/resource/The_Yardbirds, http://dbpedia.org/resource/Katrina_and_the_Waves, http://dbpedia.org/resource/New_York_Dolls, http://dbpedia.org/resource/Bow_Wow_Wow, http://dbpedia.org/resource/The_World's_Famous_Supreme_Team, http://dbpedia.org/resource/Dan_Kelly_(musician), http://dbpedia.org/resource/Runrig, http://dbpedia.org/resource/Slim_Dusty, http://dbpedia.org/resource/War_(band), http://dbpedia.org/resource/Milva, http://dbpedia.org/resource/Alice_(singer), http://dbpedia.org/resource/Juri_Camisasca, http://dbpedia.org/resource/Cliff_Richard, http://dbpedia.org/resource/Roy_Orbison, http://dbpedia.org/resource/Dubb_Union, http://dbpedia.org/resource/Baby_Bash, http://dbpedia.org/resource/Kumbia_Kings, http://dbpedia.org/resource/Pitbull_(rapper), http://dbpedia.org/resource/Shane_Nicholson_(singer), http://dbpedia.org/resource/Dick_Marx, http://dbpedia.org/resource/Keith_Urban, http://dbpedia.org/resource/Fee_Waybill, http://dbpedia.org/resource/Luther_Vandross, http://dbpedia.org/resource/Matt_Scannell, http://dbpedia.org/resource/Siouxsie_and_the_Banshees, http://dbpedia.org/resource/The_Flowers_of_Romance_(band), http://dbpedia.org/resource/The_Wiggles, http://dbpedia.org/resource/Joy_McKean, http://dbpedia.org/resource/Anne_Kirkpatrick, http://dbpedia.org/resource/Rainhard_Fendrich, http://dbpedia.org/resource/Thompson_Twins, http://dbpedia.org/resource/Savage_Progress, http://dbpedia.org/resource/Uropa_Lula, http://dbpedia.org/resource/Outlawz, http://dbpedia.org/resource/Z-Ro, http://dbpedia.org/resource/Lil'_½_Dead, http://dbpedia.org/resource/WC_(rapper), http://dbpedia.org/resource/Soopafly, http://dbpedia.org/resource/Dr._Dre, http://dbpedia.org/resource/RBX, http://dbpedia.org/resource/Kurupt, http://dbpedia.org/resource/The_Lady_of_Rage, http://dbpedia.org/resource/Young_Buck, http://dbpedia.org/resource/Anastacia, http://dbpedia.org/resource/Nelly_Furtado, http://dbpedia.org/resource/Edvin_Marton, http://dbpedia.org/resource/Timbaland, http://dbpedia.org/resource/J.R._Rotem, http://dbpedia.org/resource/Jim_Beanz, http://dbpedia.org/resource/Ryan_Tedder, http://dbpedia.org/resource/Evgeni_Plushenko, http://dbpedia.org/resource/Larisa_Dolina, http://dbpedia.org/resource/Trudy_Bellinger, http://dbpedia.org/resource/Kirk_Whalum, http://dbpedia.org/resource/Robert_Glasper, http://dbpedia.org/resource/Rahsaan_Patterson, http://dbpedia.org/resource/Premiata_Forneria_Marconi, http://dbpedia.org/resource/Roxy_Music, http://dbpedia.org/resource/Bucks_Fizz, http://dbpedia.org/resource/Emerson,_Lake_&_Palmer, http://dbpedia.org/resource/Five_Star, http://dbpedia.org/resource/Greg_Lake, http://dbpedia.org/resource/Keith_Christmas, http://dbpedia.org/resource/Ghostface_Killah, http://dbpedia.org/resource/The_Lox, http://dbpedia.org/resource/Capone-N-Noreaga, http://dbpedia.org/resource/U-God, http://dbpedia.org/resource/Cappadonna, http://dbpedia.org/resource/Gilberto_Gil, http://dbpedia.org/resource/Pitty, http://dbpedia.org/resource/Ronnie_Von, http://dbpedia.org/resource/Caetano_Veloso, http://dbpedia.org/resource/Os_Mutantes, http://dbpedia.org/resource/Tutti_Frutti_(Brazilian_band), http://dbpedia.org/resource/Chico_Buarque, http://dbpedia.org/resource/Ivan_Lins, http://dbpedia.org/resource/Milton_Nascimento, http://dbpedia.org/resource/Sueli_Costa, http://dbpedia.org/resource/Martinho_da_Vila, http://dbpedia.org/resource/Bobby_Creekwater, http://dbpedia.org/resource/Cashis, http://dbpedia.org/resource/Talib_Kweli, http://dbpedia.org/resource/Bink_(record_producer), http://dbpedia.org/resource/D12, http://dbpedia.org/resource/50_Cent, http://dbpedia.org/resource/The_Game_(rapper), http://dbpedia.org/resource/The_Easybeats, http://dbpedia.org/resource/K._Michelle, http://dbpedia.org/resource/Diamond_(rapper), http://dbpedia.org/resource/Nicki_Minaj, http://dbpedia.org/resource/Pink_Floyd, http://dbpedia.org/resource/David_Gilmour, http://dbpedia.org/resource/Anthony_Moore, http://dbpedia.org/resource/Syd_Barrett, http://dbpedia.org/resource/Identity_(Zee_album), http://dbpedia.org/resource/Jamiroquai, http://dbpedia.org/resource/Mark_Ronson]... [INFO] Predicted Answer: [http://dbpedia.org/resource/Jack_DeJohnette, http://dbpedia.org/resource/Abdul_Halim_Jaffer_Khan, http://dbpedia.org/resource/James_Tyler_(music), http://dbpedia.org/resource/John_Scott_(composer), http://dbpedia.org/resource/Wookie_(musician), http://dbpedia.org/resource/Doreen_Waddell, http://dbpedia.org/resource/Lamya, http://dbpedia.org/resource/Caron_Wheeler, http://dbpedia.org/resource/Melissa_Bell_(singer), http://dbpedia.org/resource/Nellee_Hooper, http://dbpedia.org/resource/Richie_Stephens, http://dbpedia.org/resource/Penny_Ford, http://dbpedia.org/resource/Charlotte_(singer), http://dbpedia.org/resource/Bark_Psychosis, http://dbpedia.org/resource/Lee_Harris_(drummer), http://dbpedia.org/resource/Mark_Hollis_(musician), http://dbpedia.org/resource/Paul_Webb, http://dbpedia.org/resource/Colin_Thurston, http://dbpedia.org/resource/Stephen_Budd, http://dbpedia.org/resource/Danny_and_the_Champions_of_the_World, http://dbpedia.org/resource/Victoria_Legrand, http://dbpedia.org/resource/Brian_Kehew, http://dbpedia.org/resource/Jean-Benoît_Dunckel, http://dbpedia.org/resource/Nicolas_Godin, http://dbpedia.org/resource/Yoad_Nevo, http://dbpedia.org/resource/Phoenix_(band), http://dbpedia.org/resource/Earl_Harvin, http://dbpedia.org/resource/Brian_Reitzell, http://dbpedia.org/resource/Jimmy_McShane, http://dbpedia.org/resource/P._J._Olsson, http://dbpedia.org/resource/Damian_LeGassick, http://dbpedia.org/resource/Linda_McCartney, http://dbpedia.org/resource/Greg_Smith_(American_musician), http://dbpedia.org/resource/Zeno_Roth, http://dbpedia.org/resource/Scorpions_(band), http://dbpedia.org/resource/Uli_Jon_Roth, http://dbpedia.org/resource/Maury_Muehleisen, http://dbpedia.org/resource/Patti_Dahlstrom, http://dbpedia.org/resource/Ray_Cooper, http://dbpedia.org/resource/Pete_Gage_(guitarist), http://dbpedia.org/resource/Red_Young, http://dbpedia.org/resource/Pam_Nestor, http://dbpedia.org/resource/Pino_Palladino, http://dbpedia.org/resource/Jennifer_Maidman, http://dbpedia.org/resource/The_Movies_(band), http://dbpedia.org/resource/Ricardo_Garcia_(German_musician), http://dbpedia.org/resource/Banda_Calypso, http://dbpedia.org/resource/Chris_Jones_(singer-songwriter), http://dbpedia.org/resource/Alvin_Lee, http://dbpedia.org/resource/Ric_Lee, http://dbpedia.org/resource/Joe_Gooch, http://dbpedia.org/resource/Scotty_Moore, http://dbpedia.org/resource/Mike_Vernon_(record_producer), http://dbpedia.org/resource/Chick_Churchill, http://dbpedia.org/resource/Leo_Lyons, http://dbpedia.org/resource/Tank_(American_singer), http://dbpedia.org/resource/Da_Brat, http://dbpedia.org/resource/Ginuwine, http://dbpedia.org/resource/Brandon_%22B.A.M%22_Hodge, http://dbpedia.org/resource/Piret_Järvis, http://dbpedia.org/resource/Marya_Roxx, http://dbpedia.org/resource/Triinu_Kivilaan, http://dbpedia.org/resource/Sven_Lõhmus, http://dbpedia.org/resource/Lenna_Kuurmaa, http://dbpedia.org/resource/Mystikal, http://dbpedia.org/resource/Mercedes_(singer), http://dbpedia.org/resource/Fiend_(rapper), http://dbpedia.org/resource/Lil_Ric, http://dbpedia.org/resource/Mia_X, http://dbpedia.org/resource/Mr._Serv-On, http://dbpedia.org/resource/Tre-8, http://dbpedia.org/resource/Mo_B._Dick, http://dbpedia.org/resource/Krazy_(rapper), http://dbpedia.org/resource/Kane_&_Abel_(group), http://dbpedia.org/resource/Prime_Suspects, http://dbpedia.org/resource/Skull_Duggery_(rapper), http://dbpedia.org/resource/Montell_Jordan, http://dbpedia.org/resource/Soulja_Slim, http://dbpedia.org/resource/Mac_(rapper), http://dbpedia.org/resource/Gambino_Family_(group), http://dbpedia.org/resource/Lil_Italy, http://dbpedia.org/resource/Sons_of_Funk, http://dbpedia.org/resource/Steady_Mobb'n, http://dbpedia.org/resource/702_(group), http://dbpedia.org/resource/Juvenile_(rapper), http://dbpedia.org/resource/Donald_Robertson_(producer), http://dbpedia.org/resource/Big_Ed, http://dbpedia.org/resource/Shahrdad_Rohani, http://dbpedia.org/resource/José_José, http://dbpedia.org/resource/Armen_Anassian, http://dbpedia.org/resource/Armen_Movsessian, http://dbpedia.org/resource/Charlie_Adams_(drummer), http://dbpedia.org/resource/Samvel_Yervinyan, http://dbpedia.org/resource/Karen_Briggs_(musician), http://dbpedia.org/resource/Leslie_Mills, http://dbpedia.org/resource/Víctor_Espínola, http://dbpedia.org/resource/Joel_Taylor_(musician), http://dbpedia.org/resource/Chloe_(American_singer), http://dbpedia.org/resource/Catte_Adams, http://dbpedia.org/resource/Bradley_Joseph, http://dbpedia.org/resource/Mary_Simpson_(violinist), http://dbpedia.org/resource/Ender_Thomas, http://dbpedia.org/resource/Lisa_Lavie, http://dbpedia.org/resource/Lauren_Jelencovich, http://dbpedia.org/resource/Alexander_Zhiroff, http://dbpedia.org/resource/Vanda_&_Young, http://dbpedia.org/resource/Tony_Currenti, http://dbpedia.org/resource/Kevin_Borich, http://dbpedia.org/resource/Fabrizio_Sotti, http://dbpedia.org/resource/E-Dee, http://dbpedia.org/resource/Ida_Corr, http://dbpedia.org/resource/Che'Nelle, http://dbpedia.org/resource/Leo_Aberer, http://dbpedia.org/resource/Rikrok, http://dbpedia.org/resource/Faydee, http://dbpedia.org/resource/Tamer_Hosny, http://dbpedia.org/resource/Brian_and_Tony_Gold, http://dbpedia.org/resource/Kat_DeLuna, http://dbpedia.org/resource/Lumidee, http://dbpedia.org/resource/Andrea_(Bulgarian_singer), http://dbpedia.org/resource/DJ_Montay, http://dbpedia.org/resource/Tarik_%22Rvssian%22_Johnston, http://dbpedia.org/resource/Rupee_(musician), http://dbpedia.org/resource/Lady_Saw, http://dbpedia.org/resource/Una_Healy, http://dbpedia.org/resource/Costi_Ioniță, http://dbpedia.org/resource/Gabry_Ponte, http://dbpedia.org/resource/Ms._Triniti, http://dbpedia.org/resource/Tessanne_Chin, http://dbpedia.org/resource/Rayvon, http://dbpedia.org/resource/Alsou, http://dbpedia.org/resource/Paula_Seling, http://dbpedia.org/resource/Paul_Thompson_(musician), http://dbpedia.org/resource/Splodgenessabounds, http://dbpedia.org/resource/John_Foxx, http://dbpedia.org/resource/Benny_Benassi, http://dbpedia.org/resource/Sophie_Monk, http://dbpedia.org/resource/Bella_Paige, http://dbpedia.org/resource/TJR_(musician), http://dbpedia.org/resource/Graeham_Goble, http://dbpedia.org/resource/Little_River_Band, http://dbpedia.org/resource/Beeb_Birtles, http://dbpedia.org/resource/Gregory_Douglass, http://dbpedia.org/resource/Steve_Wade_(singer), http://dbpedia.org/resource/Onur_Şengül, http://dbpedia.org/resource/Honeyroot, http://dbpedia.org/resource/Martyn_Ware, http://dbpedia.org/resource/Naomi_Thompson, http://dbpedia.org/resource/The_Human_League, http://dbpedia.org/resource/Ian_Craig_Marsh, http://dbpedia.org/resource/Billie_Godfrey, http://dbpedia.org/resource/Luxuria_(band), http://dbpedia.org/resource/Spear_of_Destiny_(band), http://dbpedia.org/resource/Pete_Shelley, http://dbpedia.org/resource/Mansun, http://dbpedia.org/resource/Dave_Formula, http://dbpedia.org/resource/Rachel_Fury, http://dbpedia.org/resource/Guy_Pratt, http://dbpedia.org/resource/Iva_Davies, http://dbpedia.org/resource/Michael_Paynter, http://dbpedia.org/resource/Roger_Mason_(musician), http://dbpedia.org/resource/Paul_Gildea, http://dbpedia.org/resource/Mike_Oldfield, http://dbpedia.org/resource/Liz_Kay, http://dbpedia.org/resource/Cado_Belle, http://dbpedia.org/resource/Simon_Little, http://dbpedia.org/resource/Billy_Mann, http://dbpedia.org/resource/Aaron_Sterling, http://dbpedia.org/resource/Dwight_Twilley, http://dbpedia.org/resource/Tom_Petty, http://dbpedia.org/resource/Marie_Fredriksson, http://dbpedia.org/resource/Marianne_Flynner, http://dbpedia.org/resource/Christoffer_Lundquist, http://dbpedia.org/resource/The_Lonely_Boys, http://dbpedia.org/resource/Per_Gessle, http://dbpedia.org/resource/Gyllene_Tider, http://dbpedia.org/resource/Rod_Argent, http://dbpedia.org/resource/Ilona_(singer), http://dbpedia.org/resource/Pablo_Manavello, http://dbpedia.org/resource/The_Book_of_Knots, http://dbpedia.org/resource/Built_Like_Alaska, http://dbpedia.org/resource/Danger_Mouse_(musician), http://dbpedia.org/resource/Gemma_Hayes, http://dbpedia.org/resource/James_Mercer_(musician), http://dbpedia.org/resource/Vic_Chesnutt, http://dbpedia.org/resource/Steven_Nistor, http://dbpedia.org/resource/PJ_Harvey, http://dbpedia.org/resource/Paul_Watson_(musician), http://dbpedia.org/resource/Mascott, http://dbpedia.org/resource/A_Camp, http://dbpedia.org/resource/Jonathan_Segel, http://dbpedia.org/resource/Piet_Veerman, http://dbpedia.org/resource/Mohammed_Abdu, http://dbpedia.org/resource/Maja_Ivarsson, http://dbpedia.org/resource/Markus_Krunegård, http://dbpedia.org/resource/Teddybears, http://dbpedia.org/resource/Robyn, http://dbpedia.org/resource/Jenny_Morris_(musician), http://dbpedia.org/resource/Rex_Goh, http://dbpedia.org/resource/MC_Shan, http://dbpedia.org/resource/Hurby_Azor, http://dbpedia.org/resource/Joey_Boy, http://dbpedia.org/resource/Flag_of_Convenience_(band), http://dbpedia.org/resource/Steve_Garvey_(musician), http://dbpedia.org/resource/Tim_Gaze, http://dbpedia.org/resource/Jak_Airport, http://dbpedia.org/resource/Sal_Solo, http://dbpedia.org/resource/X-Ray_Spex, http://dbpedia.org/resource/Neo_(UK_band), http://dbpedia.org/resource/The_News_(band), http://dbpedia.org/resource/Lars_Håvard_Haugen, http://dbpedia.org/resource/Tal_(singer), http://dbpedia.org/resource/Matthieu_Mendès, http://dbpedia.org/resource/Chris_Dave, http://dbpedia.org/resource/Angus_Stone, http://dbpedia.org/resource/Julia_Stone, http://dbpedia.org/resource/Moddi, http://dbpedia.org/resource/Blacklistt, http://dbpedia.org/resource/I_Am_Giant, http://dbpedia.org/resource/Smack_(Finnish_band), http://dbpedia.org/resource/ADS_(band), http://dbpedia.org/resource/Tim_Christensen, http://dbpedia.org/resource/Force_of_Evil_(band), http://dbpedia.org/resource/Brats_(band), http://dbpedia.org/resource/Art_Metal_(band), http://dbpedia.org/resource/Hank_Shermann, http://dbpedia.org/resource/Seven_Thorns, http://dbpedia.org/resource/Jeppe_Laursen, http://dbpedia.org/resource/Thomas_Troelsen, http://dbpedia.org/resource/Make_Out_(band), http://dbpedia.org/resource/Susanne_Georgi, http://dbpedia.org/resource/Søren_Huss, http://dbpedia.org/resource/Mads_Langer, http://dbpedia.org/resource/Dizzy_Mizz_Lizzy, http://dbpedia.org/resource/Oregon_(band), http://dbpedia.org/resource/Toni_Willé, http://dbpedia.org/resource/Werner_Theunissen, http://dbpedia.org/resource/Pussycat_(band), http://dbpedia.org/resource/Buddy_Miles, http://dbpedia.org/resource/Mass_Mental, http://dbpedia.org/resource/RJD2, http://dbpedia.org/resource/Self_Jupiter, http://dbpedia.org/resource/Z-Trip, http://dbpedia.org/resource/The_Nonce, http://dbpedia.org/resource/Kenan_Bell, http://dbpedia.org/resource/Moka_Only, http://dbpedia.org/resource/Dilated_Peoples, http://dbpedia.org/resource/Hot_Dollar, http://dbpedia.org/resource/Mario_Winans, http://dbpedia.org/resource/La_Chat, http://dbpedia.org/resource/Lil_Soldiers, http://dbpedia.org/resource/Ghetto_Commission, http://dbpedia.org/resource/Joe_Blakk, http://dbpedia.org/resource/Ghetto_Twiinz, http://dbpedia.org/resource/Gérald_de_Palmas, http://dbpedia.org/resource/Jean-Louis_Aubert, http://dbpedia.org/resource/Deuce_Mob, http://dbpedia.org/resource/Mopreme_Shakur, http://dbpedia.org/resource/Passion_(rapper), http://dbpedia.org/resource/Shock_G, http://dbpedia.org/resource/Too_Short, http://dbpedia.org/resource/Young_Maylay, http://dbpedia.org/resource/MC_Breed, http://dbpedia.org/resource/Totally_Insane, http://dbpedia.org/resource/Ras_Kass, http://dbpedia.org/resource/Chino_XL, http://dbpedia.org/resource/Justine_Ok, http://dbpedia.org/resource/Jon_Redwine, http://dbpedia.org/resource/Audra_the_Rapper, http://dbpedia.org/resource/Jim_Diamond_(music_producer), http://dbpedia.org/resource/The_Anointed_Pace_Sisters, http://dbpedia.org/resource/Utopia_(Brazilian_band), http://dbpedia.org/resource/Igor_Cavalera, http://dbpedia.org/resource/Faber_Drive, http://dbpedia.org/resource/Olivia_Louvel, http://dbpedia.org/resource/Dennis_Bovell, http://dbpedia.org/resource/The_Cimarons, http://dbpedia.org/resource/John_Kpiaye, http://dbpedia.org/resource/Darlings_of_Chelsea, http://dbpedia.org/resource/Ignatius_Jones, http://dbpedia.org/resource/Titãs, http://dbpedia.org/resource/Marisa_Monte, http://dbpedia.org/resource/Carlinhos_Brown, http://dbpedia.org/resource/Arnaldo_Antunes, http://dbpedia.org/resource/Alias_(band), http://dbpedia.org/resource/Arnold_Lanni, http://dbpedia.org/resource/Steve_DeMarchi, http://dbpedia.org/resource/Freddy_Curci, http://dbpedia.org/resource/Sami_Karppinen, http://dbpedia.org/resource/Patrik_Mennander, http://dbpedia.org/resource/Entity_Paradigm, http://dbpedia.org/resource/Javed_Bashir, http://dbpedia.org/resource/Zeb_and_Haniya, http://dbpedia.org/resource/Jon_Asher, http://dbpedia.org/resource/Al_Hurricane, http://dbpedia.org/resource/Manuela_Azevedo, http://dbpedia.org/resource/Clã, http://dbpedia.org/resource/David_Fonseca, http://dbpedia.org/resource/Tír_na_nÓg_(band), http://dbpedia.org/resource/Fiocco_(band), http://dbpedia.org/resource/Bill_Peck, http://dbpedia.org/resource/Projected, http://dbpedia.org/resource/Scott_Phillips_(musician), http://dbpedia.org/resource/Diana_Meltzer, http://dbpedia.org/resource/Creed_(band), http://dbpedia.org/resource/Mark_Tremonti, http://dbpedia.org/resource/Tremonti_(band), http://dbpedia.org/resource/Myles_Kennedy, http://dbpedia.org/resource/Brian_Marshall, http://dbpedia.org/resource/Venefica_(band), http://dbpedia.org/resource/Cristina_Scabbia, http://dbpedia.org/resource/Citizen_Swing, http://dbpedia.org/resource/The_Mayfield_Four, http://dbpedia.org/resource/Nuno_Bettencourt, http://dbpedia.org/resource/Barren_Earth, http://dbpedia.org/resource/Tomi_Joutsen, http://dbpedia.org/resource/Ajattara, http://dbpedia.org/resource/Waltari, http://dbpedia.org/resource/Hiili_Hiilesmaa, http://dbpedia.org/resource/Marco_Hietala, http://dbpedia.org/resource/Stone_(band), http://dbpedia.org/resource/Shape_of_Despair, http://dbpedia.org/resource/Esa_Holopainen, http://dbpedia.org/resource/Abhorrence, http://dbpedia.org/resource/Olli-Pekka_Laine, http://dbpedia.org/resource/Pasi_Koskinen, http://dbpedia.org/resource/Tomi_Koivusaari, http://dbpedia.org/resource/Graham_Stack_(record_producer), http://dbpedia.org/resource/Jenny_Frost, http://dbpedia.org/resource/Heidi_Range, http://dbpedia.org/resource/Robert_%22Kool%22_Bell, http://dbpedia.org/resource/Liz_McClarnon, http://dbpedia.org/resource/Natasha_Hamilton, http://dbpedia.org/resource/Andy_McCluskey, http://dbpedia.org/resource/Paul_Walsham, http://dbpedia.org/resource/Gary_Wallis, http://dbpedia.org/resource/Keke_Wyatt, http://dbpedia.org/resource/I_Wayne, http://dbpedia.org/resource/Don_Yute, http://dbpedia.org/resource/2face_Idibia, http://dbpedia.org/resource/Guerilla_Black, http://dbpedia.org/resource/Qmillion, http://dbpedia.org/resource/Josh_Stevens, http://dbpedia.org/resource/Adrian_Izes_Marshall, http://dbpedia.org/resource/Brooke_Valentine, http://dbpedia.org/resource/Arif_Cooper, http://dbpedia.org/resource/Bounty_Killer, http://dbpedia.org/resource/Reggie_Rockstone, http://dbpedia.org/resource/Zena_McNally, http://dbpedia.org/resource/Hurricane_Chris_(rapper), http://dbpedia.org/resource/Memphis_Bleek, http://dbpedia.org/resource/So_Solid_Crew, http://dbpedia.org/resource/Krept_and_Konan, http://dbpedia.org/resource/Mýa, http://dbpedia.org/resource/Supa_Dups, http://dbpedia.org/resource/Deevani, http://dbpedia.org/resource/Kim_Davis_(Canadian_singer), http://dbpedia.org/resource/Dave_Kelly_(producer), http://dbpedia.org/resource/Full_Blown_Entertainment, http://dbpedia.org/resource/Miranda_(Colombian_singer), http://dbpedia.org/resource/Naëla, http://dbpedia.org/resource/Edgar_Barrera, http://dbpedia.org/resource/Sebastian_Krys, http://dbpedia.org/resource/Sidestepper, http://dbpedia.org/resource/Andrés_Cepeda, http://dbpedia.org/resource/Norrie_Paramor, http://dbpedia.org/resource/Peter_Howarth, http://dbpedia.org/resource/Peter_Gormley, http://dbpedia.org/resource/Tim_Rice, http://dbpedia.org/resource/Hank_Marvin, http://dbpedia.org/resource/The_Shadows, http://dbpedia.org/resource/Mark_Eddinger, http://dbpedia.org/resource/Chris_Eaton_(UK_musician), http://dbpedia.org/resource/Frank_Ifield, http://dbpedia.org/resource/Adam_Faith, http://dbpedia.org/resource/Bruce_Welch, http://dbpedia.org/resource/Brian_Bennett, http://dbpedia.org/resource/The_Hunters_(instrumental_band), http://dbpedia.org/resource/Michael_J._Mullins, http://dbpedia.org/resource/David_Mackay_(producer), http://dbpedia.org/resource/Alan_Tarney, http://dbpedia.org/resource/Brian_Locking, http://dbpedia.org/resource/Trevor_Spencer, http://dbpedia.org/resource/Dori_Caymmi, http://dbpedia.org/resource/Fela_Kuti, http://dbpedia.org/resource/Funk_Trek, http://dbpedia.org/resource/Dele_Sosimi, http://dbpedia.org/resource/Femi_Kuti, http://dbpedia.org/resource/Roy_Ayers, http://dbpedia.org/resource/Seun_Kuti, http://dbpedia.org/resource/Lester_Bowie, http://dbpedia.org/resource/Kari_Jobe, http://dbpedia.org/resource/Mark_Harris_(musician), http://dbpedia.org/resource/Thomas_Miller_(pastor), http://dbpedia.org/resource/Diante_do_Trono, http://dbpedia.org/resource/Rich_Kids, http://dbpedia.org/resource/William_Lee_Self, http://dbpedia.org/resource/Terry_Edwards, http://dbpedia.org/resource/James_Stevenson_(musician), http://dbpedia.org/resource/The_Flying_Padovanis, http://dbpedia.org/resource/The_London_Cowboys, http://dbpedia.org/resource/Acid_Mothers_Temple, http://dbpedia.org/resource/Brian_Davison_(drummer), http://dbpedia.org/resource/Dave_Sturt, http://dbpedia.org/resource/Didier_Malherbe, http://dbpedia.org/resource/Gilli_Smyth, http://dbpedia.org/resource/Miquette_Giraudy, http://dbpedia.org/resource/Pierre_Moerlen's_Gong, http://dbpedia.org/resource/Allan_Holdsworth, http://dbpedia.org/resource/Steve_Hillage, http://dbpedia.org/resource/System_7_(band), http://dbpedia.org/resource/Theo_Travis, http://dbpedia.org/resource/Fabio_Golfetti, http://dbpedia.org/resource/Violeta_de_Outono, http://dbpedia.org/resource/Soft_Machine, http://dbpedia.org/resource/Giorgio_Gomelsky, http://dbpedia.org/resource/Mike_Howlett, http://dbpedia.org/resource/Tim_Blake, http://dbpedia.org/resource/Kavus_Torabi, http://dbpedia.org/resource/Daevid_Allen, http://dbpedia.org/resource/Lady_June, http://dbpedia.org/resource/Poisoned_Electrick_Head, http://dbpedia.org/resource/Spirits_Burning, http://dbpedia.org/resource/Charles_Hayward_(musician), http://dbpedia.org/resource/Mandragora_(band), http://dbpedia.org/resource/Steve_Harris_(musician), http://dbpedia.org/resource/Clive_Burr, http://dbpedia.org/resource/Desperado_(band), http://dbpedia.org/resource/Paul_Di'Anno, http://dbpedia.org/resource/Wisdom_(band), http://dbpedia.org/resource/Lauren_Harris, http://dbpedia.org/resource/The_Entire_Population_of_Hackney, http://dbpedia.org/resource/Hollywood_Monsters_(band), http://dbpedia.org/resource/Adrian_Smith, http://dbpedia.org/resource/Bruce_Dickinson, http://dbpedia.org/resource/Gillan_(band), http://dbpedia.org/resource/Blaze_Bayley, http://dbpedia.org/resource/Lionheart_(band), http://dbpedia.org/resource/Maiden_uniteD, http://dbpedia.org/resource/Streetwalkers, http://dbpedia.org/resource/Rekuiem, http://dbpedia.org/resource/Janick_Gers, http://dbpedia.org/resource/White_Spirit_(band), http://dbpedia.org/resource/Dennis_Stratton, http://dbpedia.org/resource/Praying_Mantis_(band), http://dbpedia.org/resource/More_(British_band), http://dbpedia.org/resource/McCoy_(band), http://dbpedia.org/resource/Nicko_McBrain, http://dbpedia.org/resource/ASAP_(band), http://dbpedia.org/resource/Psycho_Motel, http://dbpedia.org/resource/Urchin_(band), http://dbpedia.org/resource/Severed_Fifth, http://dbpedia.org/resource/Wolfsbane_(band), http://dbpedia.org/resource/Rod_Smallwood, http://dbpedia.org/resource/Primal_Rock_Rebellion, http://dbpedia.org/resource/Di'Anno, http://dbpedia.org/resource/Dave_Murray_(musician), http://dbpedia.org/resource/Doug_Sampson, http://dbpedia.org/resource/Paul_Day_(musician), http://dbpedia.org/resource/The_Iron_Maidens, http://dbpedia.org/resource/Thunderstick, http://dbpedia.org/resource/Rise_to_Remain, http://dbpedia.org/resource/Billy_Cobham, http://dbpedia.org/resource/Mitch_Mitchell, http://dbpedia.org/resource/Tony_Hymas, http://dbpedia.org/resource/Aruba_Red, http://dbpedia.org/resource/Vernon_Reid, http://dbpedia.org/resource/Robin_Trower, http://dbpedia.org/resource/Art_Themen, http://dbpedia.org/resource/Pete_Brown, http://dbpedia.org/resource/Carla_Bley, http://dbpedia.org/resource/Gary_Husband, http://dbpedia.org/resource/Mick_Taylor, http://dbpedia.org/resource/Eddie_Hardin, http://dbpedia.org/resource/Martin_Axenrot, http://dbpedia.org/resource/Colin_Hodgkinson, http://dbpedia.org/resource/Pete_York, http://dbpedia.org/resource/Fabolous, http://dbpedia.org/resource/J.R._Writer, http://dbpedia.org/resource/Red_Café, http://dbpedia.org/resource/Skyzoo, http://dbpedia.org/resource/Olivia_(singer), http://dbpedia.org/resource/Big_Fase_100, http://dbpedia.org/resource/Jim_Jones_(rapper), http://dbpedia.org/resource/DJ_Whoo_Kid, http://dbpedia.org/resource/Nick_Speed, http://dbpedia.org/resource/Dan_Millice, http://dbpedia.org/resource/Juelz_Santana, http://dbpedia.org/resource/Bob_Klose, http://dbpedia.org/resource/Dick_Parry, http://dbpedia.org/resource/Snowy_White, http://dbpedia.org/resource/Peter_Wood_(musician), http://dbpedia.org/resource/Jeff_Porcaro, http://dbpedia.org/resource/Andy_Bown, http://dbpedia.org/resource/Andy_Roberts_(musician), http://dbpedia.org/resource/Tim_Renwick, http://dbpedia.org/resource/Jon_Carin, http://dbpedia.org/resource/Al_Coury, http://dbpedia.org/resource/Roy_Harper, http://dbpedia.org/resource/Ron_Geesin, http://dbpedia.org/resource/Richard_Wright_(musician), http://dbpedia.org/resource/Snovi, http://dbpedia.org/resource/Jay_Stapley, http://dbpedia.org/resource/Andy_Jackson_(recording_engineer), http://dbpedia.org/resource/El_Monstero, http://dbpedia.org/resource/Roger_Waters, http://dbpedia.org/resource/Tony_Levin, http://dbpedia.org/resource/Nick_Mason, http://dbpedia.org/resource/Stars_(British_band), http://dbpedia.org/resource/Durga_McBroom, http://dbpedia.org/resource/Anthony_Moore, http://dbpedia.org/resource/Jan_Dukes_de_Grey, http://dbpedia.org/resource/Bob_Ezrin, http://dbpedia.org/resource/Bob_Geldof, http://dbpedia.org/resource/Bruce_Johnston, http://dbpedia.org/resource/Syd_Barrett, http://dbpedia.org/resource/James_Guthrie_(record_producer), http://dbpedia.org/resource/Scott_Page, http://dbpedia.org/resource/The_Australian_Pink_Floyd_Show, http://dbpedia.org/resource/Doug_Sax, http://dbpedia.org/resource/Candy_Dulfer, http://dbpedia.org/resource/Danny_Peyronel, http://dbpedia.org/resource/Willie_Wilson_(drummer), http://dbpedia.org/resource/Pink_Project, http://dbpedia.org/resource/Easy_Mo_Bee, http://dbpedia.org/resource/Afu-Ra, http://dbpedia.org/resource/Ed_O.G., http://dbpedia.org/resource/La_the_Darkman, http://dbpedia.org/resource/Kosha_Dillz, http://dbpedia.org/resource/Ti2bs, http://dbpedia.org/resource/James_Blake_(musician), http://dbpedia.org/resource/Megahertz_(record_producer), http://dbpedia.org/resource/Tom_Morello, http://dbpedia.org/resource/Che_Pope, http://dbpedia.org/resource/Paul_Banks_(singer), http://dbpedia.org/resource/Curse_(rapper), http://dbpedia.org/resource/Josh_Werner, http://dbpedia.org/resource/Glen_Matlock, http://dbpedia.org/resource/Sex_Pistols, http://dbpedia.org/resource/The_Clash, http://dbpedia.org/resource/Visage_(band), http://dbpedia.org/resource/Midge_Ure, http://dbpedia.org/resource/Vicious_White_Kids, http://dbpedia.org/resource/Slik, http://dbpedia.org/resource/Zones_(band), http://dbpedia.org/resource/Rusty_Egan, http://dbpedia.org/resource/Steve_New, http://dbpedia.org/resource/Shweta_Shetty, http://dbpedia.org/resource/Ennio_Morricone, http://dbpedia.org/resource/Minako_Honda, http://dbpedia.org/resource/Michelle_Leonard, http://dbpedia.org/resource/Olga_Seryabkina, http://dbpedia.org/resource/Anastasia_Karpova, http://dbpedia.org/resource/Elena_Temnikova, http://dbpedia.org/resource/Dr._Shiver, http://dbpedia.org/resource/Marina_Lizorkina, http://dbpedia.org/resource/Maxim_Fadeev, http://dbpedia.org/resource/Georg_Hólm, http://dbpedia.org/resource/Apparat_Organ_Quartet, http://dbpedia.org/resource/Jakobínarína, http://dbpedia.org/resource/Jónsi, http://dbpedia.org/resource/Kjartan_Sveinsson, http://dbpedia.org/resource/Orri_Páll_Dýrason, http://dbpedia.org/resource/Ágúst_Ævar_Gunnarsson, http://dbpedia.org/resource/Instrumenti, http://dbpedia.org/resource/Amiina, http://dbpedia.org/resource/Spirits_and_the_Melchizedek_Children, http://dbpedia.org/resource/Clive_Bunker, http://dbpedia.org/resource/Gong_(band), http://dbpedia.org/resource/John_McKenzie_(musician), http://dbpedia.org/resource/Andy_Anderson_(drummer), http://dbpedia.org/resource/Bluetech, http://dbpedia.org/resource/The_Orb, http://dbpedia.org/resource/Kliché, http://dbpedia.org/resource/Steffen_Brandt, http://dbpedia.org/resource/Robin_Jones_(musician), http://dbpedia.org/resource/The_Beta_Band, http://dbpedia.org/resource/Lone_Pigeon, http://dbpedia.org/resource/Bonzo_Dog_Doo-Dah_Band, http://dbpedia.org/resource/Jeff_Lynne, http://dbpedia.org/resource/The_Rutles, http://dbpedia.org/resource/Allan_Williams, http://dbpedia.org/resource/Freddie_Gorman, http://dbpedia.org/resource/Robert_Fitzpatrick, http://dbpedia.org/resource/Tommy_Quickly, http://dbpedia.org/resource/The_Beatnix, http://dbpedia.org/resource/James_Apollo, http://dbpedia.org/resource/George_Harrison, http://dbpedia.org/resource/Jimmie_Nicol, http://dbpedia.org/resource/Traveling_Wilburys, http://dbpedia.org/resource/John_Kosh, http://dbpedia.org/resource/Billy_Fury, http://dbpedia.org/resource/Billy_J._Kramer, http://dbpedia.org/resource/Klaus_Voormann, http://dbpedia.org/resource/Plastic_Ono_Band, http://dbpedia.org/resource/Billy_Preston, http://dbpedia.org/resource/Jackie_Lee_(Irish_singer), http://dbpedia.org/resource/Kiwi_Time, http://dbpedia.org/resource/Kathy_McCord, http://dbpedia.org/resource/Badfinger, http://dbpedia.org/resource/Brian_Jones, http://dbpedia.org/resource/George_Martin, http://dbpedia.org/resource/Nicky_Hopkins, http://dbpedia.org/resource/The_Ronettes, http://dbpedia.org/resource/Pete_Best, http://dbpedia.org/resource/Ringo_Starr, http://dbpedia.org/resource/The_Dirty_Mac, http://dbpedia.org/resource/Andy_White_(drummer), http://dbpedia.org/resource/Ronnie_Schneider, http://dbpedia.org/resource/Shampoo_(parody_band), http://dbpedia.org/resource/The_Quarrymen, http://dbpedia.org/resource/Tony_Sheridan, http://dbpedia.org/resource/Glyn_Johns, http://dbpedia.org/resource/Ghost_of_the_Robot, http://dbpedia.org/resource/Johnny_Gentle, http://dbpedia.org/resource/Grapefruit_(band), http://dbpedia.org/resource/C._P._Spencer, http://dbpedia.org/resource/Sound_City_Players, http://dbpedia.org/resource/Roy_Young_(musician), http://dbpedia.org/resource/Röyksopp, http://dbpedia.org/resource/Mongrel_(band), http://dbpedia.org/resource/Seb_Rochford, http://dbpedia.org/resource/Danny_Goffey, http://dbpedia.org/resource/Blackcube, http://dbpedia.org/resource/Dirty_Pretty_Things_(band), http://dbpedia.org/resource/The_Bottletop_Band, http://dbpedia.org/resource/Adam_Ficek, http://dbpedia.org/resource/Pete_Doherty, http://dbpedia.org/resource/The_Libertines, http://dbpedia.org/resource/Drew_McConnell, http://dbpedia.org/resource/Littl'ans, http://dbpedia.org/resource/Mick_Whitnall, http://dbpedia.org/resource/Patrick_Walden, http://dbpedia.org/resource/Paul_Mauriat, http://dbpedia.org/resource/Cheb_Tarik, http://dbpedia.org/resource/Jairo_(singer), http://dbpedia.org/resource/Mamas_Gun, http://dbpedia.org/resource/Jeff_Dawson, http://dbpedia.org/resource/Thomas_Lang, http://dbpedia.org/resource/Candy_DeRouge, http://dbpedia.org/resource/The_Von_Bondies, http://dbpedia.org/resource/Arthur_Russell_(musician), http://dbpedia.org/resource/Doug_Beck, http://dbpedia.org/resource/Alisha_(singer), http://dbpedia.org/resource/Cindy_Valentine, http://dbpedia.org/resource/Rascalz, http://dbpedia.org/resource/Ian_Kamau, http://dbpedia.org/resource/Michael_Peter_Olsen, http://dbpedia.org/resource/Rich_Kidd, http://dbpedia.org/resource/Thomas_D'Arcy, http://dbpedia.org/resource/Mike_Ratledge, http://dbpedia.org/resource/Duff_McKagan, http://dbpedia.org/resource/New_American_Shame, http://dbpedia.org/resource/Walking_Papers_(band), http://dbpedia.org/resource/Velvet_Revolver, http://dbpedia.org/resource/Gosling_(band), http://dbpedia.org/resource/Isaac_Carpenter_(drummer), http://dbpedia.org/resource/Fozzy, http://dbpedia.org/resource/The_Gentlemen_(band), http://dbpedia.org/resource/Dave_Dederer, http://dbpedia.org/resource/Nevada_Bachelors, http://dbpedia.org/resource/Taz_Bentley, http://dbpedia.org/resource/Alien_Crime_Syndicate, http://dbpedia.org/resource/Jeff_Rouse_(musician), http://dbpedia.org/resource/Sirens_Sister, http://dbpedia.org/resource/Martin_Feveyear, http://dbpedia.org/resource/Dave_Kushner, http://dbpedia.org/resource/Electric_Love_Hogs, http://dbpedia.org/resource/Plexi, http://dbpedia.org/resource/Geoff_Reading, http://dbpedia.org/resource/Mike_Squires_(musician), http://dbpedia.org/resource/Vendetta_Red, http://dbpedia.org/resource/Green_Apple_Quick_Step, http://dbpedia.org/resource/Jade_Trini_Goring, http://dbpedia.org/resource/D.J._D, http://dbpedia.org/resource/MC_Tee, http://dbpedia.org/resource/Angie_Stone, http://dbpedia.org/resource/Kurtis_Mantronik, http://dbpedia.org/resource/Bryce_Wilson, http://dbpedia.org/resource/Fera_(band), http://dbpedia.org/resource/John_Young_(British_musician), http://dbpedia.org/resource/Cruella_de_Ville, http://dbpedia.org/resource/Keats_(band), http://dbpedia.org/resource/Jim_Cregan, http://dbpedia.org/resource/Barry_Wickens, http://dbpedia.org/resource/Duncan_Mackay_(musician), http://dbpedia.org/resource/Steve_Harley, http://dbpedia.org/resource/Linda_Lewis, http://dbpedia.org/resource/Steve_Strange, http://dbpedia.org/resource/The_Moors_Murderers, http://dbpedia.org/resource/The_Photons, http://dbpedia.org/resource/Guy_Gerber, http://dbpedia.org/resource/Roberto_Casalino, http://dbpedia.org/resource/Ensiferum, http://dbpedia.org/resource/Sinergy, http://dbpedia.org/resource/Barathrum, http://dbpedia.org/resource/Roope_Latvala, http://dbpedia.org/resource/Kreator, http://dbpedia.org/resource/Sami_Yli-Sirniö, http://dbpedia.org/resource/Ashley_Naylor, http://dbpedia.org/resource/Michael_Armiger, http://dbpedia.org/resource/Alan_Barton, http://dbpedia.org/resource/Terry_Dobson_(singer), http://dbpedia.org/resource/Tadeusz_Nalepa, http://dbpedia.org/resource/Stars_(Australian_band), http://dbpedia.org/resource/Simon_Binks, http://dbpedia.org/resource/MaJiKer, http://dbpedia.org/resource/Kanata_Okajima, http://dbpedia.org/resource/City_of_Fire_(band), http://dbpedia.org/resource/Brent_Fitz, http://dbpedia.org/resource/Rhys_Fulber, http://dbpedia.org/resource/Ken_Fleming_(musician), http://dbpedia.org/resource/Greg_Williamson_(drummer), http://dbpedia.org/resource/The_Melodics, http://dbpedia.org/resource/Ghostwriters, http://dbpedia.org/resource/Richard_Grossman_(bassist), http://dbpedia.org/resource/Dubrovniks, http://dbpedia.org/resource/Dave_Faulkner_(musician), http://dbpedia.org/resource/The_Manikins, http://dbpedia.org/resource/The_Victims_(Australian_band), http://dbpedia.org/resource/The_Johnnys, http://dbpedia.org/resource/Fun_Things, http://dbpedia.org/resource/XL_Capris, http://dbpedia.org/resource/The_Dearhunters, http://dbpedia.org/resource/Beasts_of_Bourbon, http://dbpedia.org/resource/Even_(band), http://dbpedia.org/resource/James_Baker_(musician), http://dbpedia.org/resource/Brad_Shepherd, http://dbpedia.org/resource/Roddy_Radalj, http://dbpedia.org/resource/Steve_Balbi, http://dbpedia.org/resource/Nik_Kershaw, http://dbpedia.org/resource/Simon_Hill_(musician), http://dbpedia.org/resource/Jörn-Uwe_Fahrenkrog-Petersen, http://dbpedia.org/resource/Julia_Fordham, http://dbpedia.org/resource/Eric_B._&_Rakim, http://dbpedia.org/resource/MF_Grimm, http://dbpedia.org/resource/Rakim, http://dbpedia.org/resource/Cormega, http://dbpedia.org/resource/Large_Professor, http://dbpedia.org/resource/Akinyele_(rapper), http://dbpedia.org/resource/Rena_Scott, http://dbpedia.org/resource/Ray_Parker,_Jr., http://dbpedia.org/resource/Sylvia_Robinson, http://dbpedia.org/resource/Flávio_Venturini, http://dbpedia.org/resource/Cabaret_Voltaire_(band), http://dbpedia.org/resource/Dieter_Bohlen, http://dbpedia.org/resource/Chris_Norman, http://dbpedia.org/resource/Mickey_Finn's_T-Rex, http://dbpedia.org/resource/Mighty_Kong, http://dbpedia.org/resource/Sons_of_the_Vegetal_Mother, http://dbpedia.org/resource/Ray_Arnott, http://dbpedia.org/resource/The_Captain_Matchbox_Whoopee_Band, http://dbpedia.org/resource/The_Dingoes, http://dbpedia.org/resource/Mike_Rudd, http://dbpedia.org/resource/Mark_Kennedy_(musician), http://dbpedia.org/resource/Spider_Loc, http://dbpedia.org/resource/Danny_Brown_(rapper), http://dbpedia.org/resource/Mr._Porter, http://dbpedia.org/resource/DJ_Kay_Slay, http://dbpedia.org/resource/Jagged_Edge_(group), http://dbpedia.org/resource/Simon_Webbe, http://dbpedia.org/resource/David_A._Martin, http://dbpedia.org/resource/John_Farrar, http://dbpedia.org/resource/Christoph_Brüx, http://dbpedia.org/resource/Die_So_Fluid, http://dbpedia.org/resource/Whitey_(band), http://dbpedia.org/resource/Eduardo_Mateo, http://dbpedia.org/resource/Silvana_Kane, http://dbpedia.org/resource/Spectrum_(band), http://dbpedia.org/resource/Harvey_James, http://dbpedia.org/resource/Broderick_Smith, http://dbpedia.org/resource/David_Arden, http://dbpedia.org/resource/John_Schumann_and_the_Vagabond_Crew, http://dbpedia.org/resource/Bart_Willoughby, http://dbpedia.org/resource/Rose_Bygrave, http://dbpedia.org/resource/Graham_Davidge, http://dbpedia.org/resource/Shane_Howard, http://dbpedia.org/resource/Vandalism_(band), http://dbpedia.org/resource/Sally_Seltmann, http://dbpedia.org/resource/The_Librettos_(band), http://dbpedia.org/resource/Tin_Tin_(band), http://dbpedia.org/resource/Jordan_Young, http://dbpedia.org/resource/Mike_Ruocco, http://dbpedia.org/resource/SR-71_(band), http://dbpedia.org/resource/MC_Hammer, http://dbpedia.org/resource/James_Worthy_(producer), http://dbpedia.org/resource/The_Dandy_Warhols, http://dbpedia.org/resource/The_Gentry_(band), http://dbpedia.org/resource/Hangman_Jury_(band), http://dbpedia.org/resource/Paul_Phillips_(guitarist), http://dbpedia.org/resource/Jay_de_la_Cueva, http://dbpedia.org/resource/Fobia, http://dbpedia.org/resource/Pete_King_(British_musician), http://dbpedia.org/resource/The_Lover_Speaks, http://dbpedia.org/resource/Chaotic_Dischord, http://dbpedia.org/resource/Paul_Rooney, http://dbpedia.org/resource/Beki_Bondage, http://dbpedia.org/resource/Alan_Murphy, http://dbpedia.org/resource/Graham_Hopkins, http://dbpedia.org/resource/The_Duckworth_Lewis_Method, http://dbpedia.org/resource/Engine_Alley, http://dbpedia.org/resource/Lennon_&_Maisy, http://dbpedia.org/resource/Jaclyn_Victor, http://dbpedia.org/resource/Cindy_Gomez, http://dbpedia.org/resource/Andrea_Figallo, http://dbpedia.org/resource/Vikash_Maharaj, http://dbpedia.org/resource/Mike_Pender's_Searchers, http://dbpedia.org/resource/Lord_Large, http://dbpedia.org/resource/Mike_Elliott_(saxophonist), http://dbpedia.org/resource/Eric_Allandale, http://dbpedia.org/resource/The_Ramong_Sound, http://dbpedia.org/resource/Alan_Warner_(musician), http://dbpedia.org/resource/Alex_Hope_(songwriter), http://dbpedia.org/resource/Bison_B.C., http://dbpedia.org/resource/3_Inches_of_Blood, http://dbpedia.org/resource/Liset_Alea, http://dbpedia.org/resource/Tash_(rapper), http://dbpedia.org/resource/Cabine_C, http://dbpedia.org/resource/Smack_(Brazilian_band), http://dbpedia.org/resource/On_An_On, http://dbpedia.org/resource/Tanzplagen, http://dbpedia.org/resource/James_Iha, http://dbpedia.org/resource/Black_Light_Burns, http://dbpedia.org/resource/Green_Jellÿ, http://dbpedia.org/resource/Maynard_James_Keenan, http://dbpedia.org/resource/Puscifer, http://dbpedia.org/resource/Tool_(band), http://dbpedia.org/resource/Danny_Lohner, http://dbpedia.org/resource/Enemy_(American_band), http://dbpedia.org/resource/Jeordie_White, http://dbpedia.org/resource/Ashes_Divide, http://dbpedia.org/resource/Nine_Inch_Nails, http://dbpedia.org/resource/Tim_Alexander, http://dbpedia.org/resource/Josh_Freese, http://dbpedia.org/resource/Goon_Moon, http://dbpedia.org/resource/Tinted_Windows_(band), http://dbpedia.org/resource/Zwan, http://dbpedia.org/resource/List_of_Nine_Inch_Nails_band_members, http://dbpedia.org/resource/Tapeworm_(band), http://dbpedia.org/resource/Children_of_the_Anachronistic_Dynasty, http://dbpedia.org/resource/Skrew, http://dbpedia.org/resource/Billy_Howerdel, http://dbpedia.org/resource/Paz_Lenchantin, http://dbpedia.org/resource/Failure_(band), http://dbpedia.org/resource/The_Damning_Well, http://dbpedia.org/resource/Sweethead, http://dbpedia.org/resource/Troy_Van_Leeuwen, http://dbpedia.org/resource/Matt_McJunkins, http://dbpedia.org/resource/Herbie_Crichlow, http://dbpedia.org/resource/Rino_Gaetano, http://dbpedia.org/resource/Ray_Brown_(musician), http://dbpedia.org/resource/Alex_Elena, http://dbpedia.org/resource/Pete_Friesen, http://dbpedia.org/resource/Nick_Fyffe, http://dbpedia.org/resource/Chris_Dale, http://dbpedia.org/resource/Sack_Trick, http://dbpedia.org/resource/Dave_Moreno, http://dbpedia.org/resource/Atom_Seed, http://dbpedia.org/resource/Roy_Z, http://dbpedia.org/resource/Tribe_of_Gypsies, http://dbpedia.org/resource/Chorão, http://dbpedia.org/resource/Planet_Hemp, http://dbpedia.org/resource/Simon_Climie, http://dbpedia.org/resource/Fred_Åkerström, http://dbpedia.org/resource/Víctor_Jara, http://dbpedia.org/resource/Cocteau_Twins, http://dbpedia.org/resource/Roots_Manuva, http://dbpedia.org/resource/Martina_Topley-Bird, http://dbpedia.org/resource/Ibrahim_Ferrer, http://dbpedia.org/resource/Lou_Reed, http://dbpedia.org/resource/Damon_Albarn, http://dbpedia.org/resource/Del_the_Funky_Homosapien, http://dbpedia.org/resource/Shaun_Ryder, http://dbpedia.org/resource/Mick_Jones_(The_Clash_guitarist), http://dbpedia.org/resource/Senseless_Things, http://dbpedia.org/resource/Cibo_Matto, http://dbpedia.org/resource/Miho_Hatori, http://dbpedia.org/resource/Paul_Simonon, http://dbpedia.org/resource/Tina_Weymouth, http://dbpedia.org/resource/Bashy, http://dbpedia.org/resource/Bobby_Womack, http://dbpedia.org/resource/Cass_Browne, http://dbpedia.org/resource/Dan_the_Automator, http://dbpedia.org/resource/De_La_Soul, http://dbpedia.org/resource/James_Murphy_(electronic_musician), http://dbpedia.org/resource/Jeff_Wootton, http://dbpedia.org/resource/Kano_(rapper), http://dbpedia.org/resource/Kid_Koala, http://dbpedia.org/resource/Morgan_Nicholls, http://dbpedia.org/resource/Simon_Katz, http://dbpedia.org/resource/Simon_Tong, http://dbpedia.org/resource/Terry_Hall_(singer), http://dbpedia.org/resource/Spacemonkeyz, http://dbpedia.org/resource/Massive_Attack, http://dbpedia.org/resource/Little_Dragon, http://dbpedia.org/resource/Yukimi_Nagano, http://dbpedia.org/resource/The_Pharcyde, http://dbpedia.org/resource/Demon_Strings, http://dbpedia.org/resource/DRC_Music, http://dbpedia.org/resource/The_PSM, http://dbpedia.org/resource/John_Philip_Shenale, http://dbpedia.org/resource/Class_of_'99, http://dbpedia.org/resource/Red_Hot_Chili_Peppers, http://dbpedia.org/resource/Infectious_Grooves, http://dbpedia.org/resource/Taylor_Hawkins_and_the_Coattail_Riders, http://dbpedia.org/resource/Satellite_Party, http://dbpedia.org/resource/Kommunity_FK, http://dbpedia.org/resource/Axis_of_Justice, http://dbpedia.org/resource/Martyn_LeNoble, http://dbpedia.org/resource/Porno_for_Pyros, http://dbpedia.org/resource/Thelonious_Monster, http://dbpedia.org/resource/Dave_Navarro, http://dbpedia.org/resource/Stephen_Perkins, http://dbpedia.org/resource/Deconstruction_(band), http://dbpedia.org/resource/Eric_Avery, http://dbpedia.org/resource/Camp_Freddy, http://dbpedia.org/resource/Tex_&_the_Horseheads, http://dbpedia.org/resource/Flea_(musician), http://dbpedia.org/resource/The_Panic_Channel, http://dbpedia.org/resource/Psi_Com, http://dbpedia.org/resource/Aaron_Embry, http://dbpedia.org/resource/Linda_Good, http://dbpedia.org/resource/Perry_Farrell, http://dbpedia.org/resource/Dave_Sitek, http://dbpedia.org/resource/Rich_Costey, http://dbpedia.org/resource/The_Birds_of_Satan, http://dbpedia.org/resource/Chris_Chaney, http://dbpedia.org/resource/Paul_Kodish, http://dbpedia.org/resource/Adrian_Belew, http://dbpedia.org/resource/Vechigen_(band), http://dbpedia.org/resource/Harry_Nilsson, http://dbpedia.org/resource/Rob_May, http://dbpedia.org/resource/Manuel_Vargas_(singer), http://dbpedia.org/resource/Alejandro_Lerner, http://dbpedia.org/resource/Sergio_Rozenblat, http://dbpedia.org/resource/Jack_Bruce, http://dbpedia.org/resource/The_Blues_Band, http://dbpedia.org/resource/The_Manfreds, http://dbpedia.org/resource/Ellie_Greenwich, http://dbpedia.org/resource/Manfred_Mann_(musician), http://dbpedia.org/resource/Manfred_Mann_Chapter_Three, http://dbpedia.org/resource/Paul_Jones_(singer), http://dbpedia.org/resource/Warren_Zevon, http://dbpedia.org/resource/Dave_Richmond, http://dbpedia.org/resource/A_Band_of_Angels, http://dbpedia.org/resource/McGuinness_Flint, http://dbpedia.org/resource/Mike_d'Abo, http://dbpedia.org/resource/Eric_Clapton_and_the_Powerhouse, http://dbpedia.org/resource/Submersed, http://dbpedia.org/resource/Dave_Holland, http://dbpedia.org/resource/Oscar_Pettiford, http://dbpedia.org/resource/Clark_Terry, http://dbpedia.org/resource/Tony_Williams_(drummer), http://dbpedia.org/resource/Gary_Peacock, http://dbpedia.org/resource/Horace_Silver, http://dbpedia.org/resource/Joe_Zawinul, http://dbpedia.org/resource/Kenny_Garrett, http://dbpedia.org/resource/Lonnie_Liston_Smith, http://dbpedia.org/resource/Bill_Barber_(musician), http://dbpedia.org/resource/George_Coleman, http://dbpedia.org/resource/Pete_Cosey, http://dbpedia.org/resource/Bernard_Purdie, http://dbpedia.org/resource/John_Beasley_(musician), http://dbpedia.org/resource/Harold_Ivory_Williams, http://dbpedia.org/resource/Charlie_Parker, http://dbpedia.org/resource/Gerry_Mulligan, http://dbpedia.org/resource/Percy_Heath, http://dbpedia.org/resource/Sonny_Sharrock, http://dbpedia.org/resource/Wynton_Kelly, http://dbpedia.org/resource/Bennie_Maupin, http://dbpedia.org/resource/Michael_Brecker, http://dbpedia.org/resource/Mtume, http://dbpedia.org/resource/Teo_Macero, http://dbpedia.org/resource/Al_Foster, http://dbpedia.org/resource/Bill_Evans_(saxophonist), http://dbpedia.org/resource/Darryl_Jones, http://dbpedia.org/resource/Dave_Liebman, http://dbpedia.org/resource/Don_Alias, http://dbpedia.org/resource/George_Avakian, http://dbpedia.org/resource/James_Mtume, http://dbpedia.org/resource/Jimmy_Heath, http://dbpedia.org/resource/Kei_Akagi, http://dbpedia.org/resource/Michael_Henderson, http://dbpedia.org/resource/Mino_Cinelu, http://dbpedia.org/resource/Monte_Kay, http://dbpedia.org/resource/Reggie_Lucas, http://dbpedia.org/resource/Tommy_Potter, http://dbpedia.org/resource/Bo_Stief, http://dbpedia.org/resource/Airto_Moreira, http://dbpedia.org/resource/Bill_Evans, http://dbpedia.org/resource/Cannonball_Adderley, http://dbpedia.org/resource/Charles_Mingus, http://dbpedia.org/resource/Max_Roach, http://dbpedia.org/resource/Red_Garland, http://dbpedia.org/resource/Wayne_Shorter, http://dbpedia.org/resource/Bud_Powell, http://dbpedia.org/resource/Keith_Jarrett, http://dbpedia.org/resource/Sonny_Rollins, http://dbpedia.org/resource/John_McLaughlin_(musician), http://dbpedia.org/resource/John_Bernard_Riley]... [INFO] P: 0.397423887587822, R: 1.0, F: 0.5687950393832747 -[INFO] Cumulative metrics, sample: 337, P: 0.535, R: 0.592, macro F1: 0.543, macro F1*: 0.562 -[INFO] Current time: 4147 ms, average: 2832.36 ms/ques, entity detection: 424.79 ms/ques (linking tool: 334.18 ms/ques), relation detection: 917.42 ms/ques (similarity: 710.20 ms/ques), EDG generation: 60.31 ms/ques +[INFO] Cumulative metrics, sample: 337, P: 0.535, R: 0.592, macro F1: 0.562 +[INFO] Current question consumed time: 3783 ms, average: 2626.25 ms/ques, entity detection: 390.07 ms/ques (linking tool: 308.50 ms/ques), relation detection: 870.89 ms/ques (similarity: 697.37 ms/ques), EDG generation: 60.04 ms/ques [INFO] Entity Linking P: 0.7403560830860534, R: 0.7032640949554896, micro F1: 0.7146389713155289, macro F1: 0.7213335732431434 [INFO] Relation Linking P: 0.5984174085064292, R: 0.5712166172106825, micro F1: 0.5762611275964389, macro F1: 0.5845007245876751 [INFO] Type Linking P: 0.543026706231454, R: 0.543026706231454, micro F1: 0.543026706231454, macro F1: 0.543026706231454 @@ -4730,8 +4730,8 @@ Question 337: Name the river which flows through the closest city of Mount Raime [INFO] Golden Answer: [http://dbpedia.org/resource/Hudson_River] [INFO] Predicted Answer: [http://dbpedia.org/resource/Hudson_River] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 338, P: 0.536, R: 0.593, macro F1: 0.544, macro F1*: 0.563 -[INFO] Current time: 2817 ms, average: 2832.31 ms/ques, entity detection: 424.05 ms/ques (linking tool: 333.46 ms/ques), relation detection: 918.11 ms/ques (similarity: 711.63 ms/ques), EDG generation: 60.42 ms/ques +[INFO] Cumulative metrics, sample: 338, P: 0.536, R: 0.593, macro F1: 0.563 +[INFO] Current question consumed time: 2582 ms, average: 2626.12 ms/ques, entity detection: 389.38 ms/ques (linking tool: 307.83 ms/ques), relation detection: 871.46 ms/ques (similarity: 698.84 ms/ques), EDG generation: 60.18 ms/ques [INFO] Entity Linking P: 0.7411242603550295, R: 0.7041420118343196, micro F1: 0.7154832347140037, macro F1: 0.7221599752896501 [INFO] Relation Linking P: 0.5996055226824457, R: 0.5724852071005917, micro F1: 0.577514792899408, macro F1: 0.5857316044041393 [INFO] Type Linking P: 0.5414201183431953, R: 0.5414201183431953, micro F1: 0.5414201183431953, macro F1: 0.5414201183431953 @@ -4744,8 +4744,8 @@ Question 338: Name the scientist whose supervisor also supervised Mary Ainsworth [INFO] Golden Answer: [http://dbpedia.org/resource/Abraham_Maslow] [INFO] Predicted Answer: [http://dbpedia.org/resource/Harry_Harlow] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 339, P: 0.535, R: 0.592, macro F1: 0.542, macro F1*: 0.562 -[INFO] Current time: 1226 ms, average: 2827.58 ms/ques, entity detection: 423.37 ms/ques (linking tool: 332.78 ms/ques), relation detection: 916.46 ms/ques (similarity: 710.83 ms/ques), EDG generation: 60.31 ms/ques +[INFO] Cumulative metrics, sample: 339, P: 0.535, R: 0.592, macro F1: 0.562 +[INFO] Current question consumed time: 1080 ms, average: 2621.56 ms/ques, entity detection: 388.71 ms/ques (linking tool: 307.16 ms/ques), relation detection: 869.88 ms/ques (similarity: 698.04 ms/ques), EDG generation: 60.06 ms/ques [INFO] Entity Linking P: 0.7418879056047197, R: 0.7050147492625368, micro F1: 0.7163225172074726, macro F1: 0.7229814859919592 [INFO] Relation Linking P: 0.5978367748279252, R: 0.5707964601769911, micro F1: 0.5758112094395278, macro F1: 0.5840037825622392 [INFO] Type Linking P: 0.5398230088495575, R: 0.5398230088495575, micro F1: 0.5398230088495575, macro F1: 0.5398230088495575 @@ -4758,8 +4758,8 @@ Question 339: What are some relatives of armymen who died in Germany? [INFO] Golden Answer: [http://dbpedia.org/resource/Ferdinand_Sauerbruch, http://dbpedia.org/resource/Werner_Friebe, http://dbpedia.org/resource/Helmut_Friebe, http://dbpedia.org/resource/Caesar_von_Hofacker, http://dbpedia.org/resource/Erich_von_Manstein, http://dbpedia.org/resource/Gottfried_von_Cramm, http://dbpedia.org/resource/Kurt_Freiherr_von_Plettenberg, http://dbpedia.org/resource/Egon_Freiherr_von_Gayl, http://dbpedia.org/resource/Franz_Freiherr_von_Gayl, http://dbpedia.org/resource/Albrecht_Lanz, http://dbpedia.org/resource/Günther_von_Kluge, http://dbpedia.org/resource/Fritz_Schlieper, http://dbpedia.org/resource/Günther_Lützow, http://dbpedia.org/resource/Friedrich_Lützow, http://dbpedia.org/resource/Friedrich_Foertsch, http://dbpedia.org/resource/Ludwig_von_Schröder_(general), http://dbpedia.org/resource/Werner_Haugk, http://dbpedia.org/resource/Wolfgang_von_Gronau, http://dbpedia.org/resource/Kurt_Pflugbeil, http://dbpedia.org/resource/Rüdiger_von_Wechmar, http://dbpedia.org/resource/Kurt_von_der_Chevallerie, http://dbpedia.org/resource/Kurt_von_Tippelskirch, http://dbpedia.org/resource/Fridolin_von_Senger_und_Etterlin, http://dbpedia.org/resource/Werner_von_Fritsch, http://dbpedia.org/resource/Johann_Pflugbeil, http://dbpedia.org/resource/Generalleutnant, http://dbpedia.org/resource/Anthony_Fokker, http://dbpedia.org/resource/Heinrich_Himmler, http://dbpedia.org/resource/Erhard_Milch] [INFO] Predicted Answer: [http://dbpedia.org/resource/Hans_Scholl, http://dbpedia.org/resource/Inge_Scholl, http://dbpedia.org/resource/Otl_Aicher, http://dbpedia.org/resource/Valerie_Niehaus, http://dbpedia.org/resource/Rosa_Porten, http://dbpedia.org/resource/Ferdinand_Anton_Danneskiold-Laurvig, http://dbpedia.org/resource/Paul_Mauser, http://dbpedia.org/resource/Cornelius_Gurlitt_(composer), http://dbpedia.org/resource/Cornelius_Gurlitt_(art_collector), http://dbpedia.org/resource/Eduard_Einstein, http://dbpedia.org/resource/Elsa_Einstein, http://dbpedia.org/resource/Hans_Albert_Einstein, http://dbpedia.org/resource/Einstein_family, http://dbpedia.org/resource/Sophie_Scholl, http://dbpedia.org/resource/Aaron_Altaras, http://dbpedia.org/resource/Yon_Tumarkin, http://dbpedia.org/resource/Peter_Busch_Orthwein, http://dbpedia.org/resource/Adolphus_Busch_III, http://dbpedia.org/resource/James_Orthwein, http://dbpedia.org/resource/Robert_Ringen_Hermann,_Jr., http://dbpedia.org/resource/Adolphus_Busch_Orthwein, http://dbpedia.org/resource/Gussie_Busch, http://dbpedia.org/resource/Stephen_A._Orthwein, http://dbpedia.org/resource/Karl_Albrecht, http://dbpedia.org/resource/Jacques_Breuer, http://dbpedia.org/resource/Victor_Ehrenberg_(historian), http://dbpedia.org/resource/Leo_Rudolf_Raubal,_Jr., http://dbpedia.org/resource/Shakir_ul_Qadri, http://dbpedia.org/resource/John_Mulholland,_1st_Baron_Dunleath, http://dbpedia.org/resource/Henny_Porten, http://dbpedia.org/resource/Fanny_Lewald, http://dbpedia.org/resource/Albert_Einstein, http://dbpedia.org/resource/Rudolf_August_Oetker, http://dbpedia.org/resource/Leonard_Altaras, http://dbpedia.org/resource/Ruth_Baldor, http://dbpedia.org/resource/Jaroslav_Evans_Albrecht, http://dbpedia.org/resource/Karl_Marx, http://dbpedia.org/resource/Fridolin_Keller, http://dbpedia.org/resource/Friedrich_Barényi, http://dbpedia.org/resource/Seraphin_Keller, http://dbpedia.org/resource/Cornelius_Gurlitt_(art_historian), http://dbpedia.org/resource/Geoffrey_Elton, http://dbpedia.org/resource/Ben_Elton, http://dbpedia.org/resource/Lewis_Elton, http://dbpedia.org/resource/Hitler_family, http://dbpedia.org/resource/Ph._D._Ivor_Altaras_Penda] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 340, P: 0.533, R: 0.590, macro F1: 0.541, macro F1*: 0.560 -[INFO] Current time: 23422 ms, average: 2888.15 ms/ques, entity detection: 437.56 ms/ques (linking tool: 346.91 ms/ques), relation detection: 939.29 ms/ques (similarity: 728.11 ms/ques), EDG generation: 60.19 ms/ques +[INFO] Cumulative metrics, sample: 340, P: 0.533, R: 0.590, macro F1: 0.560 +[INFO] Current question consumed time: 19298 ms, average: 2670.61 ms/ques, entity detection: 400.82 ms/ques (linking tool: 319.26 ms/ques), relation detection: 885.40 ms/ques (similarity: 709.78 ms/ques), EDG generation: 59.94 ms/ques [INFO] Entity Linking P: 0.7426470588235294, R: 0.7058823529411765, micro F1: 0.7171568627450977, macro F1: 0.7237981487011048 [INFO] Relation Linking P: 0.596078431372549, R: 0.5691176470588235, micro F1: 0.5741176470588233, macro F1: 0.5822861243782327 [INFO] Type Linking P: 0.538235294117647, R: 0.538235294117647, micro F1: 0.538235294117647, macro F1: 0.538235294117647 @@ -4772,8 +4772,8 @@ Question 340: Deion sanders and Hinkey haines debuted their career with which co [INFO] Golden Answer: [http://dbpedia.org/resource/New_York_Yankees] [INFO] Predicted Answer: [http://dbpedia.org/resource/New_York_Yankees] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 341, P: 0.534, R: 0.591, macro F1: 0.542, macro F1*: 0.561 -[INFO] Current time: 2718 ms, average: 2887.65 ms/ques, entity detection: 437.03 ms/ques (linking tool: 346.29 ms/ques), relation detection: 939.27 ms/ques (similarity: 729.02 ms/ques), EDG generation: 60.30 ms/ques +[INFO] Cumulative metrics, sample: 341, P: 0.534, R: 0.591, macro F1: 0.561 +[INFO] Current question consumed time: 2468 ms, average: 2670.01 ms/ques, entity detection: 400.27 ms/ques (linking tool: 318.65 ms/ques), relation detection: 885.40 ms/ques (similarity: 710.83 ms/ques), EDG generation: 60.06 ms/ques [INFO] Entity Linking P: 0.7434017595307918, R: 0.7067448680351907, micro F1: 0.717986314760508, macro F1: 0.724610006256505 [INFO] Relation Linking P: 0.5972629521016618, R: 0.5703812316715543, micro F1: 0.5753665689149557, macro F1: 0.583512653915981 [INFO] Type Linking P: 0.5395894428152492, R: 0.5395894428152492, micro F1: 0.5395894428152492, macro F1: 0.5395894428152492 @@ -4786,8 +4786,8 @@ Question 341: Which garrison of East Indies Station is also the route end of Mum [INFO] Golden Answer: [http://dbpedia.org/resource/Mumbai] [INFO] Predicted Answer: [http://dbpedia.org/resource/Aden, http://dbpedia.org/resource/Colombo, http://dbpedia.org/resource/Mumbai, http://dbpedia.org/resource/Basra, http://dbpedia.org/resource/Trincomalee, Colombo, Trincomalee, Bombay, Basra and Aden@en] [INFO] P: 0.16666666666666666, R: 1.0, F: 0.2857142857142857 -[INFO] Cumulative metrics, sample: 342, P: 0.533, R: 0.592, macro F1: 0.541, macro F1*: 0.561 -[INFO] Current time: 3050 ms, average: 2888.12 ms/ques, entity detection: 438.21 ms/ques (linking tool: 347.05 ms/ques), relation detection: 937.34 ms/ques (similarity: 727.37 ms/ques), EDG generation: 60.43 ms/ques +[INFO] Cumulative metrics, sample: 342, P: 0.533, R: 0.592, macro F1: 0.561 +[INFO] Current question consumed time: 2742 ms, average: 2670.23 ms/ques, entity detection: 401.37 ms/ques (linking tool: 319.42 ms/ques), relation detection: 883.58 ms/ques (similarity: 709.24 ms/ques), EDG generation: 60.18 ms/ques [INFO] Entity Linking P: 0.7441520467836257, R: 0.706140350877193, micro F1: 0.7178362573099412, macro F1: 0.7246480616864741 [INFO] Relation Linking P: 0.5984405458089669, R: 0.5701754385964912, micro F1: 0.5756335282651069, macro F1: 0.5839661706392755 [INFO] Type Linking P: 0.5409356725146199, R: 0.5409356725146199, micro F1: 0.5409356725146199, macro F1: 0.5409356725146199 @@ -4800,8 +4800,8 @@ Question 342: To which political party does Virendra Kataria belongs ? [INFO] Golden Answer: [http://dbpedia.org/resource/Indian_National_Congress] [INFO] Predicted Answer: [http://dbpedia.org/resource/Indian_National_Congress] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 343, P: 0.535, R: 0.593, macro F1: 0.543, macro F1*: 0.563 -[INFO] Current time: 873 ms, average: 2882.25 ms/ques, entity detection: 437.35 ms/ques (linking tool: 346.27 ms/ques), relation detection: 935.27 ms/ques (similarity: 725.92 ms/ques), EDG generation: 60.29 ms/ques +[INFO] Cumulative metrics, sample: 343, P: 0.535, R: 0.593, macro F1: 0.563 +[INFO] Current question consumed time: 776 ms, average: 2664.70 ms/ques, entity detection: 400.55 ms/ques (linking tool: 318.69 ms/ques), relation detection: 881.62 ms/ques (similarity: 707.83 ms/ques), EDG generation: 60.04 ms/ques [INFO] Entity Linking P: 0.7448979591836735, R: 0.706997084548105, micro F1: 0.7186588921282796, macro F1: 0.7254528317351037 [INFO] Relation Linking P: 0.5996112730806608, R: 0.5714285714285714, micro F1: 0.5768707482993194, macro F1: 0.5851807943094249 [INFO] Type Linking P: 0.5393586005830904, R: 0.5393586005830904, micro F1: 0.5393586005830904, macro F1: 0.5393586005830904 @@ -4814,8 +4814,8 @@ Question 343: Copley Medal has been awarded to which scientists? [INFO] Golden Answer: [http://dbpedia.org/resource/Andre_Geim, http://dbpedia.org/resource/Albert_Einstein, http://dbpedia.org/resource/Alessandro_Volta, http://dbpedia.org/resource/Andrew_Huxley, http://dbpedia.org/resource/Carl_Friedrich_Gauss, http://dbpedia.org/resource/Charles_Darwin, http://dbpedia.org/resource/James_Watson, http://dbpedia.org/resource/Louis_Pasteur, http://dbpedia.org/resource/Max_Planck, http://dbpedia.org/resource/Niels_Bohr, http://dbpedia.org/resource/Roger_Penrose, http://dbpedia.org/resource/Simon_Newcomb, http://dbpedia.org/resource/Stephen_Hawking, http://dbpedia.org/resource/Theodor_Schwann, http://dbpedia.org/resource/Robin_Hill_(biochemist), http://dbpedia.org/resource/Henri_Milne-Edwards, http://dbpedia.org/resource/Arthur_Schuster, http://dbpedia.org/resource/Paul_Dirac, http://dbpedia.org/resource/Stephen_Gray_(scientist), http://dbpedia.org/resource/William_Watson_(scientist), http://dbpedia.org/resource/John_Smeaton, http://dbpedia.org/resource/Charles_Scott_Sherrington, http://dbpedia.org/resource/John_Pringle, http://dbpedia.org/resource/Josiah_Willard_Gibbs, http://dbpedia.org/resource/Rudolf_Virchow, http://dbpedia.org/resource/Edward_Waring, http://dbpedia.org/resource/Stephen_Hales, http://dbpedia.org/resource/Wilhelm_Eduard_Weber, http://dbpedia.org/resource/Jean-Baptiste_Boussingault, http://dbpedia.org/resource/Charles-Adolphe_Wurtz, http://dbpedia.org/resource/Henry_Kater, http://dbpedia.org/resource/Albert_Kluyver] [INFO] Predicted Answer: [http://dbpedia.org/resource/Dorothy_Hodgkin, http://dbpedia.org/resource/James_MacCullagh, http://dbpedia.org/resource/John_Edensor_Littlewood, http://dbpedia.org/resource/Peter_D._Mitchell, http://dbpedia.org/resource/Paul_Langevin, http://dbpedia.org/resource/William_Bayliss, http://dbpedia.org/resource/David_Cox_(statistician), http://dbpedia.org/resource/G._I._Taylor, http://dbpedia.org/resource/John_Couch_Adams, http://dbpedia.org/resource/Andre_Geim, http://dbpedia.org/resource/W._V._D._Hodge, http://dbpedia.org/resource/Abraham_Trembley, http://dbpedia.org/resource/Derek_Barton, http://dbpedia.org/resource/Edward_Charles_Howard, http://dbpedia.org/resource/Stanislao_Cannizzaro, http://dbpedia.org/resource/Abdus_Salam, http://dbpedia.org/resource/Albert_Einstein, http://dbpedia.org/resource/Alessandro_Volta, http://dbpedia.org/resource/Andrew_Huxley, http://dbpedia.org/resource/August_Wilhelm_von_Hofmann, http://dbpedia.org/resource/Carl_Friedrich_Gauss, http://dbpedia.org/resource/Charles_Darwin, http://dbpedia.org/resource/Charles_Wheatstone, http://dbpedia.org/resource/David_Brewster, http://dbpedia.org/resource/Eduard_Suess, http://dbpedia.org/resource/Frederick_Sanger, http://dbpedia.org/resource/Friedrich_Wöhler, http://dbpedia.org/resource/George_de_Hevesy, http://dbpedia.org/resource/Hans_Christian_Ørsted, http://dbpedia.org/resource/Hermann_von_Helmholtz, http://dbpedia.org/resource/Humphry_Davy, http://dbpedia.org/resource/Ivan_Pavlov, http://dbpedia.org/resource/James_Bradley, http://dbpedia.org/resource/James_Chadwick, http://dbpedia.org/resource/James_Watson, http://dbpedia.org/resource/John_Harrison, http://dbpedia.org/resource/John_William_Strutt,_3rd_Baron_Rayleigh, http://dbpedia.org/resource/Jöns_Jacob_Berzelius, http://dbpedia.org/resource/Karl_Weierstrass, http://dbpedia.org/resource/Louis_Pasteur, http://dbpedia.org/resource/Max_Planck, http://dbpedia.org/resource/Michael_Faraday, http://dbpedia.org/resource/Niels_Bohr, http://dbpedia.org/resource/Robert_Bunsen, http://dbpedia.org/resource/Roger_Penrose, http://dbpedia.org/resource/Simon_Newcomb, http://dbpedia.org/resource/Sir_George_Stokes,_1st_Baronet, http://dbpedia.org/resource/Stephen_Hawking, http://dbpedia.org/resource/Subrahmanyan_Chandrasekhar, http://dbpedia.org/resource/Theodor_Schwann, http://dbpedia.org/resource/Thomas_Henry_Huxley, http://dbpedia.org/resource/William_Crookes, http://dbpedia.org/resource/William_Herschel, http://dbpedia.org/resource/William_Hyde_Wollaston, http://dbpedia.org/resource/Hugh_Huxley, http://dbpedia.org/resource/John_Theophilus_Desaguliers, http://dbpedia.org/resource/Alan_R._Battersby, http://dbpedia.org/resource/César_Milstein, http://dbpedia.org/resource/John_Canton, http://dbpedia.org/resource/John_Mudge, http://dbpedia.org/resource/Robin_Hill_(biochemist), http://dbpedia.org/resource/Henri_Milne-Edwards, http://dbpedia.org/resource/Jesse_Ramsden, http://dbpedia.org/resource/Julius_von_Mayer, http://dbpedia.org/resource/Adam_Sedgwick, http://dbpedia.org/resource/Alexander_von_Humboldt, http://dbpedia.org/resource/Arthur_Cayley, http://dbpedia.org/resource/Arthur_Schuster, http://dbpedia.org/resource/August_Kekulé, http://dbpedia.org/resource/Benjamin_Thompson, http://dbpedia.org/resource/Carl_Ludwig, http://dbpedia.org/resource/Charles_Lyell, http://dbpedia.org/resource/Felix_Klein, http://dbpedia.org/resource/Georg_Ohm, http://dbpedia.org/resource/George_Biddell_Airy, http://dbpedia.org/resource/George_Ellery_Hale, http://dbpedia.org/resource/James_Prescott_Joule, http://dbpedia.org/resource/John_Goodricke, http://dbpedia.org/resource/John_Herschel, http://dbpedia.org/resource/John_Pond, http://dbpedia.org/resource/Joseph_Dalton_Hooker, http://dbpedia.org/resource/Karl_Gegenbaur, http://dbpedia.org/resource/Marcellin_Berthelot, http://dbpedia.org/resource/Michel_Eugène_Chevreul, http://dbpedia.org/resource/Nevil_Maskelyne, http://dbpedia.org/resource/Nevill_Francis_Mott, http://dbpedia.org/resource/Paul_Dirac, http://dbpedia.org/resource/Richard_Owen, http://dbpedia.org/resource/Rudolf_Clausius, http://dbpedia.org/resource/David_Keilin, http://dbpedia.org/resource/Max_Perutz, http://dbpedia.org/resource/William_Buckland, http://dbpedia.org/resource/William_Roy, http://dbpedia.org/resource/Élie_Metchnikoff, http://dbpedia.org/resource/Albert_von_Kölliker, http://dbpedia.org/resource/Charles_Algernon_Parsons, http://dbpedia.org/resource/John_Frederic_Daniell, http://dbpedia.org/resource/Patrick_Blackett, http://dbpedia.org/resource/Peter_Higgs, http://dbpedia.org/resource/Stephen_Gray_(scientist), http://dbpedia.org/resource/Dan_McKenzie_(geophysicist), http://dbpedia.org/resource/Joseph_Larmor, http://dbpedia.org/resource/Rodney_Robert_Porter, http://dbpedia.org/resource/Henry_Baker_(naturalist), http://dbpedia.org/resource/Frederick_Charles_Frank, http://dbpedia.org/resource/Aaron_Klug, http://dbpedia.org/resource/Bernard_Katz, http://dbpedia.org/resource/Charles_Hutton, http://dbpedia.org/resource/Claude_Bernard, http://dbpedia.org/resource/E._T._Whittaker, http://dbpedia.org/resource/Edward_Albert_Sharpey-Schafer, http://dbpedia.org/resource/Francis_Crick, http://dbpedia.org/resource/George_Darwin, http://dbpedia.org/resource/Hendrik_Lorentz, http://dbpedia.org/resource/Horace_Lamb, http://dbpedia.org/resource/John_Maynard_Smith, http://dbpedia.org/resource/Léon_Foucault, http://dbpedia.org/resource/Oswald_Avery, http://dbpedia.org/resource/Robert_Burns_Woodward, http://dbpedia.org/resource/Sydney_Brenner, http://dbpedia.org/resource/William_Watson_(scientist), http://dbpedia.org/resource/Charles_Blagden, http://dbpedia.org/resource/James_Ivory_(mathematician), http://dbpedia.org/resource/James_Joseph_Sylvester, http://dbpedia.org/resource/Jean-Baptiste_Dumas, http://dbpedia.org/resource/John_Smeaton, http://dbpedia.org/resource/Peter_Barlow_(mathematician), http://dbpedia.org/resource/Roderick_Murchison, http://dbpedia.org/resource/Thomas_Graham_(chemist), http://dbpedia.org/resource/William_Prout, http://dbpedia.org/resource/Alec_Jeffreys, http://dbpedia.org/resource/Jacques_Charles_François_Sturm, http://dbpedia.org/resource/John_Cornforth, http://dbpedia.org/resource/Michel_Chasles, http://dbpedia.org/resource/Peter_Andreas_Hansen, http://dbpedia.org/resource/Pierre_Paul_Émile_Roux, http://dbpedia.org/resource/Antoine_César_Becquerel, http://dbpedia.org/resource/Edward_Delaval, http://dbpedia.org/resource/Charles_Scott_Sherrington, http://dbpedia.org/resource/Frederick_Gowland_Hopkins, http://dbpedia.org/resource/John_Pringle, http://dbpedia.org/resource/Josiah_Willard_Gibbs, http://dbpedia.org/resource/Rudolf_Virchow, http://dbpedia.org/resource/Thomas_Hunt_Morgan, http://dbpedia.org/resource/William_Hewson_(surgeon), http://dbpedia.org/resource/William_Huggins, http://dbpedia.org/resource/William_Lawrence_Bragg, http://dbpedia.org/resource/Albert_A._Michelson, http://dbpedia.org/resource/Carlo_Matteucci, http://dbpedia.org/resource/Edward_Waring, http://dbpedia.org/resource/Henri_Victor_Regnault, http://dbpedia.org/resource/Julius_Plücker, http://dbpedia.org/resource/Norman_Pirie, http://dbpedia.org/resource/Stephen_Hales, http://dbpedia.org/resource/Tomas_Lindahl, http://dbpedia.org/resource/James_Lighthill, http://dbpedia.org/resource/G._H._Hardy, http://dbpedia.org/resource/Paul_Fildes, http://dbpedia.org/resource/Wilhelm_Eduard_Weber, http://dbpedia.org/resource/Franz_Ernst_Neumann, http://dbpedia.org/resource/Jean-Baptiste_Boussingault, http://dbpedia.org/resource/Peter_Medawar, http://dbpedia.org/resource/John_Dollond, http://dbpedia.org/resource/Astley_Cooper, http://dbpedia.org/resource/Charles-Adolphe_Wurtz, http://dbpedia.org/resource/Henry_Kater, http://dbpedia.org/resource/Peter_Woulfe, http://dbpedia.org/resource/Giovanni_Antonio_Amedeo_Plana, http://dbpedia.org/resource/Alan_Cottrell, http://dbpedia.org/resource/George_Salmon, http://dbpedia.org/resource/William_Henry_(chemist), http://dbpedia.org/resource/Theobald_Smith, http://dbpedia.org/resource/Albert_Kluyver] [INFO] P: 0.18497109826589594, R: 1.0, F: 0.3121951219512195 -[INFO] Cumulative metrics, sample: 344, P: 0.534, R: 0.595, macro F1: 0.542, macro F1*: 0.562 -[INFO] Current time: 756 ms, average: 2876.07 ms/ques, entity detection: 436.69 ms/ques (linking tool: 345.65 ms/ques), relation detection: 932.97 ms/ques (similarity: 724.15 ms/ques), EDG generation: 60.15 ms/ques +[INFO] Cumulative metrics, sample: 344, P: 0.534, R: 0.595, macro F1: 0.562 +[INFO] Current question consumed time: 655 ms, average: 2658.86 ms/ques, entity detection: 399.90 ms/ques (linking tool: 318.08 ms/ques), relation detection: 879.45 ms/ques (similarity: 706.10 ms/ques), EDG generation: 59.90 ms/ques [INFO] Entity Linking P: 0.7456395348837209, R: 0.7078488372093024, micro F1: 0.7194767441860462, macro F1: 0.7262529069767443 [INFO] Relation Linking P: 0.5978682170542635, R: 0.5697674418604651, micro F1: 0.575193798449612, macro F1: 0.5834796873492232 [INFO] Type Linking P: 0.5377906976744186, R: 0.5377906976744186, micro F1: 0.5377906976744186, macro F1: 0.5377906976744186 @@ -4828,8 +4828,8 @@ Question 344: What is the total number of awards whose presenter's headquarters [INFO] Golden Answer: [29] [INFO] Predicted Answer: [1] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 345, P: 0.532, R: 0.593, macro F1: 0.541, macro F1*: 0.561 -[INFO] Current time: 7340 ms, average: 2889.01 ms/ques, entity detection: 440.41 ms/ques (linking tool: 349.28 ms/ques), relation detection: 932.55 ms/ques (similarity: 723.94 ms/ques), EDG generation: 60.10 ms/ques +[INFO] Cumulative metrics, sample: 345, P: 0.532, R: 0.593, macro F1: 0.561 +[INFO] Current question consumed time: 6394 ms, average: 2669.69 ms/ques, entity detection: 403.10 ms/ques (linking tool: 321.24 ms/ques), relation detection: 879.14 ms/ques (similarity: 706.01 ms/ques), EDG generation: 59.85 ms/ques [INFO] Entity Linking P: 0.7434782608695653, R: 0.7057971014492753, micro F1: 0.7173913043478257, macro F1: 0.7241478260869565 [INFO] Relation Linking P: 0.596135265700483, R: 0.5681159420289855, micro F1: 0.5735265700483089, macro F1: 0.5817884418786458 [INFO] Type Linking P: 0.5362318840579711, R: 0.5362318840579711, micro F1: 0.5362318840579711, macro F1: 0.5362318840579711 @@ -4842,8 +4842,8 @@ Question 345: Who is the narrator of The Price of Beauty ? [INFO] Golden Answer: [http://dbpedia.org/resource/Jessica_Simpson] [INFO] Predicted Answer: [http://dbpedia.org/resource/Jessica_Simpson] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 346, P: 0.533, R: 0.594, macro F1: 0.542, macro F1*: 0.562 -[INFO] Current time: 1001 ms, average: 2883.55 ms/ques, entity detection: 439.68 ms/ques (linking tool: 348.70 ms/ques), relation detection: 930.55 ms/ques (similarity: 722.27 ms/ques), EDG generation: 59.95 ms/ques +[INFO] Cumulative metrics, sample: 346, P: 0.533, R: 0.594, macro F1: 0.562 +[INFO] Current question consumed time: 960 ms, average: 2664.75 ms/ques, entity detection: 402.43 ms/ques (linking tool: 320.68 ms/ques), relation detection: 877.34 ms/ques (similarity: 704.41 ms/ques), EDG generation: 59.70 ms/ques [INFO] Entity Linking P: 0.7442196531791907, R: 0.7066473988439307, micro F1: 0.7182080924855488, macro F1: 0.7249470326785343 [INFO] Relation Linking P: 0.5973025048169557, R: 0.569364161849711, micro F1: 0.5747591522157993, macro F1: 0.5829988114728668 [INFO] Type Linking P: 0.5375722543352601, R: 0.5375722543352601, micro F1: 0.5375722543352601, macro F1: 0.5375722543352601 @@ -4856,8 +4856,8 @@ Question 346: Does Dan River flow into the Jordan Valley? [INFO] Golden Answer: [true] [INFO] Predicted Answer: [false] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 347, P: 0.532, R: 0.592, macro F1: 0.540, macro F1*: 0.561 -[INFO] Current time: 3111 ms, average: 2884.20 ms/ques, entity detection: 439.94 ms/ques (linking tool: 348.71 ms/ques), relation detection: 930.27 ms/ques (similarity: 721.59 ms/ques), EDG generation: 59.80 ms/ques +[INFO] Cumulative metrics, sample: 347, P: 0.532, R: 0.592, macro F1: 0.561 +[INFO] Current question consumed time: 2946 ms, average: 2665.56 ms/ques, entity detection: 402.69 ms/ques (linking tool: 320.72 ms/ques), relation detection: 877.10 ms/ques (similarity: 703.78 ms/ques), EDG generation: 59.56 ms/ques [INFO] Entity Linking P: 0.7420749279538905, R: 0.7046109510086456, micro F1: 0.716138328530259, macro F1: 0.7228578481463197 [INFO] Relation Linking P: 0.595581171950048, R: 0.5677233429394812, micro F1: 0.5731027857829007, macro F1: 0.5813186996242417 [INFO] Type Linking P: 0.5389048991354467, R: 0.5389048991354467, micro F1: 0.5389048991354467, macro F1: 0.5389048991354467 @@ -4870,8 +4870,8 @@ Question 347: Who is the producer of the device which is the operating system of [INFO] Golden Answer: [http://dbpedia.org/resource/Apple_Inc., http://dbpedia.org/resource/Foxconn, http://dbpedia.org/resource/Sony, http://dbpedia.org/resource/Asus, http://dbpedia.org/resource/Wistron_Corporation, http://dbpedia.org/resource/Celestica, http://dbpedia.org/resource/Flextronics, http://dbpedia.org/resource/Google] [INFO] Predicted Answer: [http://dbpedia.org/resource/Now!_(Other_Dimensions_In_Music_album)] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 348, P: 0.530, R: 0.591, macro F1: 0.539, macro F1*: 0.559 -[INFO] Current time: 1083 ms, average: 2879.03 ms/ques, entity detection: 438.96 ms/ques (linking tool: 347.84 ms/ques), relation detection: 928.74 ms/ques (similarity: 720.66 ms/ques), EDG generation: 59.74 ms/ques +[INFO] Cumulative metrics, sample: 348, P: 0.530, R: 0.591, macro F1: 0.559 +[INFO] Current question consumed time: 1018 ms, average: 2660.82 ms/ques, entity detection: 401.76 ms/ques (linking tool: 319.91 ms/ques), relation detection: 875.69 ms/ques (similarity: 702.91 ms/ques), EDG generation: 59.50 ms/ques [INFO] Entity Linking P: 0.7399425287356322, R: 0.7025862068965517, micro F1: 0.7140804597701146, macro F1: 0.7207806704217612 [INFO] Relation Linking P: 0.5938697318007663, R: 0.5660919540229885, micro F1: 0.5714559386973177, macro F1: 0.5796482435908389 [INFO] Type Linking P: 0.5402298850574713, R: 0.5402298850574713, micro F1: 0.5402298850574713, macro F1: 0.5402298850574713 @@ -4884,8 +4884,8 @@ Question 348: How many artists have collaborated with famous guitar players? [INFO] Golden Answer: [161] [INFO] Predicted Answer: [1] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 349, P: 0.529, R: 0.589, macro F1: 0.537, macro F1*: 0.557 -[INFO] Current time: 1978 ms, average: 2876.45 ms/ques, entity detection: 438.13 ms/ques (linking tool: 347.14 ms/ques), relation detection: 927.33 ms/ques (similarity: 719.89 ms/ques), EDG generation: 59.60 ms/ques +[INFO] Cumulative metrics, sample: 349, P: 0.529, R: 0.589, macro F1: 0.557 +[INFO] Current question consumed time: 1838 ms, average: 2658.46 ms/ques, entity detection: 400.98 ms/ques (linking tool: 319.26 ms/ques), relation detection: 874.33 ms/ques (similarity: 702.19 ms/ques), EDG generation: 59.36 ms/ques [INFO] Entity Linking P: 0.7378223495702005, R: 0.7005730659025788, micro F1: 0.7120343839541544, macro F1: 0.7187153962944782 [INFO] Relation Linking P: 0.5921680993314231, R: 0.5644699140401146, micro F1: 0.5698185291308497, macro F1: 0.5779873603713808 [INFO] Type Linking P: 0.5386819484240688, R: 0.5386819484240688, micro F1: 0.5386819484240688, macro F1: 0.5386819484240688 @@ -4898,8 +4898,8 @@ Question 349: How many politicians live in India? [INFO] Golden Answer: [171] [INFO] Predicted Answer: [129, 75] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 350, P: 0.527, R: 0.587, macro F1: 0.536, macro F1*: 0.556 -[INFO] Current time: 22963 ms, average: 2933.84 ms/ques, entity detection: 459.08 ms/ques (linking tool: 368.04 ms/ques), relation detection: 941.80 ms/ques (similarity: 729.56 ms/ques), EDG generation: 59.45 ms/ques +[INFO] Cumulative metrics, sample: 350, P: 0.527, R: 0.587, macro F1: 0.556 +[INFO] Current question consumed time: 21383 ms, average: 2711.96 ms/ques, entity detection: 420.41 ms/ques (linking tool: 338.64 ms/ques), relation detection: 888.03 ms/ques (similarity: 711.91 ms/ques), EDG generation: 59.21 ms/ques [INFO] Entity Linking P: 0.7385714285714285, R: 0.7014285714285714, micro F1: 0.7128571428571425, macro F1: 0.7195209750566893 [INFO] Relation Linking P: 0.5933333333333333, R: 0.5657142857142857, micro F1: 0.5710476190476187, macro F1: 0.5791947411668035 [INFO] Type Linking P: 0.5371428571428571, R: 0.5371428571428571, micro F1: 0.5371428571428571, macro F1: 0.5371428571428571 @@ -4912,8 +4912,8 @@ Question 350: What is the base currency of the country which operates the Marion [INFO] Golden Answer: [http://dbpedia.org/resource/Euro] [INFO] Predicted Answer: [http://dbpedia.org/resource/Euro] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 351, P: 0.529, R: 0.588, macro F1: 0.537, macro F1*: 0.557 -[INFO] Current time: 1529 ms, average: 2929.84 ms/ques, entity detection: 458.29 ms/ques (linking tool: 367.27 ms/ques), relation detection: 940.87 ms/ques (similarity: 728.90 ms/ques), EDG generation: 59.37 ms/ques +[INFO] Cumulative metrics, sample: 351, P: 0.529, R: 0.588, macro F1: 0.557 +[INFO] Current question consumed time: 1408 ms, average: 2708.25 ms/ques, entity detection: 419.64 ms/ques (linking tool: 337.92 ms/ques), relation detection: 887.12 ms/ques (similarity: 711.30 ms/ques), EDG generation: 59.13 ms/ques [INFO] Entity Linking P: 0.7393162393162394, R: 0.7022792022792023, micro F1: 0.7136752136752134, macro F1: 0.7203219485828182 [INFO] Relation Linking P: 0.5944919278252612, R: 0.5669515669515669, micro F1: 0.5722697056030386, macro F1: 0.5803952263478019 [INFO] Type Linking P: 0.5356125356125356, R: 0.5356125356125356, micro F1: 0.5356125356125356, macro F1: 0.5356125356125356 @@ -4926,8 +4926,8 @@ Question 351: What is the region served by the governor of Stephen L.R. McNichol [INFO] Golden Answer: [http://dbpedia.org/resource/Colorado] [INFO] Predicted Answer: [http://dbpedia.org/resource/Colorado] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 352, P: 0.530, R: 0.590, macro F1: 0.538, macro F1*: 0.558 -[INFO] Current time: 1513 ms, average: 2925.81 ms/ques, entity detection: 457.38 ms/ques (linking tool: 366.48 ms/ques), relation detection: 939.96 ms/ques (similarity: 727.91 ms/ques), EDG generation: 59.49 ms/ques +[INFO] Cumulative metrics, sample: 352, P: 0.530, R: 0.590, macro F1: 0.558 +[INFO] Current question consumed time: 1367 ms, average: 2704.44 ms/ques, entity detection: 418.80 ms/ques (linking tool: 337.18 ms/ques), relation detection: 886.26 ms/ques (similarity: 710.36 ms/ques), EDG generation: 59.28 ms/ques [INFO] Entity Linking P: 0.7400568181818182, R: 0.703125, micro F1: 0.714488636363636, macro F1: 0.7211183562992126 [INFO] Relation Linking P: 0.5956439393939393, R: 0.5681818181818182, micro F1: 0.5734848484848482, macro F1: 0.5815888749167838 [INFO] Type Linking P: 0.5369318181818182, R: 0.5369318181818182, micro F1: 0.5369318181818182, macro F1: 0.5369318181818182 @@ -4940,8 +4940,8 @@ Question 352: Who operates Liverpool Playhouse ? [INFO] Golden Answer: [http://dbpedia.org/resource/Liverpool] [INFO] Predicted Answer: [Liverpool and Merseyside Theatres Trust@en] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 353, P: 0.529, R: 0.588, macro F1: 0.537, macro F1*: 0.557 -[INFO] Current time: 1422 ms, average: 2921.56 ms/ques, entity detection: 457.93 ms/ques (linking tool: 367.08 ms/ques), relation detection: 937.86 ms/ques (similarity: 726.19 ms/ques), EDG generation: 59.33 ms/ques +[INFO] Cumulative metrics, sample: 353, P: 0.529, R: 0.588, macro F1: 0.557 +[INFO] Current question consumed time: 1286 ms, average: 2700.42 ms/ques, entity detection: 419.29 ms/ques (linking tool: 337.73 ms/ques), relation detection: 884.27 ms/ques (similarity: 708.69 ms/ques), EDG generation: 59.12 ms/ques [INFO] Entity Linking P: 0.7407932011331445, R: 0.7039660056657224, micro F1: 0.7152974504249289, macro F1: 0.7219102371826919 [INFO] Relation Linking P: 0.5939565627950897, R: 0.56657223796034, micro F1: 0.5718602455146361, macro F1: 0.5799413143646115 [INFO] Type Linking P: 0.5382436260623229, R: 0.5382436260623229, micro F1: 0.5382436260623229, macro F1: 0.5382436260623229 @@ -4954,8 +4954,8 @@ Question 353: Who has composed songs produced by Terry Melcher? [INFO] Golden Answer: [http://dbpedia.org/resource/Bob_Dylan] [INFO] Predicted Answer: [http://dbpedia.org/resource/Bob_Dylan] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 354, P: 0.530, R: 0.589, macro F1: 0.538, macro F1*: 0.558 -[INFO] Current time: 1844 ms, average: 2918.51 ms/ques, entity detection: 457.18 ms/ques (linking tool: 366.32 ms/ques), relation detection: 937.43 ms/ques (similarity: 725.72 ms/ques), EDG generation: 59.20 ms/ques +[INFO] Cumulative metrics, sample: 354, P: 0.530, R: 0.589, macro F1: 0.558 +[INFO] Current question consumed time: 1714 ms, average: 2697.64 ms/ques, entity detection: 418.61 ms/ques (linking tool: 337.02 ms/ques), relation detection: 883.96 ms/ques (similarity: 708.27 ms/ques), EDG generation: 58.99 ms/ques [INFO] Entity Linking P: 0.7415254237288136, R: 0.7048022598870056, micro F1: 0.7161016949152539, macro F1: 0.7226976297669492 [INFO] Relation Linking P: 0.5951035781544256, R: 0.5677966101694916, micro F1: 0.5730696798493405, macro F1: 0.5811294860358197 [INFO] Type Linking P: 0.536723163841808, R: 0.536723163841808, micro F1: 0.536723163841808, macro F1: 0.536723163841808 @@ -4968,8 +4968,8 @@ Question 354: To which series does Lost special belong to ? [INFO] Golden Answer: [http://dbpedia.org/resource/Sherlock_Holmes] [INFO] Predicted Answer: [http://dbpedia.org/resource/Lost_(TV_series)] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 355, P: 0.528, R: 0.587, macro F1: 0.537, macro F1*: 0.556 -[INFO] Current time: 1107 ms, average: 2913.41 ms/ques, entity detection: 456.13 ms/ques (linking tool: 365.39 ms/ques), relation detection: 935.92 ms/ques (similarity: 724.89 ms/ques), EDG generation: 59.05 ms/ques +[INFO] Cumulative metrics, sample: 355, P: 0.528, R: 0.587, macro F1: 0.556 +[INFO] Current question consumed time: 1003 ms, average: 2692.86 ms/ques, entity detection: 417.63 ms/ques (linking tool: 336.15 ms/ques), relation detection: 882.50 ms/ques (similarity: 707.48 ms/ques), EDG generation: 58.85 ms/ques [INFO] Entity Linking P: 0.7394366197183099, R: 0.7028169014084507, micro F1: 0.7140845070422532, macro F1: 0.7206618617957747 [INFO] Relation Linking P: 0.596244131455399, R: 0.5690140845070423, micro F1: 0.5742723004694833, macro F1: 0.5823109486897209 [INFO] Type Linking P: 0.5380281690140845, R: 0.5380281690140845, micro F1: 0.5380281690140845, macro F1: 0.5380281690140845 @@ -4982,8 +4982,8 @@ Question 355: How many albums did the maker of The Band produce? [INFO] Golden Answer: [25] [INFO] Predicted Answer: [7, 5, 2] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 356, P: 0.527, R: 0.586, macro F1: 0.535, macro F1*: 0.555 -[INFO] Current time: 4127 ms, average: 2916.82 ms/ques, entity detection: 455.20 ms/ques (linking tool: 364.55 ms/ques), relation detection: 936.44 ms/ques (similarity: 725.44 ms/ques), EDG generation: 59.12 ms/ques +[INFO] Cumulative metrics, sample: 356, P: 0.527, R: 0.586, macro F1: 0.555 +[INFO] Current question consumed time: 3664 ms, average: 2695.59 ms/ques, entity detection: 416.72 ms/ques (linking tool: 335.36 ms/ques), relation detection: 882.89 ms/ques (similarity: 708.07 ms/ques), EDG generation: 58.92 ms/ques [INFO] Entity Linking P: 0.7401685393258427, R: 0.7036516853932584, micro F1: 0.7148876404494379, macro F1: 0.7214483233506754 [INFO] Relation Linking P: 0.5945692883895131, R: 0.5674157303370787, micro F1: 0.5726591760299623, macro F1: 0.5806752437776711 [INFO] Type Linking P: 0.5365168539325843, R: 0.5365168539325843, micro F1: 0.5365168539325843, macro F1: 0.5365168539325843 @@ -4996,8 +4996,8 @@ Question 356: List few musical artist whose notable instruments are Ludwig Drums [INFO] Golden Answer: [http://dbpedia.org/resource/Alfredo_Hernández, http://dbpedia.org/resource/Ringo_Starr, http://dbpedia.org/resource/Karen_Carpenter] [INFO] Predicted Answer: [http://dbpedia.org/resource/Alfredo_Hernández, http://dbpedia.org/resource/Ringo_Starr, http://dbpedia.org/resource/Karen_Carpenter] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 357, P: 0.528, R: 0.587, macro F1: 0.536, macro F1*: 0.556 -[INFO] Current time: 4284 ms, average: 2920.65 ms/ques, entity detection: 456.47 ms/ques (linking tool: 365.52 ms/ques), relation detection: 937.26 ms/ques (similarity: 727.28 ms/ques), EDG generation: 59.16 ms/ques +[INFO] Cumulative metrics, sample: 357, P: 0.528, R: 0.587, macro F1: 0.556 +[INFO] Current question consumed time: 3721 ms, average: 2698.46 ms/ques, entity detection: 417.80 ms/ques (linking tool: 336.16 ms/ques), relation detection: 883.67 ms/ques (similarity: 709.94 ms/ques), EDG generation: 58.95 ms/ques [INFO] Entity Linking P: 0.7408963585434174, R: 0.7044817927170869, micro F1: 0.7156862745098036, macro F1: 0.7222303650142228 [INFO] Relation Linking P: 0.5957049486461251, R: 0.5672268907563025, micro F1: 0.5729225023342667, macro F1: 0.5811172321196645 [INFO] Type Linking P: 0.5350140056022409, R: 0.5350140056022409, micro F1: 0.5350140056022409, macro F1: 0.5350140056022409 @@ -5010,8 +5010,8 @@ Question 357: How many rivers start in an English speaking country? [INFO] Golden Answer: [525] [INFO] Predicted Answer: [4] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 358, P: 0.527, R: 0.585, macro F1: 0.535, macro F1*: 0.555 -[INFO] Current time: 1017 ms, average: 2915.33 ms/ques, entity detection: 455.71 ms/ques (linking tool: 364.82 ms/ques), relation detection: 935.15 ms/ques (similarity: 725.68 ms/ques), EDG generation: 59.02 ms/ques +[INFO] Cumulative metrics, sample: 358, P: 0.527, R: 0.585, macro F1: 0.555 +[INFO] Current question consumed time: 876 ms, average: 2693.37 ms/ques, entity detection: 417.03 ms/ques (linking tool: 335.49 ms/ques), relation detection: 881.67 ms/ques (similarity: 708.39 ms/ques), EDG generation: 58.82 ms/ques [INFO] Entity Linking P: 0.7388268156424581, R: 0.702513966480447, micro F1: 0.7136871508379885, macro F1: 0.7202129617599932 [INFO] Relation Linking P: 0.5940409683426443, R: 0.5656424581005587, micro F1: 0.5713221601489755, macro F1: 0.57949399962771 [INFO] Type Linking P: 0.5335195530726257, R: 0.5335195530726257, micro F1: 0.5335195530726257, macro F1: 0.5335195530726257 @@ -5024,8 +5024,8 @@ Question 358: For a position in which institution were both Jack Ryan and Robert [INFO] Golden Answer: [http://dbpedia.org/resource/United_States_Senate] [INFO] Predicted Answer: [] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 359, P: 0.525, R: 0.584, macro F1: 0.533, macro F1*: 0.553 -[INFO] Current time: 3005 ms, average: 2915.58 ms/ques, entity detection: 456.68 ms/ques (linking tool: 365.36 ms/ques), relation detection: 934.58 ms/ques (similarity: 725.71 ms/ques), EDG generation: 59.08 ms/ques +[INFO] Cumulative metrics, sample: 359, P: 0.525, R: 0.584, macro F1: 0.553 +[INFO] Current question consumed time: 2756 ms, average: 2693.55 ms/ques, entity detection: 417.81 ms/ques (linking tool: 335.93 ms/ques), relation detection: 881.10 ms/ques (similarity: 708.43 ms/ques), EDG generation: 58.86 ms/ques [INFO] Entity Linking P: 0.7367688022284122, R: 0.7005571030640668, micro F1: 0.7116991643454036, macro F1: 0.7182067975211072 [INFO] Relation Linking P: 0.5923862581244197, R: 0.564066852367688, micro F1: 0.569730733519034, macro F1: 0.5778798102137052 [INFO] Type Linking P: 0.532033426183844, R: 0.532033426183844, micro F1: 0.532033426183844, macro F1: 0.532033426183844 @@ -5038,8 +5038,8 @@ Question 359: Where Everybody knows you is the theme album of which show ? [INFO] Golden Answer: [http://dbpedia.org/resource/Cheers] [INFO] Predicted Answer: [] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 360, P: 0.524, R: 0.582, macro F1: 0.532, macro F1*: 0.551 -[INFO] Current time: 283 ms, average: 2908.27 ms/ques, entity detection: 455.41 ms/ques (linking tool: 364.34 ms/ques), relation detection: 932.51 ms/ques (similarity: 724.03 ms/ques), EDG generation: 58.98 ms/ques +[INFO] Cumulative metrics, sample: 360, P: 0.524, R: 0.582, macro F1: 0.551 +[INFO] Current question consumed time: 271 ms, average: 2686.82 ms/ques, entity detection: 416.65 ms/ques (linking tool: 335.00 ms/ques), relation detection: 879.16 ms/ques (similarity: 706.80 ms/ques), EDG generation: 58.77 ms/ques [INFO] Entity Linking P: 0.7347222222222223, R: 0.6986111111111111, micro F1: 0.7097222222222219, macro F1: 0.7162117786391042 [INFO] Relation Linking P: 0.5907407407407407, R: 0.5625, micro F1: 0.5681481481481478, macro F1: 0.5762745885186673 [INFO] Type Linking P: 0.5333333333333333, R: 0.5333333333333333, micro F1: 0.5333333333333333, macro F1: 0.5333333333333333 @@ -5052,8 +5052,8 @@ Question 360: Food in which countries has shallot in it? [INFO] Golden Answer: [http://dbpedia.org/resource/Thailand, http://dbpedia.org/resource/Indonesia, http://dbpedia.org/resource/India, http://dbpedia.org/resource/Malaysia, http://dbpedia.org/resource/Vietnam, http://dbpedia.org/resource/Laos, http://dbpedia.org/resource/Cambodia, http://dbpedia.org/resource/France, http://dbpedia.org/resource/Singapore, http://dbpedia.org/resource/Padang] [INFO] Predicted Answer: [http://dbpedia.org/resource/Phanaeng_curry, http://dbpedia.org/resource/Red_curry, http://dbpedia.org/resource/Rendang, http://dbpedia.org/resource/Pad_Thai, http://dbpedia.org/resource/Coconut_chutney, http://dbpedia.org/resource/Bubur_pedas, http://dbpedia.org/resource/Heong_Peng, http://dbpedia.org/resource/Semur_(Indonesian_stew), http://dbpedia.org/resource/Dabu-dabu, http://dbpedia.org/resource/Mie_celor, http://dbpedia.org/resource/Tekwan, http://dbpedia.org/resource/Mie_aceh, http://dbpedia.org/resource/Mie_kocok, http://dbpedia.org/resource/Nem_Nướng̣, http://dbpedia.org/resource/Ketoprak_(dish), http://dbpedia.org/resource/Miang_kham, http://dbpedia.org/resource/Pleah_sach_ko, http://dbpedia.org/resource/Kalathappam, http://dbpedia.org/resource/Cervelle_de_canut, http://dbpedia.org/resource/Ayam_goreng, http://dbpedia.org/resource/Ikan_bakar, http://dbpedia.org/resource/Bánh_cuốn, http://dbpedia.org/resource/Khao_poon, http://dbpedia.org/resource/Ayam_bakar, http://dbpedia.org/resource/Crab_in_Padang_sauce, http://dbpedia.org/resource/Acar, http://dbpedia.org/resource/Hae_mee, http://dbpedia.org/resource/Ikan_goreng] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 361, P: 0.522, R: 0.580, macro F1: 0.530, macro F1*: 0.550 -[INFO] Current time: 602 ms, average: 2901.88 ms/ques, entity detection: 454.40 ms/ques (linking tool: 363.54 ms/ques), relation detection: 930.57 ms/ques (similarity: 722.41 ms/ques), EDG generation: 58.86 ms/ques +[INFO] Cumulative metrics, sample: 361, P: 0.522, R: 0.580, macro F1: 0.550 +[INFO] Current question consumed time: 566 ms, average: 2680.94 ms/ques, entity detection: 415.73 ms/ques (linking tool: 334.25 ms/ques), relation detection: 877.32 ms/ques (similarity: 705.22 ms/ques), EDG generation: 58.64 ms/ques [INFO] Entity Linking P: 0.7354570637119113, R: 0.6994459833795014, micro F1: 0.7105263157894733, macro F1: 0.7169996470550486 [INFO] Relation Linking P: 0.5918744228993537, R: 0.5623268698060941, micro F1: 0.5684210526315786, macro F1: 0.5767224376731301 [INFO] Type Linking P: 0.5318559556786704, R: 0.5318559556786704, micro F1: 0.5318559556786704, macro F1: 0.5318559556786704 @@ -5066,8 +5066,8 @@ Question 361: Which rivers have source origins asnorth western, Zambia ? [INFO] Golden Answer: [http://dbpedia.org/resource/Zambezi] [INFO] Predicted Answer: [http://dbpedia.org/resource/Babaroga_(company), http://dbpedia.org/resource/Tricky_Software] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 362, P: 0.521, R: 0.579, macro F1: 0.529, macro F1*: 0.548 -[INFO] Current time: 1017 ms, average: 2896.67 ms/ques, entity detection: 453.50 ms/ques (linking tool: 362.78 ms/ques), relation detection: 928.69 ms/ques (similarity: 720.98 ms/ques), EDG generation: 58.77 ms/ques +[INFO] Cumulative metrics, sample: 362, P: 0.521, R: 0.579, macro F1: 0.548 +[INFO] Current question consumed time: 910 ms, average: 2676.05 ms/ques, entity detection: 414.88 ms/ques (linking tool: 333.54 ms/ques), relation detection: 875.51 ms/ques (similarity: 703.82 ms/ques), EDG generation: 58.55 ms/ques [INFO] Entity Linking P: 0.7334254143646409, R: 0.6975138121546961, micro F1: 0.7085635359116019, macro F1: 0.7150189850466093 [INFO] Relation Linking P: 0.5902394106813996, R: 0.5607734806629834, micro F1: 0.5668508287292814, macro F1: 0.5751292817679557 [INFO] Type Linking P: 0.5303867403314917, R: 0.5303867403314917, micro F1: 0.5303867403314917, macro F1: 0.5303867403314917 @@ -5080,8 +5080,8 @@ Question 362: Organization holding stocks of Galician and German wikipedias? [INFO] Golden Answer: [http://dbpedia.org/resource/Wikimedia_Foundation] [INFO] Predicted Answer: [http://dbpedia.org/resource/Rakesh_Rajani] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 363, P: 0.519, R: 0.577, macro F1: 0.528, macro F1*: 0.547 -[INFO] Current time: 1958 ms, average: 2894.09 ms/ques, entity detection: 453.70 ms/ques (linking tool: 362.97 ms/ques), relation detection: 926.93 ms/ques (similarity: 719.16 ms/ques), EDG generation: 58.64 ms/ques +[INFO] Cumulative metrics, sample: 363, P: 0.519, R: 0.577, macro F1: 0.547 +[INFO] Current question consumed time: 1787 ms, average: 2673.60 ms/ques, entity detection: 414.97 ms/ques (linking tool: 333.65 ms/ques), relation detection: 874.00 ms/ques (similarity: 702.04 ms/ques), EDG generation: 58.43 ms/ques [INFO] Entity Linking P: 0.731404958677686, R: 0.6955922865013774, micro F1: 0.7066115702479335, macro F1: 0.7130492357765085 [INFO] Relation Linking P: 0.588613406795225, R: 0.559228650137741, micro F1: 0.5652892561983468, macro F1: 0.5735449035812671 [INFO] Type Linking P: 0.5316804407713499, R: 0.5316804407713499, micro F1: 0.5316804407713499, macro F1: 0.5316804407713499 @@ -5094,8 +5094,8 @@ Question 363: Name the river with Readin, Brekshire in its path and mouth place [INFO] Golden Answer: [http://dbpedia.org/resource/River_Thames] [INFO] Predicted Answer: [http://dbpedia.org/resource/River_Thames] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 364, P: 0.521, R: 0.578, macro F1: 0.529, macro F1*: 0.548 -[INFO] Current time: 2797 ms, average: 2893.82 ms/ques, entity detection: 452.99 ms/ques (linking tool: 362.36 ms/ques), relation detection: 928.53 ms/ques (similarity: 719.59 ms/ques), EDG generation: 58.95 ms/ques +[INFO] Cumulative metrics, sample: 364, P: 0.521, R: 0.578, macro F1: 0.548 +[INFO] Current question consumed time: 2505 ms, average: 2673.14 ms/ques, entity detection: 414.26 ms/ques (linking tool: 333.04 ms/ques), relation detection: 875.40 ms/ques (similarity: 702.49 ms/ques), EDG generation: 58.73 ms/ques [INFO] Entity Linking P: 0.7321428571428571, R: 0.695054945054945, micro F1: 0.7065018315018312, macro F1: 0.7131170081121958 [INFO] Relation Linking P: 0.5897435897435898, R: 0.5590659340659341, micro F1: 0.5655677655677652, macro F1: 0.5739951559002975 [INFO] Type Linking P: 0.5302197802197802, R: 0.5302197802197802, micro F1: 0.5302197802197802, macro F1: 0.5302197802197802 @@ -5108,8 +5108,8 @@ Question 364: Alexander Magleby, a rugby player is alma mater to which universit [INFO] Golden Answer: [http://dbpedia.org/resource/Dartmouth_College] [INFO] Predicted Answer: [http://dbpedia.org/resource/Dartmouth_College] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 365, P: 0.522, R: 0.580, macro F1: 0.530, macro F1*: 0.549 -[INFO] Current time: 1494 ms, average: 2889.98 ms/ques, entity detection: 452.05 ms/ques (linking tool: 361.61 ms/ques), relation detection: 927.46 ms/ques (similarity: 719.21 ms/ques), EDG generation: 58.86 ms/ques +[INFO] Cumulative metrics, sample: 365, P: 0.522, R: 0.580, macro F1: 0.549 +[INFO] Current question consumed time: 1328 ms, average: 2669.45 ms/ques, entity detection: 413.36 ms/ques (linking tool: 332.31 ms/ques), relation detection: 874.33 ms/ques (similarity: 702.13 ms/ques), EDG generation: 58.64 ms/ques [INFO] Entity Linking P: 0.7328767123287672, R: 0.6958904109589041, micro F1: 0.7073059360730591, macro F1: 0.7139048319520876 [INFO] Relation Linking P: 0.5908675799086758, R: 0.5602739726027397, micro F1: 0.5667579908675795, macro F1: 0.575164236848826 [INFO] Type Linking P: 0.5287671232876713, R: 0.5287671232876713, micro F1: 0.5287671232876713, macro F1: 0.5287671232876713 @@ -5122,8 +5122,8 @@ Question 365: Is Amharic the official language of Ethiopia? [INFO] Golden Answer: [true] [INFO] Predicted Answer: [true] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 366, P: 0.523, R: 0.581, macro F1: 0.531, macro F1*: 0.551 -[INFO] Current time: 2924 ms, average: 2890.08 ms/ques, entity detection: 452.34 ms/ques (linking tool: 361.81 ms/ques), relation detection: 929.94 ms/ques (similarity: 719.98 ms/ques), EDG generation: 58.73 ms/ques +[INFO] Cumulative metrics, sample: 366, P: 0.523, R: 0.581, macro F1: 0.551 +[INFO] Current question consumed time: 2729 ms, average: 2669.62 ms/ques, entity detection: 413.65 ms/ques (linking tool: 332.52 ms/ques), relation detection: 876.65 ms/ques (similarity: 702.94 ms/ques), EDG generation: 58.51 ms/ques [INFO] Entity Linking P: 0.7336065573770492, R: 0.6967213114754098, micro F1: 0.7081056466302366, macro F1: 0.7146883366997041 [INFO] Relation Linking P: 0.5919854280510018, R: 0.5614754098360656, micro F1: 0.5679417122040069, macro F1: 0.5763269110042522 [INFO] Type Linking P: 0.5300546448087432, R: 0.5300546448087432, micro F1: 0.5300546448087432, macro F1: 0.5300546448087432 @@ -5136,8 +5136,8 @@ Question 366: What sport played at Islamic azad uni employes Craig Van Tilbury? [INFO] Golden Answer: [http://dbpedia.org/resource/Chess] [INFO] Predicted Answer: [http://dbpedia.org/resource/Chess] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 367, P: 0.525, R: 0.582, macro F1: 0.533, macro F1*: 0.552 -[INFO] Current time: 1225 ms, average: 2885.54 ms/ques, entity detection: 451.65 ms/ques (linking tool: 361.22 ms/ques), relation detection: 928.21 ms/ques (similarity: 718.84 ms/ques), EDG generation: 58.62 ms/ques +[INFO] Cumulative metrics, sample: 367, P: 0.525, R: 0.582, macro F1: 0.552 +[INFO] Current question consumed time: 1088 ms, average: 2665.31 ms/ques, entity detection: 412.99 ms/ques (linking tool: 331.94 ms/ques), relation detection: 875.04 ms/ques (similarity: 701.86 ms/ques), EDG generation: 58.39 ms/ques [INFO] Entity Linking P: 0.7343324250681199, R: 0.6961852861035422, micro F1: 0.7079927338782922, macro F1: 0.7147502270663034 [INFO] Relation Linking P: 0.5930971843778383, R: 0.5613079019073569, micro F1: 0.5682107175295182, macro F1: 0.5767648464917453 [INFO] Type Linking P: 0.5286103542234333, R: 0.5286103542234333, micro F1: 0.5286103542234333, macro F1: 0.5286103542234333 @@ -5150,8 +5150,8 @@ Question 367: List the products of the company which published Tweenies: Game Ti [INFO] Golden Answer: [http://dbpedia.org/resource/Doctor_Who, Robot Wars@en, Teletubbies@en] [INFO] Predicted Answer: [http://dbpedia.org/resource/Master_P, http://dbpedia.org/resource/The_Medicine_Men, http://dbpedia.org/resource/The_Neptunes, http://dbpedia.org/resource/Nick_Cannon, http://dbpedia.org/resource/Myke_Diesel, Master P@en, Carlos Stephens Sean \"Barney\" Thomas@en, Nick Cannon@en, Myke Diesel, TheBeats.com, Master P@en] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 368, P: 0.523, R: 0.580, macro F1: 0.531, macro F1*: 0.550 -[INFO] Current time: 2727 ms, average: 2885.11 ms/ques, entity detection: 451.10 ms/ques (linking tool: 360.63 ms/ques), relation detection: 928.91 ms/ques (similarity: 719.17 ms/ques), EDG generation: 58.53 ms/ques +[INFO] Cumulative metrics, sample: 368, P: 0.523, R: 0.580, macro F1: 0.550 +[INFO] Current question consumed time: 2416 ms, average: 2664.63 ms/ques, entity detection: 412.46 ms/ques (linking tool: 331.38 ms/ques), relation detection: 875.65 ms/ques (similarity: 702.23 ms/ques), EDG generation: 58.30 ms/ques [INFO] Entity Linking P: 0.7323369565217391, R: 0.6942934782608695, micro F1: 0.70606884057971, macro F1: 0.7128079710144927 [INFO] Relation Linking P: 0.5914855072463768, R: 0.5597826086956522, micro F1: 0.5666666666666663, macro F1: 0.5751975507132351 [INFO] Type Linking P: 0.529891304347826, R: 0.529891304347826, micro F1: 0.529891304347826, macro F1: 0.529891304347826 @@ -5164,8 +5164,8 @@ Question 368: List the agencies located someplace lead by Alex Chalk. [INFO] Golden Answer: [http://dbpedia.org/resource/Government_Communications_Headquarters] [INFO] Predicted Answer: [] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 369, P: 0.522, R: 0.579, macro F1: 0.530, macro F1*: 0.549 -[INFO] Current time: 4700 ms, average: 2890.03 ms/ques, entity detection: 450.38 ms/ques (linking tool: 359.89 ms/ques), relation detection: 933.31 ms/ques (similarity: 721.24 ms/ques), EDG generation: 58.41 ms/ques +[INFO] Cumulative metrics, sample: 369, P: 0.522, R: 0.579, macro F1: 0.549 +[INFO] Current question consumed time: 4041 ms, average: 2668.36 ms/ques, entity detection: 411.74 ms/ques (linking tool: 330.69 ms/ques), relation detection: 879.42 ms/ques (similarity: 704.33 ms/ques), EDG generation: 58.19 ms/ques [INFO] Entity Linking P: 0.7303523035230353, R: 0.6924119241192412, micro F1: 0.704155374887082, macro F1: 0.7108762420957544 [INFO] Relation Linking P: 0.5898825654923215, R: 0.5582655826558266, micro F1: 0.5651309846431793, macro F1: 0.5736387497627926 [INFO] Type Linking P: 0.5284552845528455, R: 0.5284552845528455, micro F1: 0.5284552845528455, macro F1: 0.5284552845528455 @@ -5178,8 +5178,8 @@ Question 369: What religion are the characters of Absolutely Fabulous from? [INFO] Golden Answer: [http://dbpedia.org/resource/Buddhism] [INFO] Predicted Answer: [http://dbpedia.org/resource/Jennifer_Saunders, http://dbpedia.org/resource/Joanna_Lumley, http://dbpedia.org/resource/June_Whitfield, http://dbpedia.org/resource/Jane_Horrocks, http://dbpedia.org/resource/Julia_Sawalha, Jennifer Saunders@en] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 370, P: 0.520, R: 0.577, macro F1: 0.528, macro F1*: 0.547 -[INFO] Current time: 1340 ms, average: 2885.84 ms/ques, entity detection: 449.72 ms/ques (linking tool: 359.25 ms/ques), relation detection: 931.93 ms/ques (similarity: 720.55 ms/ques), EDG generation: 58.30 ms/ques +[INFO] Cumulative metrics, sample: 370, P: 0.520, R: 0.577, macro F1: 0.547 +[INFO] Current question consumed time: 1175 ms, average: 2664.32 ms/ques, entity detection: 411.07 ms/ques (linking tool: 330.05 ms/ques), relation detection: 878.11 ms/ques (similarity: 703.69 ms/ques), EDG generation: 58.07 ms/ques [INFO] Entity Linking P: 0.731081081081081, R: 0.6932432432432433, micro F1: 0.7049549549549547, macro F1: 0.7116595722857583 [INFO] Relation Linking P: 0.5882882882882883, R: 0.5567567567567567, micro F1: 0.5636036036036032, macro F1: 0.5720883747634338 [INFO] Type Linking P: 0.527027027027027, R: 0.527027027027027, micro F1: 0.527027027027027, macro F1: 0.527027027027027 @@ -5192,8 +5192,8 @@ Question 370: What are the movies whose music composer uses a Hammond organ? [INFO] Golden Answer: [http://dbpedia.org/resource/Ted_(film), http://dbpedia.org/resource/Bitter_Moon, http://dbpedia.org/resource/Dan_in_Real_Life, http://dbpedia.org/resource/Quicksilver_(film), http://dbpedia.org/resource/The_Alf_Garnett_Saga, http://dbpedia.org/resource/Scream_for_Help, http://dbpedia.org/resource/The_Derby_Stallion, http://dbpedia.org/resource/The_Wild_Nation, http://dbpedia.org/resource/Blade_Runner, http://dbpedia.org/resource/The_Secret_Adventures_of_Tom_Thumb, http://dbpedia.org/resource/Thunderbirds_Are_Go, http://dbpedia.org/resource/Keyhole_(film), http://dbpedia.org/resource/Foodfight!, http://dbpedia.org/resource/Buddy_(2003_film), http://dbpedia.org/resource/Pulse_(1995_film), http://dbpedia.org/resource/Koyla, http://dbpedia.org/resource/Bob_and_the_Monster, http://dbpedia.org/resource/Ragdoll_(film), http://dbpedia.org/resource/Hare_and_Loathing_in_Las_Vegas, http://dbpedia.org/resource/The_Landlord, http://dbpedia.org/resource/Thunderbird_6, http://dbpedia.org/resource/One_Night_with_Blue_Note, http://dbpedia.org/resource/Red_Hook_Summer, http://dbpedia.org/resource/Antarctica_(1983_film), http://dbpedia.org/resource/Night_Mayor, http://dbpedia.org/resource/Chariots_of_Fire, http://dbpedia.org/resource/The_Bounty, http://dbpedia.org/resource/Daleks_–_Invasion_Earth:_2150_A.D., http://dbpedia.org/resource/Dr._Who_and_the_Daleks, http://dbpedia.org/resource/The_Wicked_Lady_(1983_film), http://dbpedia.org/resource/Starship_(film), http://dbpedia.org/resource/La_Métamorphose_des_cloportes, http://dbpedia.org/resource/The_Mexican_Staring_Frog_of_Southern_Sri_Lanka, http://dbpedia.org/resource/Do_You_Hear_the_Dogs_Barking%3F, http://dbpedia.org/resource/1492:_Conquest_of_Paradise, http://dbpedia.org/resource/Alexander_(2004_film), http://dbpedia.org/resource/El_Greco_(2007_film), http://dbpedia.org/resource/Missing_(film), http://dbpedia.org/resource/De_Mantel_der_Liefde, http://dbpedia.org/resource/The_Middle_of_the_World_(1974_film), http://dbpedia.org/resource/The_Salamander_(1971_film), http://dbpedia.org/resource/The_Stepfather_(1987_film), http://dbpedia.org/resource/King_of_the_Corner, http://dbpedia.org/resource/The_Man_from_Snowy_River:_Arena_Spectacular_(film), http://dbpedia.org/resource/The_Sleepwalker_(2014_film), http://dbpedia.org/resource/Raw_Force, http://dbpedia.org/resource/Exit_Through_the_Gift_Shop, http://dbpedia.org/resource/Francesco_(film), http://dbpedia.org/resource/Laugh_It_Up,_Fuzzball:_The_Family_Guy_Trilogy, http://dbpedia.org/resource/Ted_2, http://dbpedia.org/resource/Trashed_(film), http://dbpedia.org/resource/The_Forbidden_Room_(2015_film), http://dbpedia.org/resource/My_Winnipeg, http://dbpedia.org/resource/Elling, http://dbpedia.org/resource/Crime_and_Passion, http://dbpedia.org/resource/Entertaining_Mr_Sloane_(film), http://dbpedia.org/resource/This_Is_the_Sea_(film), http://dbpedia.org/resource/Journey_to_Shiloh, http://dbpedia.org/resource/Brand_upon_the_Brain!, http://dbpedia.org/resource/The_Shout, http://dbpedia.org/resource/Doppelgänger_(1969_film), http://dbpedia.org/resource/The_Hard_Word, http://dbpedia.org/resource/The_Plague_(1992_film), http://dbpedia.org/resource/Mrs._Brown,_You've_Got_a_Lovely_Daughter_(film), http://dbpedia.org/resource/Crossroads_to_Crime, http://dbpedia.org/resource/Angst_(2000_film), http://dbpedia.org/resource/Filmed_in_Supermarionation] [INFO] Predicted Answer: [http://dbpedia.org/resource/Billy_Joel, http://dbpedia.org/resource/Mike_Scott_(musician), http://dbpedia.org/resource/Lonnie_Smith_(jazz_musician), http://dbpedia.org/resource/Ron_%22Pigpen%22_McKernan, http://dbpedia.org/resource/Krister_Axel, http://dbpedia.org/resource/Mark_Stein_(musician), http://dbpedia.org/resource/Marco_Benevento, http://dbpedia.org/resource/Jay_Darlington, http://dbpedia.org/resource/Mother_Earth_(UK_band), http://dbpedia.org/resource/Morten_Abel, http://dbpedia.org/resource/Tim_Hinkley, http://dbpedia.org/resource/Atle_Bakken, http://dbpedia.org/resource/Michael_McMorrow, http://dbpedia.org/resource/David_Garcia_(musician), http://dbpedia.org/resource/Folk_rock, http://dbpedia.org/resource/Jazz, http://dbpedia.org/resource/Rhythm_and_blues, http://dbpedia.org/resource/Tori_Amos, http://dbpedia.org/resource/Ed_Lincoln, http://dbpedia.org/resource/Soul_jazz, http://dbpedia.org/resource/Vincent_Crane, http://dbpedia.org/resource/Speedy_Keen, http://dbpedia.org/resource/Gene_Harris, http://dbpedia.org/resource/Rachael_Sage, http://dbpedia.org/resource/Eric_Steckel, http://dbpedia.org/resource/Bernie_Herms, http://dbpedia.org/resource/Anthony_Thistlethwaite, http://dbpedia.org/resource/Gary_Brooker, http://dbpedia.org/resource/Jesse_Carmichael, http://dbpedia.org/resource/Rick_van_der_Linden, http://dbpedia.org/resource/Don_Pullen, http://dbpedia.org/resource/Geoff_Barrow, http://dbpedia.org/resource/Hard_bop, http://dbpedia.org/resource/Billy_Preston, http://dbpedia.org/resource/Mick_Gallagher, http://dbpedia.org/resource/Steinar_Nickelsen, http://dbpedia.org/resource/Marlowe_Morris, http://dbpedia.org/resource/Thomas_Dybdahl, http://dbpedia.org/resource/Milt_Buckner, http://dbpedia.org/resource/Red_Young, http://dbpedia.org/resource/Dave_Nicholas, http://dbpedia.org/resource/Denny_DeMarchi, http://dbpedia.org/resource/Keith_Hale, http://dbpedia.org/resource/Anne_Grete_Preus, http://dbpedia.org/resource/Robert_Walter_(musician), http://dbpedia.org/resource/Brit_funk, http://dbpedia.org/resource/Adam_Barry, http://dbpedia.org/resource/Sam_Williams_(record_producer), http://dbpedia.org/resource/Dallas_Kruse, http://dbpedia.org/resource/Shea_Marshall, http://dbpedia.org/resource/Cheick_Tidiane_Seck, http://dbpedia.org/resource/Rob_Halverson, http://dbpedia.org/resource/Arjen_Anthony_Lucassen, http://dbpedia.org/resource/Blues_rock, http://dbpedia.org/resource/Funk, http://dbpedia.org/resource/Graham_Bond, http://dbpedia.org/resource/Greg_Ham, http://dbpedia.org/resource/Jimmy_McGriff, http://dbpedia.org/resource/Larry_Knechtel, http://dbpedia.org/resource/Soul_music, http://dbpedia.org/resource/Vangelis, http://dbpedia.org/resource/Zoot_Money, http://dbpedia.org/resource/Al_Kooper, http://dbpedia.org/resource/David_Gates, http://dbpedia.org/resource/Ian_McLagan, http://dbpedia.org/resource/Isaiah_%22Ikey%22_Owens, http://dbpedia.org/resource/James_Booker, http://dbpedia.org/resource/Jimmy_Smith_(musician), http://dbpedia.org/resource/John_Paul_Jones_(musician), http://dbpedia.org/resource/Walter_Murphy, http://dbpedia.org/resource/Dave_Greenslade, http://dbpedia.org/resource/Jason_Freese, http://dbpedia.org/resource/Jeff_Fatt, http://dbpedia.org/resource/Johnny_%22Hammond%22_Smith, http://dbpedia.org/resource/Richard_Wright_(musician), http://dbpedia.org/resource/Janove_Ottesen, http://dbpedia.org/resource/Steve_Knight_(musician), http://dbpedia.org/resource/Dave_Lewis_(musician), http://dbpedia.org/resource/Pete_Levin, http://dbpedia.org/resource/Robert_Roth_(musician), http://dbpedia.org/resource/Christian_R&B, http://dbpedia.org/resource/Blimus, http://dbpedia.org/resource/Barbara_Dennerlein, http://dbpedia.org/resource/Punk_blues, http://dbpedia.org/resource/Steve_Winwood, http://dbpedia.org/resource/Joey_DeFrancesco, http://dbpedia.org/resource/Matthew_Fisher, http://dbpedia.org/resource/Mike_Finnigan, http://dbpedia.org/resource/The_Prisoners, http://dbpedia.org/resource/Thijs_van_Leer, http://dbpedia.org/resource/Tony_Banks_(musician), http://dbpedia.org/resource/Tony_Kaye_(musician), http://dbpedia.org/resource/Tyrone_Downie, http://dbpedia.org/resource/Billy_Powell, http://dbpedia.org/resource/Charlie_Gillingham, http://dbpedia.org/resource/Christopher_North_(Ambrosia), http://dbpedia.org/resource/Eddy_Louiss, http://dbpedia.org/resource/Gene_Ludwig, http://dbpedia.org/resource/Georgie_Fame, http://dbpedia.org/resource/Gregg_Rolie, http://dbpedia.org/resource/Neal_Doughty, http://dbpedia.org/resource/Patrick_Moraz, http://dbpedia.org/resource/Paul_Clark_(keyboardist), http://dbpedia.org/resource/Jesper_Strömblad, http://dbpedia.org/resource/Jon_Batiste, http://dbpedia.org/resource/Mic_Michaeli, http://dbpedia.org/resource/Richard_Tee, http://dbpedia.org/resource/Billy_Ritchie_(musician), http://dbpedia.org/resource/Tim_&_Bob, http://dbpedia.org/resource/Harry_Waters, http://dbpedia.org/resource/Pete_Mazich, http://dbpedia.org/resource/Jackie_Davis, http://dbpedia.org/resource/Jean_Ven_Robert_Hal, http://dbpedia.org/resource/James_Prime, http://dbpedia.org/resource/Rami_Jaffee, http://dbpedia.org/resource/Joe_Doria, http://dbpedia.org/resource/Ron_Rawls, http://dbpedia.org/resource/Barry_Gray, http://dbpedia.org/resource/Merl_Saunders, http://dbpedia.org/resource/Norman_Petty, http://dbpedia.org/resource/Benmont_Tench, http://dbpedia.org/resource/Shooter_Jennings, http://dbpedia.org/resource/Booker_T._Jones, http://dbpedia.org/resource/Josh_Klinghoffer, http://dbpedia.org/resource/Goran_Karan, http://dbpedia.org/resource/Alexander_Hawkins, http://dbpedia.org/resource/Ehud_Asherie, http://dbpedia.org/resource/Harald_Dahlstrøm, http://dbpedia.org/resource/Mike_LeDonne, http://dbpedia.org/resource/Soul_Basement, http://dbpedia.org/resource/Goldy_McJohn, http://dbpedia.org/resource/Bo_Hansson, http://dbpedia.org/resource/Jon_Cleary_(musician), http://dbpedia.org/resource/Rick_DePiro, http://dbpedia.org/resource/Milt_Herth, http://dbpedia.org/resource/Lenny_Dee, http://dbpedia.org/resource/Christopher_Stevens_(musician), http://dbpedia.org/resource/Roy_Phillips_(musician), http://dbpedia.org/resource/Jean-Jacques_Kravetz, http://dbpedia.org/resource/Lars_Lillo-Stenberg, http://dbpedia.org/resource/Jeff_Pardo, http://dbpedia.org/resource/Garth_Porter, http://dbpedia.org/resource/Sondre_Lerche, http://dbpedia.org/resource/Charles_Giordano, http://dbpedia.org/resource/Greg_Wells, http://dbpedia.org/resource/Mick_Weaver, http://dbpedia.org/resource/David_Thrussell, http://dbpedia.org/resource/Atsuko_Hashimoto, http://dbpedia.org/resource/Dan_Fogel_(musician), http://dbpedia.org/resource/John_DeFrancesco, http://dbpedia.org/resource/Judy_Blair, http://dbpedia.org/resource/Pat_Bianchi, http://dbpedia.org/resource/Paul_Wagnberg, http://dbpedia.org/resource/Cassie_Taylor, http://dbpedia.org/resource/Lyman_Woodard, http://dbpedia.org/resource/Michał_Łapaj, http://dbpedia.org/resource/Nathan_Nockels, http://dbpedia.org/resource/Nathan_Gaunt, http://dbpedia.org/resource/Serabee, http://dbpedia.org/resource/Jason_Staczek, http://dbpedia.org/resource/Raymond_Angry, http://dbpedia.org/resource/Mike_Mangan_(musician), http://dbpedia.org/resource/John_Emil_Augustine, http://dbpedia.org/resource/Craig_Brockman] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 371, P: 0.519, R: 0.576, macro F1: 0.527, macro F1*: 0.546 -[INFO] Current time: 1576 ms, average: 2882.31 ms/ques, entity detection: 449.24 ms/ques (linking tool: 358.84 ms/ques), relation detection: 930.77 ms/ques (similarity: 720.00 ms/ques), EDG generation: 58.19 ms/ques +[INFO] Cumulative metrics, sample: 371, P: 0.519, R: 0.576, macro F1: 0.546 +[INFO] Current question consumed time: 1441 ms, average: 2661.03 ms/ques, entity detection: 410.65 ms/ques (linking tool: 329.70 ms/ques), relation detection: 876.98 ms/ques (similarity: 703.15 ms/ques), EDG generation: 57.96 ms/ques [INFO] Entity Linking P: 0.7318059299191375, R: 0.6940700808625337, micro F1: 0.705750224618149, macro F1: 0.7124386652331869 [INFO] Relation Linking P: 0.5893980233602875, R: 0.5566037735849056, micro F1: 0.5638814016172503, macro F1: 0.5725316745930193 [INFO] Type Linking P: 0.5256064690026954, R: 0.5256064690026954, micro F1: 0.5256064690026954, macro F1: 0.5256064690026954 @@ -5206,8 +5206,8 @@ Question 371: Was Love Hurts in Short Cut Draw Blood? [INFO] Golden Answer: [true] [INFO] Predicted Answer: [true] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 372, P: 0.520, R: 0.577, macro F1: 0.528, macro F1*: 0.547 -[INFO] Current time: 1305 ms, average: 2878.07 ms/ques, entity detection: 448.98 ms/ques (linking tool: 358.53 ms/ques), relation detection: 928.82 ms/ques (similarity: 718.37 ms/ques), EDG generation: 58.06 ms/ques +[INFO] Cumulative metrics, sample: 372, P: 0.520, R: 0.577, macro F1: 0.547 +[INFO] Current question consumed time: 1110 ms, average: 2656.86 ms/ques, entity detection: 410.34 ms/ques (linking tool: 329.39 ms/ques), relation detection: 875.09 ms/ques (similarity: 701.55 ms/ques), EDG generation: 57.83 ms/ques [INFO] Entity Linking P: 0.7325268817204301, R: 0.6935483870967742, micro F1: 0.7056451612903224, macro F1: 0.7125049405612478 [INFO] Relation Linking P: 0.5878136200716846, R: 0.5551075268817204, micro F1: 0.562365591397849, macro F1: 0.5709926109516403 [INFO] Type Linking P: 0.5268817204301075, R: 0.5268817204301075, micro F1: 0.5268817204301075, macro F1: 0.5268817204301075 @@ -5220,8 +5220,8 @@ Question 372: Janaka Thissakuttiarachchi follows what religion, which once emplo [INFO] Golden Answer: [http://dbpedia.org/resource/Buddhism] [INFO] Predicted Answer: [http://dbpedia.org/resource/Buddhism] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 373, P: 0.522, R: 0.578, macro F1: 0.529, macro F1*: 0.548 -[INFO] Current time: 755 ms, average: 2872.38 ms/ques, entity detection: 448.22 ms/ques (linking tool: 357.79 ms/ques), relation detection: 926.85 ms/ques (similarity: 716.77 ms/ques), EDG generation: 57.99 ms/ques +[INFO] Cumulative metrics, sample: 373, P: 0.522, R: 0.578, macro F1: 0.548 +[INFO] Current question consumed time: 700 ms, average: 2651.61 ms/ques, entity detection: 409.64 ms/ques (linking tool: 328.71 ms/ques), relation detection: 873.25 ms/ques (similarity: 700.01 ms/ques), EDG generation: 57.76 ms/ques [INFO] Entity Linking P: 0.7332439678284183, R: 0.6943699731903485, micro F1: 0.7064343163538872, macro F1: 0.7132777001598509 [INFO] Relation Linking P: 0.5889186773905273, R: 0.5563002680965148, micro F1: 0.5635388739946376, macro F1: 0.5721449499424163 [INFO] Type Linking P: 0.5281501340482574, R: 0.5281501340482574, micro F1: 0.5281501340482574, macro F1: 0.5281501340482574 @@ -5234,8 +5234,8 @@ Question 373: Whihc birthpalce of Svetlana Gounkina is also the largest city of [INFO] Golden Answer: [http://dbpedia.org/resource/Moscow] [INFO] Predicted Answer: [http://dbpedia.org/resource/Moscow] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 374, P: 0.523, R: 0.579, macro F1: 0.531, macro F1*: 0.550 -[INFO] Current time: 1568 ms, average: 2868.89 ms/ques, entity detection: 448.06 ms/ques (linking tool: 357.30 ms/ques), relation detection: 925.32 ms/ques (similarity: 715.45 ms/ques), EDG generation: 58.12 ms/ques +[INFO] Cumulative metrics, sample: 374, P: 0.523, R: 0.579, macro F1: 0.550 +[INFO] Current question consumed time: 1488 ms, average: 2648.50 ms/ques, entity detection: 409.57 ms/ques (linking tool: 328.29 ms/ques), relation detection: 871.83 ms/ques (similarity: 698.71 ms/ques), EDG generation: 57.89 ms/ques [INFO] Entity Linking P: 0.733957219251337, R: 0.6951871657754011, micro F1: 0.7072192513368982, macro F1: 0.7140463124615439 [INFO] Relation Linking P: 0.5900178253119429, R: 0.5574866310160428, micro F1: 0.5647058823529407, macro F1: 0.5732911063807694 [INFO] Type Linking P: 0.5294117647058824, R: 0.5294117647058824, micro F1: 0.5294117647058824, macro F1: 0.5294117647058824 @@ -5248,8 +5248,8 @@ Question 374: Name the TV show with artist Christopher Franke and cast member Ja [INFO] Golden Answer: [http://dbpedia.org/resource/Babylon_5] [INFO] Predicted Answer: [http://dbpedia.org/resource/Babylon_5] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 375, P: 0.524, R: 0.580, macro F1: 0.532, macro F1*: 0.551 -[INFO] Current time: 2830 ms, average: 2868.78 ms/ques, entity detection: 447.52 ms/ques (linking tool: 356.90 ms/ques), relation detection: 925.14 ms/ques (similarity: 716.16 ms/ques), EDG generation: 58.06 ms/ques +[INFO] Cumulative metrics, sample: 375, P: 0.524, R: 0.580, macro F1: 0.551 +[INFO] Current question consumed time: 2628 ms, average: 2648.45 ms/ques, entity detection: 409.08 ms/ques (linking tool: 327.93 ms/ques), relation detection: 871.65 ms/ques (similarity: 699.46 ms/ques), EDG generation: 57.82 ms/ques [INFO] Entity Linking P: 0.7346666666666667, R: 0.696, micro F1: 0.7079999999999999, macro F1: 0.7148108108108108 [INFO] Relation Linking P: 0.591111111111111, R: 0.5586666666666666, micro F1: 0.5658666666666663, macro F1: 0.5744311300090195 [INFO] Type Linking P: 0.528, R: 0.528, micro F1: 0.528, macro F1: 0.528 @@ -5262,8 +5262,8 @@ Question 375: Which artist married to Lee Krasner? [INFO] Golden Answer: [http://dbpedia.org/resource/Jackson_Pollock] [INFO] Predicted Answer: [http://dbpedia.org/resource/Jackson_Pollock, 1945, 1956, his death@en] [INFO] P: 0.25, R: 1.0, F: 0.4 -[INFO] Cumulative metrics, sample: 376, P: 0.523, R: 0.581, macro F1: 0.532, macro F1*: 0.551 -[INFO] Current time: 801 ms, average: 2863.28 ms/ques, entity detection: 446.70 ms/ques (linking tool: 356.16 ms/ques), relation detection: 923.30 ms/ques (similarity: 714.83 ms/ques), EDG generation: 57.93 ms/ques +[INFO] Cumulative metrics, sample: 376, P: 0.523, R: 0.581, macro F1: 0.551 +[INFO] Current question consumed time: 762 ms, average: 2643.43 ms/ques, entity detection: 408.34 ms/ques (linking tool: 327.24 ms/ques), relation detection: 869.95 ms/ques (similarity: 698.20 ms/ques), EDG generation: 57.69 ms/ques [INFO] Entity Linking P: 0.7353723404255319, R: 0.6968085106382979, micro F1: 0.7087765957446807, macro F1: 0.7155712281949465 [INFO] Relation Linking P: 0.5921985815602837, R: 0.5598404255319149, micro F1: 0.5670212765957443, macro F1: 0.5755650700351199 [INFO] Type Linking P: 0.526595744680851, R: 0.526595744680851, micro F1: 0.526595744680851, macro F1: 0.526595744680851 @@ -5276,8 +5276,8 @@ Question 376: What teams did the ex teamplayers of lighthouse boy club start the [INFO] Golden Answer: [http://dbpedia.org/resource/Philadelphia_Phillies] [INFO] Predicted Answer: [http://dbpedia.org/resource/United_States_men's_national_soccer_team, http://dbpedia.org/resource/Lighthouse_Boys_Club, http://dbpedia.org/resource/Temple_University, http://dbpedia.org/resource/Philadelphia_Nationals, http://dbpedia.org/resource/Brookhattan, http://dbpedia.org/resource/New_York_Hakoah, http://dbpedia.org/resource/Uhrik_Truckers, http://dbpedia.org/resource/Harrison_F.C., http://dbpedia.org/resource/Philadelphia_Merchant_Ship, http://dbpedia.org/resource/Bethlehem_Steel_FC, http://dbpedia.org/resource/Disston_A.A., http://dbpedia.org/resource/Fleisher_Yarn, http://dbpedia.org/resource/New_York_Galicia, http://dbpedia.org/resource/Baltimore_Pompei, http://dbpedia.org/resource/Ludlow_Lusitano] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 377, P: 0.522, R: 0.580, macro F1: 0.530, macro F1*: 0.549 -[INFO] Current time: 2112 ms, average: 2861.29 ms/ques, entity detection: 446.12 ms/ques (linking tool: 355.69 ms/ques), relation detection: 922.77 ms/ques (similarity: 714.70 ms/ques), EDG generation: 57.86 ms/ques +[INFO] Cumulative metrics, sample: 377, P: 0.522, R: 0.580, macro F1: 0.549 +[INFO] Current question consumed time: 2024 ms, average: 2641.79 ms/ques, entity detection: 407.79 ms/ques (linking tool: 326.79 ms/ques), relation detection: 869.52 ms/ques (similarity: 698.14 ms/ques), EDG generation: 57.79 ms/ques [INFO] Entity Linking P: 0.7360742705570292, R: 0.6976127320954907, micro F1: 0.7095490716180369, macro F1: 0.7163275972488389 [INFO] Relation Linking P: 0.5906277630415562, R: 0.5583554376657824, micro F1: 0.5655172413793099, macro F1: 0.5740383722366182 [INFO] Type Linking P: 0.5278514588859416, R: 0.5278514588859416, micro F1: 0.5278514588859416, macro F1: 0.5278514588859416 @@ -5290,8 +5290,8 @@ Question 377: Where was the deadbody discovered of one of the alumni of General [INFO] Golden Answer: [http://dbpedia.org/resource/Nebraska, http://dbpedia.org/resource/Omaha,_Nebraska, http://dbpedia.org/resource/Westroads_Mall, ,@en, , U.S.@en] [INFO] Predicted Answer: [http://dbpedia.org/resource/Nebraska, http://dbpedia.org/resource/Omaha,_Nebraska, http://dbpedia.org/resource/Westroads_Mall, ,@en, , U.S.@en] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 378, P: 0.523, R: 0.581, macro F1: 0.531, macro F1*: 0.551 -[INFO] Current time: 1097 ms, average: 2856.62 ms/ques, entity detection: 445.34 ms/ques (linking tool: 354.98 ms/ques), relation detection: 921.23 ms/ques (similarity: 713.35 ms/ques), EDG generation: 57.78 ms/ques +[INFO] Cumulative metrics, sample: 378, P: 0.523, R: 0.581, macro F1: 0.551 +[INFO] Current question consumed time: 1028 ms, average: 2637.52 ms/ques, entity detection: 407.07 ms/ques (linking tool: 326.13 ms/ques), relation detection: 868.11 ms/ques (similarity: 696.85 ms/ques), EDG generation: 57.71 ms/ques [INFO] Entity Linking P: 0.7367724867724867, R: 0.6984126984126984, micro F1: 0.7103174603174601, macro F1: 0.7170799502596737 [INFO] Relation Linking P: 0.591710758377425, R: 0.5595238095238095, micro F1: 0.5666666666666662, macro F1: 0.5751673323484889 [INFO] Type Linking P: 0.5291005291005291, R: 0.5291005291005291, micro F1: 0.5291005291005291, macro F1: 0.5291005291005291 @@ -5304,8 +5304,8 @@ Question 378: Who designed the bridge in Oddamavadi? [INFO] Golden Answer: [http://dbpedia.org/resource/Sri_Lanka, http://dbpedia.org/resource/British_Ceylon] [INFO] Predicted Answer: [http://dbpedia.org/resource/Sri_Lanka, http://dbpedia.org/resource/Batticaloa_District] [INFO] P: 0.5, R: 0.5, F: 0.5 -[INFO] Cumulative metrics, sample: 379, P: 0.523, R: 0.581, macro F1: 0.531, macro F1*: 0.550 -[INFO] Current time: 946 ms, average: 2851.58 ms/ques, entity detection: 444.27 ms/ques (linking tool: 354.13 ms/ques), relation detection: 919.63 ms/ques (similarity: 711.98 ms/ques), EDG generation: 57.87 ms/ques +[INFO] Cumulative metrics, sample: 379, P: 0.523, R: 0.581, macro F1: 0.550 +[INFO] Current question consumed time: 893 ms, average: 2632.91 ms/ques, entity detection: 406.10 ms/ques (linking tool: 325.35 ms/ques), relation detection: 866.59 ms/ques (similarity: 695.50 ms/ques), EDG generation: 57.83 ms/ques [INFO] Entity Linking P: 0.7348284960422163, R: 0.6965699208443272, micro F1: 0.7084432717678099, macro F1: 0.7151879187286455 [INFO] Relation Linking P: 0.5901495162708883, R: 0.558047493403694, micro F1: 0.5651715039577833, macro F1: 0.5736497404425563 [INFO] Type Linking P: 0.5277044854881267, R: 0.5277044854881267, micro F1: 0.5277044854881267, macro F1: 0.5277044854881267 @@ -5318,8 +5318,8 @@ Question 379: What is the leader name of the settlements whose governing body is [INFO] Golden Answer: [http://dbpedia.org/resource/Melania_Carolina_Hotu_Hey, http://dbpedia.org/resource/Pedro_Pablo_Edmunds_Paoa, http://dbpedia.org/resource/Indian_National_Congress, http://dbpedia.org/resource/Jørn_Pedersen] [INFO] Predicted Answer: [http://dbpedia.org/resource/Melania_Carolina_Hotu_Hey, http://dbpedia.org/resource/Pedro_Pablo_Edmunds_Paoa, http://dbpedia.org/resource/Indian_National_Congress, http://dbpedia.org/resource/Jørn_Pedersen, Felipe Paredes Vergara@en, Melania Carolina Hotu Hey@en, Pedro Edmunds Paoa@en, Samir Dip Blon@en, Deep Sharma@en, Suresh Korde@en] [INFO] P: 0.4444444444444444, R: 1.0, F: 0.6153846153846153 -[INFO] Cumulative metrics, sample: 380, P: 0.523, R: 0.582, macro F1: 0.532, macro F1*: 0.551 -[INFO] Current time: 5403 ms, average: 2858.30 ms/ques, entity detection: 443.72 ms/ques (linking tool: 353.62 ms/ques), relation detection: 926.34 ms/ques (similarity: 716.69 ms/ques), EDG generation: 57.81 ms/ques +[INFO] Cumulative metrics, sample: 380, P: 0.523, R: 0.582, macro F1: 0.551 +[INFO] Current question consumed time: 4098 ms, average: 2636.77 ms/ques, entity detection: 405.54 ms/ques (linking tool: 324.83 ms/ques), relation detection: 870.65 ms/ques (similarity: 697.68 ms/ques), EDG generation: 57.77 ms/ques [INFO] Entity Linking P: 0.7355263157894737, R: 0.6973684210526315, micro F1: 0.7092105263157893, macro F1: 0.7159392972790102 [INFO] Relation Linking P: 0.5912280701754385, R: 0.5592105263157895, micro F1: 0.5663157894736838, macro F1: 0.5747737624656379 [INFO] Type Linking P: 0.5263157894736842, R: 0.5263157894736842, micro F1: 0.5263157894736842, macro F1: 0.5263157894736842 @@ -5332,8 +5332,8 @@ Question 380: Name the co-founder of the record labels which have one of the fou [INFO] Golden Answer: [http://dbpedia.org/resource/Brad_Paisley, http://dbpedia.org/resource/Chris_DuBois, http://dbpedia.org/resource/Frank_Rogers_(record_producer)] [INFO] Predicted Answer: [http://dbpedia.org/resource/Brad_Paisley, http://dbpedia.org/resource/Chris_DuBois, http://dbpedia.org/resource/Frank_Rogers_(record_producer)] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 381, P: 0.524, R: 0.583, macro F1: 0.533, macro F1*: 0.552 -[INFO] Current time: 1239 ms, average: 2854.05 ms/ques, entity detection: 443.06 ms/ques (linking tool: 352.94 ms/ques), relation detection: 924.92 ms/ques (similarity: 715.69 ms/ques), EDG generation: 57.78 ms/ques +[INFO] Cumulative metrics, sample: 381, P: 0.524, R: 0.583, macro F1: 0.552 +[INFO] Current question consumed time: 1140 ms, average: 2632.84 ms/ques, entity detection: 404.89 ms/ques (linking tool: 324.19 ms/ques), relation detection: 869.28 ms/ques (similarity: 696.71 ms/ques), EDG generation: 57.73 ms/ques [INFO] Entity Linking P: 0.7362204724409449, R: 0.6981627296587927, micro F1: 0.7099737532808398, macro F1: 0.7166867179114048 [INFO] Relation Linking P: 0.5923009623797025, R: 0.5603674540682415, micro F1: 0.5674540682414694, macro F1: 0.5758918655092213 [INFO] Type Linking P: 0.5249343832020997, R: 0.5249343832020997, micro F1: 0.5249343832020997, macro F1: 0.5249343832020997 @@ -5346,8 +5346,8 @@ Question 381: List the companies operating in Australia? [INFO] Golden Answer: [http://dbpedia.org/resource/Energy_Star, http://dbpedia.org/resource/Woolworths_(South_Africa), http://dbpedia.org/resource/CDS_Global, http://dbpedia.org/resource/Morphy_Richards, http://dbpedia.org/resource/Outrigger_Hotels_&_Resorts, http://dbpedia.org/resource/Uniden, http://dbpedia.org/resource/Vevo, http://dbpedia.org/resource/Hellenic_Australian_Business_Council, http://dbpedia.org/resource/Hoyts, http://dbpedia.org/resource/Illuminates_of_Thanateros, http://dbpedia.org/resource/OPSM, http://dbpedia.org/resource/Icom_Incorporated, http://dbpedia.org/resource/Sydney_Airport_Corporation, http://dbpedia.org/resource/Jaycar, http://dbpedia.org/resource/Todd_Corporation, http://dbpedia.org/resource/Inghams_Enterprises, http://dbpedia.org/resource/Presto_(company), http://dbpedia.org/resource/Schweppes_Australia, http://dbpedia.org/resource/Ford_Performance_Vehicles, http://dbpedia.org/resource/Marcopolo_S.A., http://dbpedia.org/resource/Moriwaki, http://dbpedia.org/resource/James_Pascoe_Group, http://dbpedia.org/resource/Virgin_Active, http://dbpedia.org/resource/Genesee_&_Wyoming, http://dbpedia.org/resource/Adzuna, http://dbpedia.org/resource/Quicksales, http://dbpedia.org/resource/Smiggle, http://dbpedia.org/resource/Meinhardt_Group, http://dbpedia.org/resource/Dairy_Farmers, http://dbpedia.org/resource/YHA_Australia, http://dbpedia.org/resource/Small_World_Social, http://dbpedia.org/resource/Vinamilk, http://dbpedia.org/resource/Asian_Couture_Federation, http://dbpedia.org/resource/Macquarie_Generation, http://dbpedia.org/resource/Kathmandu_(company), http://dbpedia.org/resource/Retravision, http://dbpedia.org/resource/Royal_Australasian_College_of_Medical_Administrators, http://dbpedia.org/resource/Aussie_(financial_group), http://dbpedia.org/resource/Jumbuck_Pastoral_Company, http://dbpedia.org/resource/Australian_Dairy_Foods, http://dbpedia.org/resource/Cruise_Whitsundays, http://dbpedia.org/resource/Aldi, http://dbpedia.org/resource/Australian_Film_Institute, http://dbpedia.org/resource/Optus_Television, http://dbpedia.org/resource/Australian_Cinematographers_Society, http://dbpedia.org/resource/Canson, http://dbpedia.org/resource/Stüssy, http://dbpedia.org/resource/Insurance_Australia_Group, http://dbpedia.org/resource/National_LGBTI_Health_Alliance, http://dbpedia.org/resource/Glen_Raven,_Inc., http://dbpedia.org/resource/Manufacturers_Mutual_Insurance, http://dbpedia.org/resource/Sematext, http://dbpedia.org/resource/Stan_(company), http://dbpedia.org/resource/Z_Energy, http://dbpedia.org/resource/Bank_of_Melbourne_(2011), http://dbpedia.org/resource/AusRegistry, http://dbpedia.org/resource/Exetel, http://dbpedia.org/resource/Sutherland_Astronomical_Society, http://dbpedia.org/resource/Leap_Frog_Films, http://dbpedia.org/resource/Everyman's_Welfare_Service, http://dbpedia.org/resource/Maccabi_Australia, http://dbpedia.org/resource/Turner_International_Australia, http://dbpedia.org/resource/AFC_Ajax_N.V., http://dbpedia.org/resource/FlaxUK, http://dbpedia.org/resource/Clarke_Energy, http://dbpedia.org/resource/Niagara_Therapy, http://dbpedia.org/resource/Club_21, http://dbpedia.org/resource/Guzman_y_Gomez, http://dbpedia.org/resource/Laser_Corporation, http://dbpedia.org/resource/Modica_Group, http://dbpedia.org/resource/Sydney_Seed_Fund, http://dbpedia.org/resource/Kookaburra_bats, http://dbpedia.org/resource/Asmex_Digital, http://dbpedia.org/resource/Kobo_Inc., http://dbpedia.org/resource/Cuscal, http://dbpedia.org/resource/David_Jones_Limited, http://dbpedia.org/resource/Hutchison_3G, http://dbpedia.org/resource/ING_Direct_Australia, http://dbpedia.org/resource/Wesfarmers, http://dbpedia.org/resource/Amcal, http://dbpedia.org/resource/Crazy_John's, http://dbpedia.org/resource/Endeca, http://dbpedia.org/resource/Boost_Mobile, http://dbpedia.org/resource/Divers_Alert_Network, http://dbpedia.org/resource/Outback_Steakhouse, http://dbpedia.org/resource/STA_Travel, http://dbpedia.org/resource/Secret_Recipe_(restaurant), http://dbpedia.org/resource/Yogurtland, http://dbpedia.org/resource/ANCA_(company), http://dbpedia.org/resource/Ausmed, http://dbpedia.org/resource/Bakers_Delight, http://dbpedia.org/resource/CSR_Limited, http://dbpedia.org/resource/Elders_Limited, http://dbpedia.org/resource/Grocon, http://dbpedia.org/resource/Huon_Aquaculture, http://dbpedia.org/resource/Lowes_Menswear, http://dbpedia.org/resource/Oneflare, http://dbpedia.org/resource/Promina_Group, http://dbpedia.org/resource/Star_Entertainment_Group, http://dbpedia.org/resource/Suncorp_Group, http://dbpedia.org/resource/TechnologyOne, http://dbpedia.org/resource/Tip_Top_Bakeries, http://dbpedia.org/resource/Silver_Spring_Networks, http://dbpedia.org/resource/Civic_Video, http://dbpedia.org/resource/Video_Ezy, http://dbpedia.org/resource/Kuwahara_(bicycle_company), http://dbpedia.org/resource/Homeart, http://dbpedia.org/resource/M2_Group, http://dbpedia.org/resource/Toll_Domestic_Forwarding, http://dbpedia.org/resource/Truphone, http://dbpedia.org/resource/Snakk_Media, http://dbpedia.org/resource/The_Foundation_for_Young_Australians, http://dbpedia.org/resource/Tori_Richard, http://dbpedia.org/resource/Places_Victoria, http://dbpedia.org/resource/Vilmorin, http://dbpedia.org/resource/Walker_Corporation, http://dbpedia.org/resource/Stockland, http://dbpedia.org/resource/Sportscraft, http://dbpedia.org/resource/Sportsgirl, http://dbpedia.org/resource/Supré, http://dbpedia.org/resource/Stageco, http://dbpedia.org/resource/Ayam_Brand, http://dbpedia.org/resource/Hoyts_Kiosk, http://dbpedia.org/resource/Energy_Networks_Association_(Australia), http://dbpedia.org/resource/Khaadi, http://dbpedia.org/resource/World_Malayalee_Council, http://dbpedia.org/resource/Commonwealth_Oil_Refineries, http://dbpedia.org/resource/Vivo_Miles, http://dbpedia.org/resource/Pepkor, http://dbpedia.org/resource/MadBid, http://dbpedia.org/resource/ReachLocal, http://dbpedia.org/resource/Dairy_Industry_Association_of_Australia, http://dbpedia.org/resource/Flirtomatic, http://dbpedia.org/resource/Ernest_Hillier_Chocolates, http://dbpedia.org/resource/Connexus_Internet, http://dbpedia.org/resource/Traplord, http://dbpedia.org/resource/SESL_Australia, http://dbpedia.org/resource/Inteliment, http://dbpedia.org/resource/Essential_Foods, http://dbpedia.org/resource/Austin_Adventures, http://dbpedia.org/resource/MyNetFone, http://dbpedia.org/resource/Pet_Circle, http://dbpedia.org/resource/Australia_Post, http://dbpedia.org/resource/Australian_Skeptics, http://dbpedia.org/resource/Australian_Youth_Climate_Coalition, http://dbpedia.org/resource/Bank_of_Queensland, http://dbpedia.org/resource/National_Storage, http://dbpedia.org/resource/Repco, http://dbpedia.org/resource/Royal_Australian_College_of_General_Practitioners, http://dbpedia.org/resource/Vero_Insurance, http://dbpedia.org/resource/Oberon_Books, http://dbpedia.org/resource/Woolworths_Limited, http://dbpedia.org/resource/Royal_Australian_and_New_Zealand_College_of_Psychiatrists, http://dbpedia.org/resource/BT_Financial_Group, http://dbpedia.org/resource/Tax_Institute_(Australia), http://dbpedia.org/resource/Stripe_(company), http://dbpedia.org/resource/Nutro_Products, http://dbpedia.org/resource/Puma_Energy, http://dbpedia.org/resource/Frucor, http://dbpedia.org/resource/Tobu_Top_Tours, http://dbpedia.org/resource/Australian_Underwater_Federation, http://dbpedia.org/resource/Board_of_Studies,_Teaching_and_Educational_Standards, http://dbpedia.org/resource/Pacific_Hydro, http://dbpedia.org/resource/Bancolombia, http://dbpedia.org/resource/Amaysim, http://dbpedia.org/resource/Terria_(consortium), http://dbpedia.org/resource/Youth_Climate_Movement, http://dbpedia.org/resource/GreenPlumbers, http://dbpedia.org/resource/Country_Garden, http://dbpedia.org/resource/Artemis_(pet_food), http://dbpedia.org/resource/CA_Sports, http://dbpedia.org/resource/Penta_Security, http://dbpedia.org/resource/Chocolates_El_Rey, http://dbpedia.org/resource/Google_Glass_breastfeeding_app_trial, http://dbpedia.org/resource/Luxe_Link, http://dbpedia.org/resource/Ruralco, http://dbpedia.org/resource/Iveco, http://dbpedia.org/resource/Isagenix_International, http://dbpedia.org/resource/Protandim, http://dbpedia.org/resource/Pandora_Radio, http://dbpedia.org/resource/Keolis_Downer, http://dbpedia.org/resource/Pacha_Group, http://dbpedia.org/resource/Sunsuper, http://dbpedia.org/resource/Umbrella_Entertainment, http://dbpedia.org/resource/AMP_Capital, http://dbpedia.org/resource/Spearmint_Rhino, http://dbpedia.org/resource/Disney_Media_Distribution, http://dbpedia.org/resource/Australian_Academy_of_Cinema_and_Television_Arts, http://dbpedia.org/resource/Parker_Hale, http://dbpedia.org/resource/Virgin_Unite, http://dbpedia.org/resource/Australian_Automobile_Association, http://dbpedia.org/resource/Penske_Automotive_Group, http://dbpedia.org/resource/Australian_Industry_Group, http://dbpedia.org/resource/APN_News_&_Media, http://dbpedia.org/resource/Vanguard_Dutch_Marine, http://dbpedia.org/resource/Australian_Men's_Shed_Association, http://dbpedia.org/resource/Transgender_Victoria, http://dbpedia.org/resource/Cave_Divers_Association_of_Australia, http://dbpedia.org/resource/Classic_International_Cruises, http://dbpedia.org/resource/Bank_of_Melbourne_(1989), http://dbpedia.org/resource/Australian_Watercolour_Institute, http://dbpedia.org/resource/Trustpilot, http://dbpedia.org/resource/MGM_Distribution, http://dbpedia.org/resource/TPG_Telecom, http://dbpedia.org/resource/Bridgecorp_Holdings, http://dbpedia.org/resource/Australian_National_Committee_on_Large_Dams, http://dbpedia.org/resource/Qube_Holdings, http://dbpedia.org/resource/Bandit.fm, http://dbpedia.org/resource/GEO_Group_Australia, http://dbpedia.org/resource/Teachers_Mutual_Bank, http://dbpedia.org/resource/Toll_Global_Express, http://dbpedia.org/resource/Toll_Resources_&_Government_Logistics, http://dbpedia.org/resource/Bank_of_Beirut, http://dbpedia.org/resource/Suncorp_Bank, http://dbpedia.org/resource/HotelQuickly, http://dbpedia.org/resource/Mortein, http://dbpedia.org/resource/Climate_Council, http://dbpedia.org/resource/Seek_Limited, http://dbpedia.org/resource/Sydney_Airport_Holdings, http://dbpedia.org/resource/WAPET, http://dbpedia.org/resource/Australia_Pacific_Airports_Corporation, http://dbpedia.org/resource/WellPet, http://dbpedia.org/resource/Budget_Direct, http://dbpedia.org/resource/Dansko, http://dbpedia.org/resource/Kirby_Institute, http://dbpedia.org/resource/Warsop_Stebbing, http://dbpedia.org/resource/DittoTV, http://dbpedia.org/resource/Australian_Diabetes_Council, http://dbpedia.org/resource/Bellamy's_Australia, http://dbpedia.org/resource/Temperzone, http://dbpedia.org/resource/Yodel_Australia, http://dbpedia.org/resource/National_Australia_Bank, http://dbpedia.org/resource/H&R_Block, http://dbpedia.org/resource/Beyondblue, http://dbpedia.org/resource/Board_of_Studies, http://dbpedia.org/resource/Singtel, http://dbpedia.org/resource/Lowe's, http://dbpedia.org/resource/XL_Leisure_Group, http://dbpedia.org/resource/Crossmark, http://dbpedia.org/resource/Sparx_Systems, http://dbpedia.org/resource/Thumpstar, http://dbpedia.org/resource/Chase_Motor_Truck_Company, http://dbpedia.org/resource/XYZnetworks, http://dbpedia.org/resource/Goodman_Group, http://dbpedia.org/resource/Carrian_Group, http://dbpedia.org/resource/Carpet_One, http://dbpedia.org/resource/CSG_Limited, http://dbpedia.org/resource/Fastway_Couriers, http://dbpedia.org/resource/Attivio, http://dbpedia.org/resource/Buzzle, http://dbpedia.org/resource/Penfield_Outdoor_Apparel, http://dbpedia.org/resource/City_Beach_(retailer), http://dbpedia.org/resource/Lebara, http://dbpedia.org/resource/LeadGenius, http://dbpedia.org/resource/Laguna_Resources, http://dbpedia.org/resource/Broadspectrum, http://dbpedia.org/resource/Transpacific_Industries_Group, http://dbpedia.org/resource/Bunnell_Incorporated, http://dbpedia.org/resource/DirectMoney, http://dbpedia.org/resource/One_Way_Sport, http://dbpedia.org/resource/Anthropological_Society_of_New_South_Wales, http://dbpedia.org/resource/Bongotones.com, http://dbpedia.org/resource/Travel_All_Russia_LLC, http://dbpedia.org/resource/Aboriginal_Provisional_Government, http://dbpedia.org/resource/Mega_International_Commercial_Bank, http://dbpedia.org/resource/Tassal, http://dbpedia.org/resource/Desert_Fireball_Network, http://dbpedia.org/resource/Freeview_(Australia), http://dbpedia.org/resource/The_Reject_Shop, http://dbpedia.org/resource/Australian_Institute_of_Building, http://dbpedia.org/resource/Powershop, http://dbpedia.org/resource/Quickflix, http://dbpedia.org/resource/Dollar_Shave_Club, http://dbpedia.org/resource/Wyndham_Vacation_Resorts_Asia_Pacific, http://dbpedia.org/resource/MSD_Ignition, http://dbpedia.org/resource/Community_Broadcasting_Association_of_Australia, http://dbpedia.org/resource/Origin_Net, http://dbpedia.org/resource/Patersons_Securities, http://dbpedia.org/resource/Harvey_World_Travel, http://dbpedia.org/resource/BigAir, http://dbpedia.org/resource/Mamee_Double-Decker, http://dbpedia.org/resource/Fastsigns, http://dbpedia.org/resource/Bold_Earth_Teen_Adventures, http://dbpedia.org/resource/Ocean_Power_Technologies_Australasia, http://dbpedia.org/resource/Flight_Centre_Australia, http://dbpedia.org/resource/Preston_Group, http://dbpedia.org/resource/Aquila_Shoes, http://dbpedia.org/resource/National_Hearing_Care, http://dbpedia.org/resource/ACCO_Brands, http://dbpedia.org/resource/National_Foods, http://dbpedia.org/resource/New_Hope_Coal, http://dbpedia.org/resource/AAMI, http://dbpedia.org/resource/Stagecoach_Theatre_Arts, http://dbpedia.org/resource/Padini, http://dbpedia.org/resource/Surf_Life_Saving_Australia, http://dbpedia.org/resource/SEEK_Learning, http://dbpedia.org/resource/Australian_Multiplex_Cinemas, http://dbpedia.org/resource/P&O_Cruises_Australia, http://dbpedia.org/resource/Royal_Purple_(lubricant_manufacturer), http://dbpedia.org/resource/Kookaburra_Sport, http://dbpedia.org/resource/Bellamy's_Organic, http://dbpedia.org/resource/Caparison_Guitars, http://dbpedia.org/resource/Metricon, http://dbpedia.org/resource/Linfox, http://dbpedia.org/resource/National_Safety_Council_of_Australia, http://dbpedia.org/resource/Mad_Mex_Fresh_Mexican_Grill, http://dbpedia.org/resource/Safe_Schools_Coalition_Australia, http://dbpedia.org/resource/People_with_Disability_Australia, http://dbpedia.org/resource/Spiegelworld, http://dbpedia.org/resource/Centro_Shopping_America_Trust, http://dbpedia.org/resource/Nagarro, http://dbpedia.org/resource/Datawatch_Corporation, http://dbpedia.org/resource/SolGold, http://dbpedia.org/resource/Raja_Fashions, http://dbpedia.org/resource/Betstar, http://dbpedia.org/resource/Stratton_(Financial_Services), http://dbpedia.org/resource/Juvenilia_Press, http://dbpedia.org/resource/Syspro, http://dbpedia.org/resource/Global_Payments, http://dbpedia.org/resource/Berkeley_Cinemas, http://dbpedia.org/resource/TicketLeap, http://dbpedia.org/resource/Yupi_(confectioner), http://dbpedia.org/resource/Akraya_Inc., http://dbpedia.org/resource/Stockspot, http://dbpedia.org/resource/Ochre_Health, http://dbpedia.org/resource/Spuul, http://dbpedia.org/resource/Community_First_Credit_Union, http://dbpedia.org/resource/Salsa's_Fresh_Mex_Grill, http://dbpedia.org/resource/Recommind, http://dbpedia.org/resource/Superannuated_Commonwealth_Officers'_Association, http://dbpedia.org/resource/Mortgage_Choice, http://dbpedia.org/resource/Origin_Kebabs, http://dbpedia.org/resource/Dreamtilt, http://dbpedia.org/resource/OurSay, http://dbpedia.org/resource/Turkey_Travel_Centre, http://dbpedia.org/resource/Intervarsity_Choral_Festival_(Australia)__Australian_Intervarsity_Choral_Societies_Associati__1] [INFO] Predicted Answer: [http://dbpedia.org/resource/Skytraders] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 382, P: 0.523, R: 0.581, macro F1: 0.531, macro F1*: 0.551 -[INFO] Current time: 13041 ms, average: 2880.71 ms/ques, entity detection: 451.49 ms/ques (linking tool: 361.38 ms/ques), relation detection: 934.21 ms/ques (similarity: 718.26 ms/ques), EDG generation: 57.66 ms/ques +[INFO] Cumulative metrics, sample: 382, P: 0.523, R: 0.581, macro F1: 0.551 +[INFO] Current question consumed time: 12015 ms, average: 2657.40 ms/ques, entity detection: 412.59 ms/ques (linking tool: 331.90 ms/ques), relation detection: 878.06 ms/ques (similarity: 699.30 ms/ques), EDG generation: 57.61 ms/ques [INFO] Entity Linking P: 0.7369109947643979, R: 0.6989528795811518, micro F1: 0.7107329842931935, macro F1: 0.7174302118581376 [INFO] Relation Linking P: 0.5907504363001745, R: 0.5589005235602095, micro F1: 0.5659685863874342, macro F1: 0.5743842951806631 [INFO] Type Linking P: 0.5261780104712042, R: 0.5261780104712042, micro F1: 0.5261780104712042, macro F1: 0.5261780104712042 @@ -5360,8 +5360,8 @@ Question 382: In which countries is coconut used as a food ingredient? [INFO] Golden Answer: [Uncertain; South Asia or Scotland@en, http://dbpedia.org/resource/Vietnam, http://dbpedia.org/resource/Turkey, http://dbpedia.org/resource/Malaysia, http://dbpedia.org/resource/India, http://dbpedia.org/resource/United_States, http://dbpedia.org/resource/Brisbane, http://dbpedia.org/resource/Maharashtra, http://dbpedia.org/resource/Sri_Lanka, http://dbpedia.org/resource/England, http://dbpedia.org/resource/Philippines, http://dbpedia.org/resource/Caribbean, Bosnia/Croatia/Serbia@en, http://dbpedia.org/resource/China, http://dbpedia.org/resource/New_Zealand, http://dbpedia.org/resource/Karnataka, http://dbpedia.org/resource/Indonesia, http://dbpedia.org/resource/Jamaica, http://dbpedia.org/resource/Mexico, http://dbpedia.org/resource/Colombia, http://dbpedia.org/resource/Myanmar, http://dbpedia.org/resource/Panama, http://dbpedia.org/resource/Thailand, http://dbpedia.org/resource/Venezuela, http://dbpedia.org/resource/Hong_Kong, Caribbean@en, http://dbpedia.org/resource/Brazil, Laos and Thailand@en, http://dbpedia.org/resource/Australia, http://dbpedia.org/resource/Cuba, http://dbpedia.org/resource/Cambodia, http://dbpedia.org/resource/Norway, http://dbpedia.org/resource/Tahiti, http://dbpedia.org/resource/Portugal, http://dbpedia.org/resource/Angola, http://dbpedia.org/resource/Singapore, http://dbpedia.org/resource/Canada, http://dbpedia.org/resource/Bangladesh, http://dbpedia.org/resource/Laos, http://dbpedia.org/resource/Java, http://dbpedia.org/resource/Levant, United States@en, Bangladesh@en, India@en] [INFO] Predicted Answer: [http://dbpedia.org/resource/Chicken_tikka_masala, http://dbpedia.org/resource/Coconut_milk, http://dbpedia.org/resource/Kho_(cooking_technique), http://dbpedia.org/resource/Cezerye, http://dbpedia.org/resource/Coconut_jam, http://dbpedia.org/resource/Coconut_chutney, http://dbpedia.org/resource/Dhondas, http://dbpedia.org/resource/Kulolo, http://dbpedia.org/resource/Naan_qalia, http://dbpedia.org/resource/Ada_(food), http://dbpedia.org/resource/Lamington, http://dbpedia.org/resource/Modak, http://dbpedia.org/resource/Puttu, http://dbpedia.org/resource/Liquorice_allsorts, http://dbpedia.org/resource/Coconut_candy, http://dbpedia.org/resource/Palitaw, http://dbpedia.org/resource/Sugar_cake, http://dbpedia.org/resource/Xacuti, http://dbpedia.org/resource/Ruske_kape, http://dbpedia.org/resource/Nuomici, http://dbpedia.org/resource/Cascaron, http://dbpedia.org/resource/Cameo_Creme, http://dbpedia.org/resource/Krispie, http://dbpedia.org/resource/Chicken_Chettinad, http://dbpedia.org/resource/Kora_Khai, http://dbpedia.org/resource/Nice_biscuit, http://dbpedia.org/resource/German_chocolate_cake, http://dbpedia.org/resource/Kutsinta, http://dbpedia.org/resource/Ambrosia_(fruit_salad), http://dbpedia.org/resource/Mangalore_bajji, http://dbpedia.org/resource/Sata_(food), http://dbpedia.org/resource/Getuk, http://dbpedia.org/resource/Grater_cake, http://dbpedia.org/resource/Toto_(dessert), http://dbpedia.org/resource/Kẹo_dừa, http://dbpedia.org/resource/Pudachi_Vadi, http://dbpedia.org/resource/Coconut_cake, http://dbpedia.org/resource/Lane_cake, http://dbpedia.org/resource/Cocadas, http://dbpedia.org/resource/Coconut_rice, http://dbpedia.org/resource/Serabi, http://dbpedia.org/resource/Cocktail_bun, http://dbpedia.org/resource/Espasol, http://dbpedia.org/resource/Ducana, http://dbpedia.org/resource/Beijinho, http://dbpedia.org/resource/Miang_kham, http://dbpedia.org/resource/Es_campur, http://dbpedia.org/resource/Es_teler, http://dbpedia.org/resource/Manjar_branco, http://dbpedia.org/resource/Maria-mole, http://dbpedia.org/resource/Olho-de-sogra, http://dbpedia.org/resource/Quindim, http://dbpedia.org/resource/White_Christmas_(food), http://dbpedia.org/resource/Cucurucho, http://dbpedia.org/resource/Amok_trey, http://dbpedia.org/resource/Pan_de_coco, http://dbpedia.org/resource/Kalathappam, http://dbpedia.org/resource/Manda_Pitha, http://dbpedia.org/resource/Kuluban, http://dbpedia.org/resource/Skolebrød, http://dbpedia.org/resource/Otap_(food), http://dbpedia.org/resource/Theeyal, http://dbpedia.org/resource/Urap, http://dbpedia.org/resource/Manchester_tart, http://dbpedia.org/resource/Po'e, http://dbpedia.org/resource/Queijadinha, http://dbpedia.org/resource/Cocada_amarela, http://dbpedia.org/resource/Baye_baye, http://dbpedia.org/resource/Buntil, http://dbpedia.org/resource/Kue_putu_mangkok, http://dbpedia.org/resource/Conkies, http://dbpedia.org/resource/Coconut_doughnut, http://dbpedia.org/resource/Klepon, http://dbpedia.org/resource/Lupis_(food), http://dbpedia.org/resource/Malpua, http://dbpedia.org/resource/Nam_khao, http://dbpedia.org/resource/Serundeng, http://dbpedia.org/resource/Wingko, http://dbpedia.org/resource/Gizzada, http://dbpedia.org/resource/Dragon's_beard_candy, http://dbpedia.org/resource/Sabudana_Khichadi, http://dbpedia.org/resource/Meghli, http://dbpedia.org/resource/Ceylon_(curry), http://dbpedia.org/resource/Sno_Balls, http://dbpedia.org/resource/Kozhukkatta, http://dbpedia.org/resource/Enduri_Pitha, http://dbpedia.org/resource/Puliyogare, http://dbpedia.org/resource/Kakara_pitha, http://dbpedia.org/resource/Happy_Cake, http://dbpedia.org/resource/Geplak, http://dbpedia.org/resource/Unnakai, http://dbpedia.org/resource/Daab_Chingri, http://dbpedia.org/resource/Kobbari_Lavuju, http://dbpedia.org/resource/Alu_Potala_Rasa] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 383, P: 0.522, R: 0.580, macro F1: 0.530, macro F1*: 0.549 -[INFO] Current time: 2240 ms, average: 2879.04 ms/ques, entity detection: 450.68 ms/ques (linking tool: 360.64 ms/ques), relation detection: 934.21 ms/ques (similarity: 719.04 ms/ques), EDG generation: 57.55 ms/ques +[INFO] Cumulative metrics, sample: 383, P: 0.522, R: 0.580, macro F1: 0.549 +[INFO] Current question consumed time: 2092 ms, average: 2655.93 ms/ques, entity detection: 411.83 ms/ques (linking tool: 331.21 ms/ques), relation detection: 878.02 ms/ques (similarity: 700.10 ms/ques), EDG generation: 57.50 ms/ques [INFO] Entity Linking P: 0.737597911227154, R: 0.6997389033942559, micro F1: 0.7114882506527413, macro F1: 0.718169810023169 [INFO] Relation Linking P: 0.5918189730200174, R: 0.5587467362924282, micro F1: 0.5662315056570927, macro F1: 0.5748075350663407 [INFO] Type Linking P: 0.5248041775456919, R: 0.5248041775456919, micro F1: 0.5248041775456919, macro F1: 0.5248041775456919 @@ -5374,8 +5374,8 @@ Question 383: How many people have worked for teams chaired by Patricia Amorim? [INFO] Golden Answer: [103] [INFO] Predicted Answer: [6, 4] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 384, P: 0.520, R: 0.578, macro F1: 0.529, macro F1*: 0.548 -[INFO] Current time: 4054 ms, average: 2882.10 ms/ques, entity detection: 449.86 ms/ques (linking tool: 359.90 ms/ques), relation detection: 935.54 ms/ques (similarity: 721.24 ms/ques), EDG generation: 57.45 ms/ques +[INFO] Cumulative metrics, sample: 384, P: 0.520, R: 0.578, macro F1: 0.548 +[INFO] Current question consumed time: 3751 ms, average: 2658.78 ms/ques, entity detection: 411.03 ms/ques (linking tool: 330.50 ms/ques), relation detection: 879.33 ms/ques (similarity: 702.35 ms/ques), EDG generation: 57.43 ms/ques [INFO] Entity Linking P: 0.73828125, R: 0.7005208333333334, micro F1: 0.7122395833333331, macro F1: 0.7189055429864253 [INFO] Relation Linking P: 0.591579861111111, R: 0.55859375, micro F1: 0.5660590277777774, macro F1: 0.5746137971698113 [INFO] Type Linking P: 0.5234375, R: 0.5234375, micro F1: 0.5234375, macro F1: 0.5234375 @@ -5388,8 +5388,8 @@ Question 384: How many kinds of games can be played on the Amiga? [INFO] Golden Answer: [173] [INFO] Predicted Answer: [1170] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 385, P: 0.519, R: 0.577, macro F1: 0.527, macro F1*: 0.546 -[INFO] Current time: 2207 ms, average: 2880.35 ms/ques, entity detection: 449.66 ms/ques (linking tool: 359.66 ms/ques), relation detection: 934.55 ms/ques (similarity: 720.63 ms/ques), EDG generation: 57.35 ms/ques +[INFO] Cumulative metrics, sample: 385, P: 0.519, R: 0.577, macro F1: 0.546 +[INFO] Current question consumed time: 1876 ms, average: 2656.75 ms/ques, entity detection: 410.85 ms/ques (linking tool: 330.30 ms/ques), relation detection: 878.37 ms/ques (similarity: 701.79 ms/ques), EDG generation: 57.32 ms/ques [INFO] Entity Linking P: 0.7389610389610389, R: 0.7012987012987013, micro F1: 0.7129870129870128, macro F1: 0.719637441008045 [INFO] Relation Linking P: 0.5926406926406926, R: 0.5584415584415584, micro F1: 0.5663203463203459, macro F1: 0.5750330902643802 [INFO] Type Linking P: 0.5246753246753246, R: 0.5246753246753246, micro F1: 0.5246753246753246, macro F1: 0.5246753246753246 @@ -5402,8 +5402,8 @@ Question 385: Is the book The Buddha and his Dhamma a book about Buddhism? [INFO] Golden Answer: [true] [INFO] Predicted Answer: [true] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 386, P: 0.520, R: 0.578, macro F1: 0.529, macro F1*: 0.547 -[INFO] Current time: 2827 ms, average: 2880.21 ms/ques, entity detection: 450.57 ms/ques (linking tool: 360.52 ms/ques), relation detection: 934.45 ms/ques (similarity: 720.24 ms/ques), EDG generation: 57.24 ms/ques +[INFO] Cumulative metrics, sample: 386, P: 0.520, R: 0.578, macro F1: 0.547 +[INFO] Current question consumed time: 2696 ms, average: 2656.85 ms/ques, entity detection: 411.65 ms/ques (linking tool: 331.04 ms/ques), relation detection: 878.31 ms/ques (similarity: 701.41 ms/ques), EDG generation: 57.22 ms/ques [INFO] Entity Linking P: 0.7396373056994818, R: 0.7020725388601037, micro F1: 0.7137305699481864, macro F1: 0.7203655340325592 [INFO] Relation Linking P: 0.5936960276338514, R: 0.5595854922279793, micro F1: 0.567443868739205, macro F1: 0.576136317344419 [INFO] Type Linking P: 0.5259067357512953, R: 0.5259067357512953, micro F1: 0.5259067357512953, macro F1: 0.5259067357512953 @@ -5416,8 +5416,8 @@ Question 386: How many relatives are there of people fought in the Battle of the [INFO] Golden Answer: [8] [INFO] Predicted Answer: [330] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 387, P: 0.519, R: 0.576, macro F1: 0.527, macro F1*: 0.546 -[INFO] Current time: 1857 ms, average: 2877.57 ms/ques, entity detection: 450.06 ms/ques (linking tool: 360.10 ms/ques), relation detection: 933.42 ms/ques (similarity: 719.66 ms/ques), EDG generation: 57.15 ms/ques +[INFO] Cumulative metrics, sample: 387, P: 0.519, R: 0.576, macro F1: 0.546 +[INFO] Current question consumed time: 1688 ms, average: 2654.34 ms/ques, entity detection: 411.19 ms/ques (linking tool: 330.68 ms/ques), relation detection: 877.25 ms/ques (similarity: 700.95 ms/ques), EDG generation: 57.12 ms/ques [INFO] Entity Linking P: 0.7403100775193798, R: 0.7028423772609819, micro F1: 0.714470284237726, macro F1: 0.7210898516930038 [INFO] Relation Linking P: 0.5921619293712317, R: 0.5581395348837209, micro F1: 0.5659776055124888, macro F1: 0.5746475930101957 [INFO] Type Linking P: 0.5271317829457365, R: 0.5271317829457365, micro F1: 0.5271317829457365, macro F1: 0.5271317829457365 @@ -5430,8 +5430,8 @@ Question 387: How many genres of games can I find on the commodore CDTV? [INFO] Golden Answer: [7] [INFO] Predicted Answer: [10] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 388, P: 0.517, R: 0.575, macro F1: 0.526, macro F1*: 0.545 -[INFO] Current time: 1085 ms, average: 2872.95 ms/ques, entity detection: 449.17 ms/ques (linking tool: 359.30 ms/ques), relation detection: 931.84 ms/ques (similarity: 718.55 ms/ques), EDG generation: 57.08 ms/ques +[INFO] Cumulative metrics, sample: 388, P: 0.517, R: 0.575, macro F1: 0.545 +[INFO] Current question consumed time: 1005 ms, average: 2650.09 ms/ques, entity detection: 410.35 ms/ques (linking tool: 329.94 ms/ques), relation detection: 875.80 ms/ques (similarity: 699.96 ms/ques), EDG generation: 57.05 ms/ques [INFO] Entity Linking P: 0.740979381443299, R: 0.7036082474226805, micro F1: 0.7152061855670102, macro F1: 0.7218104233149709 [INFO] Relation Linking P: 0.593213058419244, R: 0.5579896907216495, micro F1: 0.5662371134020614, macro F1: 0.5750625096168642 [INFO] Type Linking P: 0.5283505154639175, R: 0.5283505154639175, micro F1: 0.5283505154639175, macro F1: 0.5283505154639175 @@ -5444,8 +5444,8 @@ Question 388: Does Jevan Snead play as Quarterback? [INFO] Golden Answer: [true] [INFO] Predicted Answer: [false] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 389, P: 0.516, R: 0.573, macro F1: 0.524, macro F1*: 0.543 -[INFO] Current time: 2853 ms, average: 2872.89 ms/ques, entity detection: 449.81 ms/ques (linking tool: 359.77 ms/ques), relation detection: 931.92 ms/ques (similarity: 717.98 ms/ques), EDG generation: 56.96 ms/ques +[INFO] Cumulative metrics, sample: 389, P: 0.516, R: 0.573, macro F1: 0.543 +[INFO] Current question consumed time: 2555 ms, average: 2649.85 ms/ques, entity detection: 410.89 ms/ques (linking tool: 330.37 ms/ques), relation detection: 875.85 ms/ques (similarity: 699.46 ms/ques), EDG generation: 56.95 ms/ques [INFO] Entity Linking P: 0.7390745501285347, R: 0.7017994858611826, micro F1: 0.7133676092544986, macro F1: 0.7199548695275289 [INFO] Relation Linking P: 0.591688089117395, R: 0.5565552699228792, micro F1: 0.5647814910025702, macro F1: 0.5735841998235045 [INFO] Type Linking P: 0.5295629820051414, R: 0.5295629820051414, micro F1: 0.5295629820051414, macro F1: 0.5295629820051414 @@ -5458,8 +5458,8 @@ Question 389: Give me the sitcoms where Eliza Schneider and Isaac Hayes gave voi [INFO] Golden Answer: [http://dbpedia.org/resource/South_Park] [INFO] Predicted Answer: [http://dbpedia.org/resource/South_Park] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 390, P: 0.517, R: 0.574, macro F1: 0.526, macro F1*: 0.544 -[INFO] Current time: 3294 ms, average: 2873.97 ms/ques, entity detection: 450.26 ms/ques (linking tool: 359.76 ms/ques), relation detection: 931.71 ms/ques (similarity: 718.43 ms/ques), EDG generation: 56.91 ms/ques +[INFO] Cumulative metrics, sample: 390, P: 0.517, R: 0.574, macro F1: 0.544 +[INFO] Current question consumed time: 2977 ms, average: 2650.69 ms/ques, entity detection: 411.27 ms/ques (linking tool: 330.36 ms/ques), relation detection: 875.58 ms/ques (similarity: 700.00 ms/ques), EDG generation: 56.89 ms/ques [INFO] Entity Linking P: 0.7397435897435898, R: 0.7025641025641025, micro F1: 0.714102564102564, macro F1: 0.720674643874644 [INFO] Relation Linking P: 0.5901709401709402, R: 0.5551282051282052, micro F1: 0.5633333333333329, macro F1: 0.5721134711060085 [INFO] Type Linking P: 0.5282051282051282, R: 0.5282051282051282, micro F1: 0.5282051282051282, macro F1: 0.5282051282051282 @@ -5472,8 +5472,8 @@ Question 390: Directors of which movies are buried in Ronda? [INFO] Golden Answer: [http://dbpedia.org/resource/One_Man_Band_(unfinished_film), http://dbpedia.org/resource/The_Heroine, http://dbpedia.org/resource/Citizen_Kane, http://dbpedia.org/resource/The_Magnificent_Ambersons_(film), http://dbpedia.org/resource/The_Lady_from_Shanghai, http://dbpedia.org/resource/The_Stranger_(1946_film), http://dbpedia.org/resource/David_and_Goliath_(1960_film), http://dbpedia.org/resource/F_for_Fake, http://dbpedia.org/resource/The_Trial_(1962_film), http://dbpedia.org/resource/Chimes_at_Midnight, http://dbpedia.org/resource/Citizen_Kane_trailer, http://dbpedia.org/resource/Mr._Arkadin, http://dbpedia.org/resource/Touch_of_Evil, http://dbpedia.org/resource/Journey_into_Fear_(1943_film), http://dbpedia.org/resource/Too_Much_Johnson, http://dbpedia.org/resource/The_Deep_(unfinished_film), http://dbpedia.org/resource/Moby_Dick_(unfinished_film), http://dbpedia.org/resource/It's_All_True_(film), http://dbpedia.org/resource/In_the_Land_of_Don_Quixote, http://dbpedia.org/resource/The_Merchant_of_Venice_(1969_film), http://dbpedia.org/resource/The_Miracle_of_St._Anne, http://dbpedia.org/resource/Vienna_(film), http://dbpedia.org/resource/Othello_(1952_film), http://dbpedia.org/resource/Around_the_World_with_Orson_Welles, http://dbpedia.org/resource/Black_Magic_(1949_film), http://dbpedia.org/resource/Filming_Othello, http://dbpedia.org/resource/Filming_The_Trial, http://dbpedia.org/resource/Orson_Welles'_Magic_Show, http://dbpedia.org/resource/The_Fountain_of_Youth_(film), http://dbpedia.org/resource/The_Immortal_Story, http://dbpedia.org/resource/The_Other_Side_of_the_Wind, http://dbpedia.org/resource/Three_Cases_of_Murder, http://dbpedia.org/resource/Macbeth_(1948_film), http://dbpedia.org/resource/Don_Quixote_(unfinished_film), http://dbpedia.org/resource/Orson_Welles_and_People] [INFO] Predicted Answer: [http://dbpedia.org/resource/Orson_Welles] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 391, P: 0.516, R: 0.573, macro F1: 0.524, macro F1*: 0.543 -[INFO] Current time: 1636 ms, average: 2870.81 ms/ques, entity detection: 449.59 ms/ques (linking tool: 359.07 ms/ques), relation detection: 930.85 ms/ques (similarity: 718.22 ms/ques), EDG generation: 56.82 ms/ques +[INFO] Cumulative metrics, sample: 391, P: 0.516, R: 0.573, macro F1: 0.543 +[INFO] Current question consumed time: 1445 ms, average: 2647.60 ms/ques, entity detection: 410.58 ms/ques (linking tool: 329.68 ms/ques), relation detection: 874.78 ms/ques (similarity: 699.94 ms/ques), EDG generation: 56.83 ms/ques [INFO] Entity Linking P: 0.7404092071611253, R: 0.7033248081841432, micro F1: 0.7148337595907927, macro F1: 0.7213907244262514 [INFO] Relation Linking P: 0.5912190963341858, R: 0.5549872122762148, micro F1: 0.5635976129582263, macro F1: 0.5725305045943547 [INFO] Type Linking P: 0.5294117647058824, R: 0.5294117647058824, micro F1: 0.5294117647058824, macro F1: 0.5294117647058824 @@ -5486,8 +5486,8 @@ Question 391: List the debut team of the baseball player who born in Houston. [INFO] Golden Answer: [http://dbpedia.org/resource/Cincinnati_Reds, http://dbpedia.org/resource/Houston_Astros, http://dbpedia.org/resource/Tampa_Bay_Rays, http://dbpedia.org/resource/San_Francisco_Giants, http://dbpedia.org/resource/San_Diego_Padres, http://dbpedia.org/resource/Philadelphia_Phillies, http://dbpedia.org/resource/Los_Angeles_Dodgers, http://dbpedia.org/resource/Boston_Red_Sox, http://dbpedia.org/resource/Texas_Rangers_(baseball), http://dbpedia.org/resource/Atlanta_Braves, http://dbpedia.org/resource/Pittsburgh_Pirates, http://dbpedia.org/resource/Seattle_Mariners, http://dbpedia.org/resource/Milwaukee_Brewers, http://dbpedia.org/resource/St._Louis_Cardinals, http://dbpedia.org/resource/Washington_Nationals, http://dbpedia.org/resource/New_York_Yankees, http://dbpedia.org/resource/Minnesota_Twins, http://dbpedia.org/resource/Montreal_Expos, http://dbpedia.org/resource/Toronto_Blue_Jays, http://dbpedia.org/resource/Colorado_Rockies, http://dbpedia.org/resource/Chicago_White_Sox, http://dbpedia.org/resource/Kansas_City_Royals, http://dbpedia.org/resource/Chiba_Lotte_Marines, http://dbpedia.org/resource/Detroit_Tigers, http://dbpedia.org/resource/Cleveland_Indians, http://dbpedia.org/resource/Arizona_Diamondbacks, http://dbpedia.org/resource/Birmingham_Black_Barons, http://dbpedia.org/resource/Chicago_Cubs, http://dbpedia.org/resource/Oakland_Athletics, http://dbpedia.org/resource/Los_Angeles_Angels_of_Anaheim, http://dbpedia.org/resource/New_York_Mets, http://dbpedia.org/resource/Miami_Marlins, http://dbpedia.org/resource/Chicago_American_Giants, http://dbpedia.org/resource/Kansas_City_Cowboys_(American_Association)] [INFO] Predicted Answer: [http://dbpedia.org/resource/Cincinnati_Reds, http://dbpedia.org/resource/Houston_Astros, http://dbpedia.org/resource/Tampa_Bay_Rays, http://dbpedia.org/resource/San_Francisco_Giants, http://dbpedia.org/resource/San_Diego_Padres, http://dbpedia.org/resource/Philadelphia_Phillies, http://dbpedia.org/resource/Los_Angeles_Dodgers, http://dbpedia.org/resource/Boston_Red_Sox, http://dbpedia.org/resource/Texas_Rangers_(baseball), http://dbpedia.org/resource/Atlanta_Braves, http://dbpedia.org/resource/Pittsburgh_Pirates, http://dbpedia.org/resource/Seattle_Mariners, http://dbpedia.org/resource/Milwaukee_Brewers, http://dbpedia.org/resource/St._Louis_Cardinals, http://dbpedia.org/resource/Washington_Nationals, http://dbpedia.org/resource/New_York_Yankees, http://dbpedia.org/resource/Minnesota_Twins, http://dbpedia.org/resource/Montreal_Expos, http://dbpedia.org/resource/Toronto_Blue_Jays, http://dbpedia.org/resource/Colorado_Rockies, http://dbpedia.org/resource/Chicago_White_Sox, http://dbpedia.org/resource/Kansas_City_Royals, http://dbpedia.org/resource/Chiba_Lotte_Marines, http://dbpedia.org/resource/Detroit_Tigers, http://dbpedia.org/resource/Cleveland_Indians, http://dbpedia.org/resource/Arizona_Diamondbacks, http://dbpedia.org/resource/Birmingham_Black_Barons, http://dbpedia.org/resource/Chicago_Cubs, http://dbpedia.org/resource/Oakland_Athletics, http://dbpedia.org/resource/Los_Angeles_Angels_of_Anaheim, http://dbpedia.org/resource/New_York_Mets, http://dbpedia.org/resource/Miami_Marlins, http://dbpedia.org/resource/Chicago_American_Giants, http://dbpedia.org/resource/Kansas_City_Cowboys_(American_Association)] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 392, P: 0.517, R: 0.574, macro F1: 0.526, macro F1*: 0.544 -[INFO] Current time: 6730 ms, average: 2880.65 ms/ques, entity detection: 450.42 ms/ques (linking tool: 359.91 ms/ques), relation detection: 935.67 ms/ques (similarity: 722.65 ms/ques), EDG generation: 56.74 ms/ques +[INFO] Cumulative metrics, sample: 392, P: 0.517, R: 0.574, macro F1: 0.544 +[INFO] Current question consumed time: 5992 ms, average: 2656.14 ms/ques, entity detection: 411.24 ms/ques (linking tool: 330.35 ms/ques), relation detection: 878.92 ms/ques (similarity: 704.43 ms/ques), EDG generation: 56.76 ms/ques [INFO] Entity Linking P: 0.7410714285714286, R: 0.7040816326530612, micro F1: 0.7155612244897958, macro F1: 0.7221031395788677 [INFO] Relation Linking P: 0.5922619047619048, R: 0.5561224489795918, micro F1: 0.5647108843537411, macro F1: 0.5736235256783202 [INFO] Type Linking P: 0.5280612244897959, R: 0.5280612244897959, micro F1: 0.5280612244897959, macro F1: 0.5280612244897959 @@ -5500,8 +5500,8 @@ Question 392: Does Cascade Airways have a hub in Seattle? [INFO] Golden Answer: [true] [INFO] Predicted Answer: [true] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 393, P: 0.519, R: 0.575, macro F1: 0.527, macro F1*: 0.545 -[INFO] Current time: 4475 ms, average: 2884.71 ms/ques, entity detection: 451.61 ms/ques (linking tool: 360.88 ms/ques), relation detection: 939.79 ms/ques (similarity: 724.32 ms/ques), EDG generation: 56.64 ms/ques +[INFO] Cumulative metrics, sample: 393, P: 0.519, R: 0.575, macro F1: 0.545 +[INFO] Current question consumed time: 4063 ms, average: 2659.72 ms/ques, entity detection: 412.26 ms/ques (linking tool: 331.24 ms/ques), relation detection: 882.80 ms/ques (similarity: 706.10 ms/ques), EDG generation: 56.66 ms/ques [INFO] Entity Linking P: 0.7417302798982188, R: 0.7035623409669212, micro F1: 0.7154368108566581, macro F1: 0.7221423323656955 [INFO] Relation Linking P: 0.5932994062765055, R: 0.5572519083969466, micro F1: 0.5658184902459706, macro F1: 0.5747109619220997 [INFO] Type Linking P: 0.5292620865139949, R: 0.5292620865139949, micro F1: 0.5292620865139949, macro F1: 0.5292620865139949 @@ -5514,8 +5514,8 @@ Question 393: Who edited British Gazette ? [INFO] Golden Answer: [http://dbpedia.org/resource/Winston_Churchill] [INFO] Predicted Answer: [http://dbpedia.org/resource/Winston_Churchill] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 394, P: 0.520, R: 0.576, macro F1: 0.528, macro F1*: 0.547 -[INFO] Current time: 444 ms, average: 2878.52 ms/ques, entity detection: 450.58 ms/ques (linking tool: 360.07 ms/ques), relation detection: 937.71 ms/ques (similarity: 722.67 ms/ques), EDG generation: 56.51 ms/ques +[INFO] Cumulative metrics, sample: 394, P: 0.520, R: 0.576, macro F1: 0.547 +[INFO] Current question consumed time: 421 ms, average: 2654.03 ms/ques, entity detection: 411.36 ms/ques (linking tool: 330.53 ms/ques), relation detection: 880.84 ms/ques (similarity: 704.48 ms/ques), EDG generation: 56.55 ms/ques [INFO] Entity Linking P: 0.7423857868020305, R: 0.7043147208121827, micro F1: 0.7161590524534686, macro F1: 0.7228493187282928 [INFO] Relation Linking P: 0.594331641285956, R: 0.5583756345177665, micro F1: 0.5669204737732652, macro F1: 0.5757928561449261 [INFO] Type Linking P: 0.5304568527918782, R: 0.5304568527918782, micro F1: 0.5304568527918782, macro F1: 0.5304568527918782 @@ -5528,8 +5528,8 @@ Question 394: List the things for which the relatives of Mrunalini Kunte are kno [INFO] Golden Answer: [http://dbpedia.org/resource/Chess] [INFO] Predicted Answer: [http://dbpedia.org/resource/Abhijit_Kunte] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 395, P: 0.518, R: 0.575, macro F1: 0.527, macro F1*: 0.545 -[INFO] Current time: 730 ms, average: 2873.08 ms/ques, entity detection: 449.77 ms/ques (linking tool: 359.30 ms/ques), relation detection: 935.95 ms/ques (similarity: 721.41 ms/ques), EDG generation: 56.45 ms/ques +[INFO] Cumulative metrics, sample: 395, P: 0.518, R: 0.575, macro F1: 0.545 +[INFO] Current question consumed time: 683 ms, average: 2649.04 ms/ques, entity detection: 410.67 ms/ques (linking tool: 329.84 ms/ques), relation detection: 879.18 ms/ques (similarity: 703.29 ms/ques), EDG generation: 56.49 ms/ques [INFO] Entity Linking P: 0.7430379746835443, R: 0.7050632911392405, micro F1: 0.7168776371308015, macro F1: 0.7235527131096751 [INFO] Relation Linking P: 0.5953586497890295, R: 0.5582278481012658, micro F1: 0.5671729957805902, macro F1: 0.5761956793495874 [INFO] Type Linking P: 0.5316455696202531, R: 0.5316455696202531, micro F1: 0.5316455696202531, macro F1: 0.5316455696202531 @@ -5542,8 +5542,8 @@ Question 395: What acts and artists are associated with people who have worked w [INFO] Golden Answer: [http://dbpedia.org/resource/Radu_Captari, http://dbpedia.org/resource/Divertis, http://dbpedia.org/resource/Eurovision_Song_Contest, http://dbpedia.org/resource/Cătălin_Măruță, http://dbpedia.org/resource/Marius_Moga, http://dbpedia.org/resource/Faimoşii, http://dbpedia.org/resource/Ploaia_de_stele] [INFO] Predicted Answer: [2006-12-31] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 396, P: 0.517, R: 0.573, macro F1: 0.525, macro F1*: 0.544 -[INFO] Current time: 1060 ms, average: 2868.50 ms/ques, entity detection: 448.84 ms/ques (linking tool: 358.53 ms/ques), relation detection: 934.53 ms/ques (similarity: 720.68 ms/ques), EDG generation: 56.49 ms/ques +[INFO] Cumulative metrics, sample: 396, P: 0.517, R: 0.573, macro F1: 0.544 +[INFO] Current question consumed time: 976 ms, average: 2644.82 ms/ques, entity detection: 409.80 ms/ques (linking tool: 329.13 ms/ques), relation detection: 877.88 ms/ques (similarity: 702.62 ms/ques), EDG generation: 56.53 ms/ques [INFO] Entity Linking P: 0.7411616161616161, R: 0.7032828282828283, micro F1: 0.71506734006734, macro F1: 0.7217255597937416 [INFO] Relation Linking P: 0.5938552188552189, R: 0.5568181818181818, micro F1: 0.5657407407407402, macro F1: 0.5747406397552703 [INFO] Type Linking P: 0.5328282828282829, R: 0.5328282828282829, micro F1: 0.5328282828282829, macro F1: 0.5328282828282829 @@ -5556,8 +5556,8 @@ Question 396: What are the notableworks of Russell T Davies? [INFO] Golden Answer: [http://dbpedia.org/resource/Doctor_Who, http://dbpedia.org/resource/Torchwood, http://dbpedia.org/resource/Cucumber_(TV_series), http://dbpedia.org/resource/Queer_as_Folk_(UK_TV_series), http://dbpedia.org/resource/The_Sarah_Jane_Adventures, http://dbpedia.org/resource/Casanova_(2005_TV_serial), http://dbpedia.org/resource/Banana_(TV_series), http://dbpedia.org/resource/Bob_&_Rose, http://dbpedia.org/resource/The_Second_Coming_(TV_serial), *@en, ''@en, *''@en] [INFO] Predicted Answer: [http://dbpedia.org/resource/Doctor_Who, http://dbpedia.org/resource/Torchwood, http://dbpedia.org/resource/Cucumber_(TV_series), http://dbpedia.org/resource/Queer_as_Folk_(UK_TV_series), http://dbpedia.org/resource/The_Sarah_Jane_Adventures, http://dbpedia.org/resource/Casanova_(2005_TV_serial), http://dbpedia.org/resource/Banana_(TV_series), http://dbpedia.org/resource/Bob_&_Rose, http://dbpedia.org/resource/The_Second_Coming_(TV_serial), *@en, ''@en, *''@en] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 397, P: 0.518, R: 0.574, macro F1: 0.526, macro F1*: 0.545 -[INFO] Current time: 1037 ms, average: 2863.88 ms/ques, entity detection: 448.31 ms/ques (linking tool: 357.97 ms/ques), relation detection: 932.88 ms/ques (similarity: 719.29 ms/ques), EDG generation: 56.38 ms/ques +[INFO] Cumulative metrics, sample: 397, P: 0.518, R: 0.574, macro F1: 0.545 +[INFO] Current question consumed time: 899 ms, average: 2640.42 ms/ques, entity detection: 409.28 ms/ques (linking tool: 328.61 ms/ques), relation detection: 876.34 ms/ques (similarity: 701.27 ms/ques), EDG generation: 56.41 ms/ques [INFO] Entity Linking P: 0.7418136020151134, R: 0.7040302267002518, micro F1: 0.7157850545759865, macro F1: 0.7224282291401539 [INFO] Relation Linking P: 0.5948782535684298, R: 0.5579345088161209, micro F1: 0.5668345927791767, macro F1: 0.5758144202421291 [INFO] Type Linking P: 0.5340050377833753, R: 0.5340050377833753, micro F1: 0.5340050377833753, macro F1: 0.5340050377833753 @@ -5570,8 +5570,8 @@ Question 397: Which company has developed games whose music is done by Gerard Ma [INFO] Golden Answer: [http://dbpedia.org/resource/PlayStation_4, http://dbpedia.org/resource/SCE_Santa_Monica_Studio, http://dbpedia.org/resource/Ready_at_Dawn, http://dbpedia.org/resource/Daybreak_Game_Company, http://dbpedia.org/resource/IOS, http://dbpedia.org/resource/Beenox, http://dbpedia.org/resource/Gameloft, http://dbpedia.org/resource/PlayStation_3, http://dbpedia.org/resource/Wii, http://dbpedia.org/resource/Xbox_360, http://dbpedia.org/resource/Nintendo_3DS] [INFO] Predicted Answer: [http://dbpedia.org/resource/SCE_Santa_Monica_Studio, http://dbpedia.org/resource/Ready_at_Dawn] [INFO] P: 1.0, R: 0.18181818181818182, F: 0.3076923076923077 -[INFO] Cumulative metrics, sample: 398, P: 0.520, R: 0.573, macro F1: 0.526, macro F1*: 0.545 -[INFO] Current time: 2065 ms, average: 2861.88 ms/ques, entity detection: 447.62 ms/ques (linking tool: 357.33 ms/ques), relation detection: 932.85 ms/ques (similarity: 719.62 ms/ques), EDG generation: 56.29 ms/ques +[INFO] Cumulative metrics, sample: 398, P: 0.520, R: 0.573, macro F1: 0.545 +[INFO] Current question consumed time: 1833 ms, average: 2638.39 ms/ques, entity detection: 408.66 ms/ques (linking tool: 328.03 ms/ques), relation detection: 876.03 ms/ques (similarity: 701.66 ms/ques), EDG generation: 56.33 ms/ques [INFO] Entity Linking P: 0.742462311557789, R: 0.7047738693467337, micro F1: 0.7164991624790619, macro F1: 0.7231273555276382 [INFO] Relation Linking P: 0.5946398659966499, R: 0.5577889447236181, micro F1: 0.5666666666666662, macro F1: 0.5756252191188501 [INFO] Type Linking P: 0.5326633165829145, R: 0.5326633165829145, micro F1: 0.5326633165829145, macro F1: 0.5326633165829145 @@ -5584,8 +5584,8 @@ Question 398: Who all have developed softwares for Unix Like OSes? [INFO] Golden Answer: [http://dbpedia.org/resource/Wietse_Venema, http://dbpedia.org/resource/Oracle_Corporation, http://dbpedia.org/resource/Sleepycat_Software, http://dbpedia.org/resource/Apple_Inc., http://dbpedia.org/resource/Debian, http://dbpedia.org/resource/The_Tor_Project,_Inc, http://dbpedia.org/resource/Suckless.org, http://dbpedia.org/resource/KDE, http://dbpedia.org/resource/Udi_Manber, http://dbpedia.org/resource/EURid, http://www.march-hare.com/, http://dbpedia.org/resource/GNU_Project, http://dbpedia.org/resource/Dug_Song, http://dbpedia.org/resource/The_GNOME_Project, http://dbpedia.org/resource/Vern_Paxson, http://dbpedia.org/resource/Daniel_J._Bernstein, http://dbpedia.org/resource/Russian_Forth_Interest_Group, http://dbpedia.org/resource/Jamie_Zawinski, https://github.com/haraka/Haraka/graphs/contributors, http://dbpedia.org/resource/Red_Hat, http://dbpedia.org/resource/Evan_Phoenix, http://dbpedia.org/resource/Lanedo, http://dbpedia.org/resource/Imendio, http://dbpedia.org/resource/Julian_Assange, http://dbpedia.org/resource/Rob_Landley, http://dbpedia.org/resource/Hans_Peter_Anvin, http://dbpedia.org/resource/CERN, http://dbpedia.org/resource/World_Wide_Web_Consortium, http://git.exim.org/exim.git/blob/HEAD:/src/ACKNOWLEDGMENTS, http://dbpedia.org/resource/Marko_Macek, http://dbpedia.org/resource/John_Hughes_(computer_scientist), http://dbpedia.org/resource/Michael_Seifert_(programmer,_CEO), http://dbpedia.org/resource/Richard_Stallman, http://dbpedia.org/resource/Stan_Kugell, http://dbpedia.org/resource/University_of_Washington, http://dbpedia.org/resource/Thomas_Esser, http://sourceforge.net/project/memberlist.php?group_id=39518, http://dbpedia.org/resource/Timo_Sirainen, http://dbpedia.org/resource/Miklos_Szeredi, http://dbpedia.org/resource/Olin_Shivers, http://dbpedia.org/resource/GNU, http://dbpedia.org/resource/Artifex_Software,_Inc., http://dbpedia.org/resource/Tomasz_Pawlak, http://dbpedia.org/resource/Électricité_de_France, http://dbpedia.org/resource/Aki_Helin, http://dbpedia.org/resource/Xamarin, http://dbpedia.org/resource/The_Open_Group, http://www.ribbonsoft.com, http://dbpedia.org/resource/BasysKom, http://kolabsys.com/, http://www.kolab.org/, http://dbpedia.org/resource/Kolab_Now, http://okular.kde.org/team.php, http://dbpedia.org/resource/GNOME, https://github.com/Razor-qt/razor-qt/contributors, http://dbpedia.org/resource/Xfce, http://dbpedia.org/resource/Internet_Systems_Consortium, http://www.jikesrvm.org/ProjectOrganization/, http://bazaar.launchpad.net/~kirkland/byobu/trunk/view/head:/AUTHORS, https://github.com/znc/znc/graphs/contributors, http://dbpedia.org/resource/Normation, http://dbpedia.org/resource/Serena_Software, http://dbpedia.org/resource/Andrew_S._Tanenbaum, http://dbpedia.org/resource/Ceriel_Jacobs, http://dbpedia.org/resource/Bill_Joy, http://dbpedia.org/resource/Open_Virtualization_Alliance, http://dbpedia.org/resource/Christophe_de_Dinechin, http://dbpedia.org/resource/Martin_Karplus, http://dbpedia.org/resource/Accelrys, http://dbpedia.org/resource/Puppet_Labs, http://dbpedia.org/resource/Consona_Corporation, http://dbpedia.org/resource/Freedesktop.org, http://dbpedia.org/resource/Swarm_Development_Group, http://dbpedia.org/resource/Open_Cascade, http://dbpedia.org/resource/Intel, http://dbpedia.org/resource/Canonical_(company), http://dbpedia.org/resource/Robert_Landley, http://dbpedia.org/resource/Joseph_Wright_(programmer), http://dbpedia.org/resource/Till_Tantau, http://dbpedia.org/resource/Vedran_Miletić, http://dbpedia.org/resource/MKS_Inc., http://dbpedia.org/resource/Alan_Cox, http://savannah.nongnu.org/project/memberlist.php?detailed=1&group=cvs, http://dbpedia.org/resource/Hugo_Leisink, http://dbpedia.org/resource/TheKompany, http://dbpedia.org/resource/Linux_Mint, http://dbpedia.org/resource/Bert_Hubert, http://dbpedia.org/resource/Solar_Designer, http://www.alternc.com/L-equipe-d-AlternC, http://www.purplepixie.org, http://dbpedia.org/resource/Theodore_Ts'o, http://dbpedia.org/resource/Tuxera, http://dbpedia.org/resource/CZ.NIC, http://dbpedia.org/resource/NLnet_Labs, http://dbpedia.org/resource/Freifunk, http://dbpedia.org/resource/Aerospike_(company), http://dbpedia.org/resource/Elwood_Charles_Downey, http://onlinegroups.net/, http://bioinformatics.org/ghemical/ghemical/authors.html, http://dbpedia.org/resource/Igalia, http://sbml.org/About#The_SBML_Team] [INFO] Predicted Answer: [http://dbpedia.org/resource/Joe_Ossanna, http://dbpedia.org/resource/Brian_Kernighan, http://dbpedia.org/resource/Ken_Thompson, http://dbpedia.org/resource/Douglas_McIlroy, http://dbpedia.org/resource/Dennis_Ritchie, Ken Thompson, Dennis Ritchie, Brian Kernighan, Douglas McIlroy, and Joe Ossanna at Bell Labs@en] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 399, P: 0.518, R: 0.572, macro F1: 0.525, macro F1*: 0.544 -[INFO] Current time: 765 ms, average: 2856.62 ms/ques, entity detection: 446.82 ms/ques (linking tool: 356.71 ms/ques), relation detection: 931.29 ms/ques (similarity: 718.16 ms/ques), EDG generation: 56.19 ms/ques +[INFO] Cumulative metrics, sample: 399, P: 0.518, R: 0.572, macro F1: 0.544 +[INFO] Current question consumed time: 730 ms, average: 2633.61 ms/ques, entity detection: 407.95 ms/ques (linking tool: 327.48 ms/ques), relation detection: 874.59 ms/ques (similarity: 700.27 ms/ques), EDG generation: 56.22 ms/ques [INFO] Entity Linking P: 0.7406015037593985, R: 0.7030075187969925, micro F1: 0.714703425229741, macro F1: 0.7213150062656642 [INFO] Relation Linking P: 0.595655806182122, R: 0.5576441102756893, micro F1: 0.5669172932330822, macro F1: 0.5760235431025953 [INFO] Type Linking P: 0.531328320802005, R: 0.531328320802005, micro F1: 0.531328320802005, macro F1: 0.531328320802005 @@ -5598,8 +5598,8 @@ Question 399: What are some party leaders of the parties which have had a seat i [INFO] Golden Answer: [http://dbpedia.org/resource/Stefan_Körner, http://dbpedia.org/resource/Ali_Khavari, http://dbpedia.org/resource/Jan_Timke, http://dbpedia.org/resource/Katja_Kipping, http://dbpedia.org/resource/Bernd_Riexinger, http://dbpedia.org/resource/Walter_Nauhaus, http://dbpedia.org/resource/Lothar_Bisky, http://dbpedia.org/resource/Martin_Sonneborn, http://dbpedia.org/resource/Carlos_A._Gebauer, http://dbpedia.org/resource/Eva_Schoeller] [INFO] Predicted Answer: [http://dbpedia.org/resource/Germany] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 400, P: 0.517, R: 0.571, macro F1: 0.523, macro F1*: 0.542 -[INFO] Current time: 9011 ms, average: 2872.01 ms/ques, entity detection: 448.24 ms/ques (linking tool: 358.12 ms/ques), relation detection: 938.72 ms/ques (similarity: 725.16 ms/ques), EDG generation: 56.15 ms/ques +[INFO] Cumulative metrics, sample: 400, P: 0.517, R: 0.571, macro F1: 0.542 +[INFO] Current question consumed time: 8525 ms, average: 2648.34 ms/ques, entity detection: 409.44 ms/ques (linking tool: 328.98 ms/ques), relation detection: 881.38 ms/ques (similarity: 707.29 ms/ques), EDG generation: 56.20 ms/ques [INFO] Entity Linking P: 0.74125, R: 0.70375, micro F1: 0.7154166666666666, macro F1: 0.7220134083044983 [INFO] Relation Linking P: 0.5941666666666666, R: 0.55625, micro F1: 0.5654999999999994, macro F1: 0.5745834842448387 [INFO] Type Linking P: 0.53, R: 0.53, micro F1: 0.53, macro F1: 0.53 @@ -5612,8 +5612,8 @@ Question 400: Was James maker in the band Noko? [INFO] Golden Answer: [true] [INFO] Predicted Answer: [false] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 401, P: 0.516, R: 0.569, macro F1: 0.522, macro F1*: 0.541 -[INFO] Current time: 2580 ms, average: 2871.28 ms/ques, entity detection: 448.39 ms/ques (linking tool: 358.10 ms/ques), relation detection: 937.53 ms/ques (similarity: 724.04 ms/ques), EDG generation: 56.03 ms/ques +[INFO] Cumulative metrics, sample: 401, P: 0.516, R: 0.569, macro F1: 0.541 +[INFO] Current question consumed time: 2278 ms, average: 2647.41 ms/ques, entity detection: 409.60 ms/ques (linking tool: 329.00 ms/ques), relation detection: 880.24 ms/ques (similarity: 706.21 ms/ques), EDG generation: 56.09 ms/ques [INFO] Entity Linking P: 0.7394014962593516, R: 0.7019950124688279, micro F1: 0.7136325852036575, macro F1: 0.7202128761142126 [INFO] Relation Linking P: 0.5926849542809642, R: 0.5548628428927681, micro F1: 0.5640897755610967, macro F1: 0.5731506077255251 [INFO] Type Linking P: 0.5311720698254364, R: 0.5311720698254364, micro F1: 0.5311720698254364, macro F1: 0.5311720698254364 @@ -5626,8 +5626,8 @@ Question 401: Was Nick Jonas a member of the Jonas Brothers? [INFO] Golden Answer: [true] [INFO] Predicted Answer: [true] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 402, P: 0.517, R: 0.570, macro F1: 0.523, macro F1*: 0.542 -[INFO] Current time: 2291 ms, average: 2869.84 ms/ques, entity detection: 448.37 ms/ques (linking tool: 357.93 ms/ques), relation detection: 937.97 ms/ques (similarity: 723.85 ms/ques), EDG generation: 55.93 ms/ques +[INFO] Cumulative metrics, sample: 402, P: 0.517, R: 0.570, macro F1: 0.542 +[INFO] Current question consumed time: 1974 ms, average: 2645.74 ms/ques, entity detection: 409.53 ms/ques (linking tool: 328.80 ms/ques), relation detection: 880.57 ms/ques (similarity: 706.07 ms/ques), EDG generation: 55.99 ms/ques [INFO] Entity Linking P: 0.7400497512437811, R: 0.7027363184079602, micro F1: 0.7143449419568821, macro F1: 0.7209105335392005 [INFO] Relation Linking P: 0.5912106135986733, R: 0.5534825870646766, micro F1: 0.5626865671641785, macro F1: 0.5717248599451132 [INFO] Type Linking P: 0.5323383084577115, R: 0.5323383084577115, micro F1: 0.5323383084577115, macro F1: 0.5323383084577115 @@ -5640,8 +5640,8 @@ Question 402: Which tenant of New Sanno Hotel is the military branch of the Gary [INFO] Golden Answer: [http://dbpedia.org/resource/United_States_Navy] [INFO] Predicted Answer: [http://dbpedia.org/resource/United_States_Navy] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 403, P: 0.518, R: 0.571, macro F1: 0.524, macro F1*: 0.543 -[INFO] Current time: 1541 ms, average: 2866.54 ms/ques, entity detection: 448.17 ms/ques (linking tool: 357.66 ms/ques), relation detection: 936.22 ms/ques (similarity: 722.41 ms/ques), EDG generation: 55.89 ms/ques +[INFO] Cumulative metrics, sample: 403, P: 0.518, R: 0.571, macro F1: 0.543 +[INFO] Current question consumed time: 1338 ms, average: 2642.50 ms/ques, entity detection: 409.25 ms/ques (linking tool: 328.50 ms/ques), relation detection: 878.89 ms/ques (similarity: 704.67 ms/ques), EDG generation: 55.94 ms/ques [INFO] Entity Linking P: 0.7406947890818859, R: 0.7034739454094293, micro F1: 0.7150537634408601, macro F1: 0.7216047171983321 [INFO] Relation Linking P: 0.5922249793217534, R: 0.5545905707196029, micro F1: 0.5637717121588084, macro F1: 0.5727902612841481 [INFO] Type Linking P: 0.533498759305211, R: 0.533498759305211, micro F1: 0.533498759305211, macro F1: 0.533498759305211 @@ -5654,8 +5654,8 @@ Question 403: List the director of The Adventures of mimi ? [INFO] Golden Answer: [http://dbpedia.org/resource/Randy_Jackson, http://dbpedia.org/resource/Sanaa_Hamri] [INFO] Predicted Answer: [http://dbpedia.org/resource/Craig_Handley, http://dbpedia.org/resource/George_Whaley_(director)] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 404, P: 0.517, R: 0.570, macro F1: 0.523, macro F1*: 0.542 -[INFO] Current time: 1396 ms, average: 2862.90 ms/ques, entity detection: 447.68 ms/ques (linking tool: 357.17 ms/ques), relation detection: 934.93 ms/ques (similarity: 721.24 ms/ques), EDG generation: 55.78 ms/ques +[INFO] Cumulative metrics, sample: 404, P: 0.517, R: 0.570, macro F1: 0.542 +[INFO] Current question consumed time: 1282 ms, average: 2639.13 ms/ques, entity detection: 408.86 ms/ques (linking tool: 328.09 ms/ques), relation detection: 877.71 ms/ques (similarity: 703.56 ms/ques), EDG generation: 55.84 ms/ques [INFO] Entity Linking P: 0.7388613861386139, R: 0.7017326732673267, micro F1: 0.7132838283828382, macro F1: 0.7198185669082372 [INFO] Relation Linking P: 0.5932343234323432, R: 0.5556930693069307, micro F1: 0.5648514851485144, macro F1: 0.5738503652878751 [INFO] Type Linking P: 0.5346534653465347, R: 0.5346534653465347, micro F1: 0.5346534653465347, macro F1: 0.5346534653465347 @@ -5668,8 +5668,8 @@ Question 404: List sports managers who managed Middlesbrough F.C. and England na [INFO] Golden Answer: [http://dbpedia.org/resource/Gareth_Southgate] [INFO] Predicted Answer: [http://dbpedia.org/resource/Bryan_Robson, http://dbpedia.org/resource/Steve_McClaren, http://dbpedia.org/resource/Herbert_Bamlett, http://dbpedia.org/resource/Jack_Robson, http://dbpedia.org/resource/James_Howie, http://dbpedia.org/resource/Stan_Anderson, http://dbpedia.org/resource/Steve_Agnew, http://dbpedia.org/resource/Thomas_H._McIntosh, http://dbpedia.org/resource/Walter_Rowley, http://dbpedia.org/resource/Wilf_Gillow, http://dbpedia.org/resource/Willie_Maddren, http://dbpedia.org/resource/Lennie_Lawrence, http://dbpedia.org/resource/John_Neal_(footballer,_born_1932), http://dbpedia.org/resource/Bruce_Rioch, http://dbpedia.org/resource/David_Jack, http://dbpedia.org/resource/Tony_Mowbray, http://dbpedia.org/resource/Malcolm_Allison, http://dbpedia.org/resource/Leo_Percovich, http://dbpedia.org/resource/Jack_Charlton, http://dbpedia.org/resource/Colin_Cooper, http://dbpedia.org/resource/Gordon_Strachan, http://dbpedia.org/resource/Peter_McWilliam, http://dbpedia.org/resource/Mark_Venus, http://dbpedia.org/resource/Alex_Smith_(footballer,_born_1944), http://dbpedia.org/resource/Andy_Aitken_(footballer,_born_1877), http://dbpedia.org/resource/Aitor_Karanka, http://dbpedia.org/resource/Gareth_Southgate, http://dbpedia.org/resource/Raich_Carter, http://dbpedia.org/resource/Colin_Todd, http://dbpedia.org/resource/Bobby_Murdoch, http://dbpedia.org/resource/Mark_Proctor_(footballer), http://dbpedia.org/resource/Danny_Bergara] [INFO] P: 0.03125, R: 1.0, F: 0.06060606060606061 -[INFO] Cumulative metrics, sample: 405, P: 0.516, R: 0.571, macro F1: 0.522, macro F1*: 0.542 -[INFO] Current time: 2922 ms, average: 2863.04 ms/ques, entity detection: 448.83 ms/ques (linking tool: 358.05 ms/ques), relation detection: 934.55 ms/ques (similarity: 720.37 ms/ques), EDG generation: 55.89 ms/ques +[INFO] Cumulative metrics, sample: 405, P: 0.516, R: 0.571, macro F1: 0.542 +[INFO] Current question consumed time: 2657 ms, average: 2639.18 ms/ques, entity detection: 409.99 ms/ques (linking tool: 328.99 ms/ques), relation detection: 877.16 ms/ques (similarity: 702.72 ms/ques), EDG generation: 56.08 ms/ques [INFO] Entity Linking P: 0.7395061728395061, R: 0.7012345679012346, micro F1: 0.7131687242798354, macro F1: 0.7198620499962974 [INFO] Relation Linking P: 0.594238683127572, R: 0.5555555555555556, micro F1: 0.5651028806584357, macro F1: 0.5742464010180546 [INFO] Type Linking P: 0.5333333333333333, R: 0.5333333333333333, micro F1: 0.5333333333333333, macro F1: 0.5333333333333333 @@ -5682,8 +5682,8 @@ Question 405: Name all the soccer players who played in a under 18 club for La L [INFO] Golden Answer: [http://dbpedia.org/resource/Juan_Ramón_López_Caro, http://dbpedia.org/resource/Mariano_García_Remón, http://dbpedia.org/resource/Ricardo_(footballer,_born_1971), http://dbpedia.org/resource/Enrique_Collar, http://dbpedia.org/resource/Albert_Crusat, http://dbpedia.org/resource/Albert_Jorquera, http://dbpedia.org/resource/Albert_Luque, http://dbpedia.org/resource/Antonio_Adán, http://dbpedia.org/resource/Daniel_Vázquez_Evuy, http://dbpedia.org/resource/David_de_Gea, http://dbpedia.org/resource/Franck_Songo'o, http://dbpedia.org/resource/Javier_Portillo, http://dbpedia.org/resource/Manuel_Sanchís_Hontiyuelo, http://dbpedia.org/resource/Roberto_Soldado, http://dbpedia.org/resource/Pepe_Mel, http://dbpedia.org/resource/Coke_(footballer), http://dbpedia.org/resource/Iván_Pérez_Muñoz, http://dbpedia.org/resource/Jordi_Gómez, http://dbpedia.org/resource/Rubén_Rochina, http://dbpedia.org/resource/Alfonso_Pérez, http://dbpedia.org/resource/Fernando_Llorente_Mañas, http://dbpedia.org/resource/Joyce_Moreno_(footballer), http://dbpedia.org/resource/Chedric_Seedorf, http://dbpedia.org/resource/Ferran_Olivella, http://dbpedia.org/resource/José_Isidoro, http://dbpedia.org/resource/Raúl_Bravo, http://dbpedia.org/resource/Ángel_Pérez_García, http://dbpedia.org/resource/Chano_(footballer,_born_1965), http://dbpedia.org/resource/Fábio_Miguel_Rico_Lopes, http://dbpedia.org/resource/Dinu_Moldovan, http://dbpedia.org/resource/Gaspar_Gálvez_Burgos, http://dbpedia.org/resource/Dani_Nieto, http://dbpedia.org/resource/Mamadou_Sylla_Diallo, http://dbpedia.org/resource/Antoñito, http://dbpedia.org/resource/David_Juncà, http://dbpedia.org/resource/Emilio_José_Viqueira, http://dbpedia.org/resource/Keko_(footballer,_born_1991), http://dbpedia.org/resource/Lluis_Codina, http://dbpedia.org/resource/David_Rodríguez_Sánchez, http://dbpedia.org/resource/Luis_Rubiales, http://dbpedia.org/resource/José_Manuel_Martínez_Toral, http://dbpedia.org/resource/Manuel_Gavilán_(Spanish_footballer), http://dbpedia.org/resource/Ángel_Pedraza, http://dbpedia.org/resource/Jesús_Olmo, http://dbpedia.org/resource/Javier_Dorado, http://dbpedia.org/resource/David_Belenguer, http://dbpedia.org/resource/Diego_Orejuela, http://dbpedia.org/resource/Paco_Clos, http://dbpedia.org/resource/Juan_José_Maqueda, http://dbpedia.org/resource/David_Casablanca, http://dbpedia.org/resource/José_Vicente_Sánchez, http://dbpedia.org/resource/Borja_Pérez, http://dbpedia.org/resource/Daniel_Toribio, http://dbpedia.org/resource/Fernando_Román_Álvarez, http://dbpedia.org/resource/Fernando_Sales, http://dbpedia.org/resource/Javi_Barral, http://dbpedia.org/resource/Javi_Poves, http://dbpedia.org/resource/Jorge_Molino, http://dbpedia.org/resource/Jorge_Ortiz_Mendoza, http://dbpedia.org/resource/Marc_Nierga, http://dbpedia.org/resource/Sergio_Rodríguez_Hurtado, http://dbpedia.org/resource/Albert_Dalmau, http://dbpedia.org/resource/Aleix_Coch, http://dbpedia.org/resource/Borja_Sánchez_Gil, http://dbpedia.org/resource/Carlos_Caballero_Pérez, http://dbpedia.org/resource/Carlos_Expósito, http://dbpedia.org/resource/Christian_Alfonso, http://dbpedia.org/resource/Diego_Segura, http://dbpedia.org/resource/Felipe_Sanchón, http://dbpedia.org/resource/Fernando_Maestro, http://dbpedia.org/resource/Francisco_Parreño, http://dbpedia.org/resource/Jorge_Merino, http://dbpedia.org/resource/José_Palau_Mira, http://dbpedia.org/resource/Pablo_Villa, http://dbpedia.org/resource/Óscar_López_Martínez, http://dbpedia.org/resource/Alberto_Escassi, http://dbpedia.org/resource/Alberto_Marcos_Rey, http://dbpedia.org/resource/Sito_(footballer), http://dbpedia.org/resource/Diego_Reyes_Muñoz, http://dbpedia.org/resource/Javi_Guerrero, http://dbpedia.org/resource/Javier_Calleja, http://dbpedia.org/resource/Juan_Eduardo_Lescano, http://dbpedia.org/resource/Pablo_Niño, http://dbpedia.org/resource/Raúl_Valbuena, http://dbpedia.org/resource/Sergio_Rodríguez_García, http://dbpedia.org/resource/Toni_Moral, http://dbpedia.org/resource/Mario_Rosas, http://dbpedia.org/resource/Juan_Martínez_Martínez, http://dbpedia.org/resource/Ernesto_Gómez_Gómez, http://dbpedia.org/resource/David_Sánchez_Parrilla, http://dbpedia.org/resource/Felipe_Vaqueriza, http://dbpedia.org/resource/Francisco_Llorente_Gento, http://dbpedia.org/resource/Jesús_Enrique_Velasco, http://dbpedia.org/resource/Roberto_Rojas_(Spanish_footballer), http://dbpedia.org/resource/Enrique_Castaño_Cervera, http://dbpedia.org/resource/Fernando_Guerrero_(footballer), http://dbpedia.org/resource/Fran_Sol, http://dbpedia.org/resource/Javi_Soria, http://dbpedia.org/resource/José_Luis_Cabrera_Cava, http://dbpedia.org/resource/José_Manuel_Roca_Cases, http://dbpedia.org/resource/Lionnel_Franck_Djimgou, http://dbpedia.org/resource/Antonio_Sánchez_Cabeza, http://dbpedia.org/resource/Javier_Oliete, http://dbpedia.org/resource/Pablo_Vargas, http://dbpedia.org/resource/Víctor_Daniel_Bravo, http://dbpedia.org/resource/José_Zamora, http://dbpedia.org/resource/Walter_Fernández, http://dbpedia.org/resource/Emilio_Gutiérrez_González, http://dbpedia.org/resource/Albert_Puigdollers, http://dbpedia.org/resource/Antonio_Hidalgo_(footballer,_born_1979), http://dbpedia.org/resource/Iván_Peñaranda, http://dbpedia.org/resource/Óscar_Sielva, http://dbpedia.org/resource/Pol_Calvet, http://dbpedia.org/resource/Chumbi_(footballer), http://dbpedia.org/resource/Jordi_Masó_(footballer), http://dbpedia.org/resource/José_Moragón, http://dbpedia.org/resource/Álvaro_Arroyo, http://dbpedia.org/resource/Alberto_Liñán, http://dbpedia.org/resource/José_Luis_Morales_Martín, http://dbpedia.org/resource/Mario_Gimeno, http://dbpedia.org/resource/Jonathan_dos_Santos, http://dbpedia.org/resource/Óscar_García_Junyent, http://dbpedia.org/resource/Adrián_López_Rodríguez, http://dbpedia.org/resource/Efraín_Juárez, http://dbpedia.org/resource/Mauro_Icardi, http://dbpedia.org/resource/Roberto_Jiménez_Gago, http://dbpedia.org/resource/Sergi_Roberto, http://dbpedia.org/resource/Aridane_Santana, http://dbpedia.org/resource/Carles_Planas, http://dbpedia.org/resource/Fran_Mérida, http://dbpedia.org/resource/Jordi_Alba, http://dbpedia.org/resource/Míchel_(footballer), http://dbpedia.org/resource/Pedro_Obiang, http://dbpedia.org/resource/Diego_Llorente, http://dbpedia.org/resource/Kiko_Casilla, http://dbpedia.org/resource/Lucas_Vázquez, http://dbpedia.org/resource/Mario_Hermoso, http://dbpedia.org/resource/Omar_Mascarell, http://dbpedia.org/resource/Rafael_Martín_Vázquez, http://dbpedia.org/resource/Raúl_de_Tomás, http://dbpedia.org/resource/Rubén_Yáñez, http://dbpedia.org/resource/Vicente_del_Bosque, http://dbpedia.org/resource/Álvaro_Medrán, http://dbpedia.org/resource/Quique_Fornos, http://dbpedia.org/resource/Rafinha_(footballer,_born_1993), http://dbpedia.org/resource/Ramón_Calderé, http://dbpedia.org/resource/Jordan_Larsson, http://dbpedia.org/resource/Álvaro_Domínguez_Soto, http://dbpedia.org/resource/Julio_Llorente, http://dbpedia.org/resource/Felip_Ortiz, http://dbpedia.org/resource/Gabri_García, http://dbpedia.org/resource/Jordi_Ferrón, http://dbpedia.org/resource/Samuel_Okunowo, http://dbpedia.org/resource/Toni_Velamazán, http://dbpedia.org/resource/Adri_Embarba, http://dbpedia.org/resource/Alexandre_Moreno_Lopera, http://dbpedia.org/resource/Carlos_Vigaray, http://dbpedia.org/resource/Dani_Iglesias, http://dbpedia.org/resource/Emiliano_Buendía, http://dbpedia.org/resource/Eric_Bertrand_Bailly, http://dbpedia.org/resource/Fernando_Pacheco_Flores, http://dbpedia.org/resource/Gregorio_Benito, http://dbpedia.org/resource/Javier_Casquero, http://dbpedia.org/resource/Javier_Castañeda, http://dbpedia.org/resource/José_María_Salmerón, http://dbpedia.org/resource/Lass_Bangoura, http://dbpedia.org/resource/Luis_Cembranos, http://dbpedia.org/resource/Manu_Trigueros, http://dbpedia.org/resource/Martín_Domínguez, http://dbpedia.org/resource/Miguel_Ángel_Sánchez_Muñoz, http://dbpedia.org/resource/Nacho_(footballer,_born_1989), http://dbpedia.org/resource/Nino_(footballer), http://dbpedia.org/resource/Pablo_Sarabia, http://dbpedia.org/resource/Quique_Álvarez, http://dbpedia.org/resource/Raúl_García_Carnero, http://dbpedia.org/resource/Roberto_Peragón, http://dbpedia.org/resource/Roberto_Trashorras, http://dbpedia.org/resource/Sergio_Mantecón, http://dbpedia.org/resource/Sergio_Marcos_González, http://dbpedia.org/resource/Sergio_Mora_Sánchez, http://dbpedia.org/resource/Sergio_Pelegrín, http://dbpedia.org/resource/Víctor_Rodríguez_(Spanish_footballer), http://dbpedia.org/resource/Víctor_Ruiz_(Spanish_footballer), http://dbpedia.org/resource/Ángel_Férez, http://dbpedia.org/resource/Ángel_Martín_González, http://dbpedia.org/resource/Javi_Hernández, http://dbpedia.org/resource/Avtandil_Ebralidze, http://dbpedia.org/resource/Enrique_Ramos_González, http://dbpedia.org/resource/David_Aganzo, http://dbpedia.org/resource/Ramiro_Figueiras_Amarelle, http://dbpedia.org/resource/Bacari, http://dbpedia.org/resource/José_Luis_Deus, http://dbpedia.org/resource/Sandro_(Spanish_footballer), http://dbpedia.org/resource/David_Bermudo, http://dbpedia.org/resource/Filipe_Cândido, http://dbpedia.org/resource/Diego_Mainz, http://dbpedia.org/resource/Édgar_Méndez, http://dbpedia.org/resource/Aarón_Bueno, http://dbpedia.org/resource/José_Moré_Bonet, http://dbpedia.org/resource/José_Ramón_González_Pérez, http://dbpedia.org/resource/Diego_Díaz_Garrido, http://dbpedia.org/resource/José_Luis_Santamaría, http://dbpedia.org/resource/Juan_Antonio_Chesa, http://dbpedia.org/resource/Pablo_Maqueda, http://dbpedia.org/resource/Ramón_Masó, http://dbpedia.org/resource/Víctor_Sánchez_(footballer,_born_1987), http://dbpedia.org/resource/Jacobo_Ynclán, http://dbpedia.org/resource/Pablo_Navas_Alors, http://dbpedia.org/resource/José_Ángel_Pozo, http://dbpedia.org/resource/Juan_Planelles, http://dbpedia.org/resource/Jaime_Ramos_Hernández, http://dbpedia.org/resource/Raúl_Torres_Claverías, http://dbpedia.org/resource/Fernando_López_Fernández, http://dbpedia.org/resource/Andreu_Guerao, http://dbpedia.org/resource/Rafael_Wellington, http://dbpedia.org/resource/Gerard_Moreno_(footballer), http://dbpedia.org/resource/Rober_Correa, http://dbpedia.org/resource/Víctor_Álvarez_(footballer), http://dbpedia.org/resource/Aleix_Domínguez, http://dbpedia.org/resource/Iago_Beceiro, http://dbpedia.org/resource/Dani_Erencia, http://dbpedia.org/resource/Jaime_Astrain, http://dbpedia.org/resource/Daniel_López_(footballer,_born_1992), http://dbpedia.org/resource/Diego_José_Gómez_Heredia, http://dbpedia.org/resource/Matías_Chiacchio, http://dbpedia.org/resource/Adrián_González_Morales, http://dbpedia.org/resource/Bojan_Krkić, http://dbpedia.org/resource/Carles_Puyol, http://dbpedia.org/resource/Daniel_Jarque, http://dbpedia.org/resource/David_Mateos, http://dbpedia.org/resource/Juanito_(footballer,_born_1954), http://dbpedia.org/resource/Pepe_Reina, http://dbpedia.org/resource/Cesc_Fàbregas, http://dbpedia.org/resource/Cristian_Ceballos, http://dbpedia.org/resource/Fernando_Torres, http://dbpedia.org/resource/Juan_Mata, http://dbpedia.org/resource/Pedro_(footballer,_born_July_1987), http://dbpedia.org/resource/Chendo, http://dbpedia.org/resource/Dani_López_(footballer,_born_1985), http://dbpedia.org/resource/Guti_(footballer), http://dbpedia.org/resource/Raúl_Tamudo, http://dbpedia.org/resource/Gerard_Autet, http://dbpedia.org/resource/Keita_Baldé_Diao, http://dbpedia.org/resource/Mamadou_Tounkara, http://dbpedia.org/resource/Patricio_Gabarrón, http://dbpedia.org/resource/Josep_Maria_Fusté, http://dbpedia.org/resource/Maxi_Rolón, http://dbpedia.org/resource/Jesús_García_Tena, http://dbpedia.org/resource/Alberto_Edjogo-Owono, http://dbpedia.org/resource/Isaías_Sánchez, http://dbpedia.org/resource/Moisés_Hurtado, http://dbpedia.org/resource/Óscar_Plano, http://dbpedia.org/resource/Gai_Assulin, http://dbpedia.org/resource/Guillermo_Amor, http://dbpedia.org/resource/Corona_(footballer), http://dbpedia.org/resource/Enrique_de_Lucas, http://dbpedia.org/resource/Joaquín_(footballer), http://dbpedia.org/resource/Rubén_de_la_Red, http://dbpedia.org/resource/David_Cobeño, http://dbpedia.org/resource/Salvador_Sadurní, http://dbpedia.org/resource/Alberto_Toril, http://dbpedia.org/resource/Alfi_Conteh-Lacalle, http://dbpedia.org/resource/Dani_Ceballos, http://dbpedia.org/resource/Daniel_Martín, http://dbpedia.org/resource/Daniel_Pacheco, http://dbpedia.org/resource/Fabián_Ruiz_Peña, http://dbpedia.org/resource/Fernando_Sanz, http://dbpedia.org/resource/Francesc_Arnau, http://dbpedia.org/resource/Francisco_Bizcocho, http://dbpedia.org/resource/Francisco_Molinero, http://dbpedia.org/resource/Francisco_Varela_(footballer), http://dbpedia.org/resource/Ignacio_Camacho, http://dbpedia.org/resource/Javier_Ontiveros, http://dbpedia.org/resource/Josep_Seguer, http://dbpedia.org/resource/José_Antonio_Caro, http://dbpedia.org/resource/Juan_José_Cañas_Gutiérrez, http://dbpedia.org/resource/Juan_Merino, http://dbpedia.org/resource/Manuel_Torres_Jiménez, http://dbpedia.org/resource/Melli_(Spanish_footballer), http://dbpedia.org/resource/Miguel_Torres_Gómez, http://dbpedia.org/resource/Pedro_Contreras, http://dbpedia.org/resource/Vicente_Valcarce, http://dbpedia.org/resource/Víctor_Torres_Mestre, http://dbpedia.org/resource/Álex_Martínez, http://dbpedia.org/resource/Álvaro_Cejudo, http://dbpedia.org/resource/Álvaro_Vadillo, http://dbpedia.org/resource/Tha'er_Bawab, http://dbpedia.org/resource/Iván_Balliu, http://dbpedia.org/resource/Víctor_Chou, http://dbpedia.org/resource/Juan_Cutillas, http://dbpedia.org/resource/Miguel_Ángel_Luque, http://dbpedia.org/resource/Chupe_(footballer), http://dbpedia.org/resource/José_Antonio_Miranda, http://dbpedia.org/resource/Miguel_Boriba, http://dbpedia.org/resource/Pablo_Ganet, http://dbpedia.org/resource/Pepín_(footballer,_born_1996), http://dbpedia.org/resource/Rubén_Belima, http://dbpedia.org/resource/Óscar_Engonga, http://dbpedia.org/resource/César_Gómez, http://dbpedia.org/resource/Alejandro_Pozuelo, http://dbpedia.org/resource/Francisco_Martos, http://dbpedia.org/resource/Richard_Sánchez, http://dbpedia.org/resource/Alejandro_Zamora, http://dbpedia.org/resource/Eugenio_Leal, http://dbpedia.org/resource/Isidoro_San_José, http://dbpedia.org/resource/Rafael_Páez, http://dbpedia.org/resource/Ramón_Grosso, http://dbpedia.org/resource/Salvador_García_(footballer), http://dbpedia.org/resource/David_Rochela, http://dbpedia.org/resource/José_Manuel_Rojas_Olmedo, http://dbpedia.org/resource/Lucas_Porcar, http://dbpedia.org/resource/Marcos_Tébar, http://dbpedia.org/resource/Raúl_Llorente, http://dbpedia.org/resource/Biel_Ribas, http://dbpedia.org/resource/Antonio_Moreno_(footballer), http://dbpedia.org/resource/Borja_García_Freire, http://dbpedia.org/resource/Carlos_Clerc, http://dbpedia.org/resource/Daniel_Provencio, http://dbpedia.org/resource/Germán_Parreño, http://dbpedia.org/resource/Isaac_Becerra, http://dbpedia.org/resource/Javier_Hernández_García, http://dbpedia.org/resource/Jito_(footballer), http://dbpedia.org/resource/Juan_Carlos_Duque, http://dbpedia.org/resource/Julio_Bañuelos, http://dbpedia.org/resource/Pablo_Maffeo, http://dbpedia.org/resource/Rubén_Alcaraz, http://dbpedia.org/resource/Rubén_Sobrino, http://dbpedia.org/resource/Ángel_Sánchez_Martín, http://dbpedia.org/resource/Federico_Bessone, http://dbpedia.org/resource/George_Saunders_(footballer,_born_1989), http://dbpedia.org/resource/Genís_García_Junyent, http://dbpedia.org/resource/Oliver_Cuadrado_Martín, http://dbpedia.org/resource/Agustín_Pastoriza, http://dbpedia.org/resource/Ignacio_Abeledo, http://dbpedia.org/resource/Juan_José_Narváez, http://dbpedia.org/resource/Juan_Manuel_García_García, http://dbpedia.org/resource/Pedro_López_Galisteo, http://dbpedia.org/resource/Enrique_Corrales, http://dbpedia.org/resource/Kevin_Franck, http://dbpedia.org/resource/Daniel_Rodríguez_Vázquez, http://dbpedia.org/resource/Luis_García_(footballer,_born_1979), http://dbpedia.org/resource/Abdelkader_Oueslati, http://dbpedia.org/resource/Paul_Quaye, http://dbpedia.org/resource/Sergio_Torres_Guardeño, http://dbpedia.org/resource/José_Cañas, http://dbpedia.org/resource/Isaac_Gómez, http://dbpedia.org/resource/David_López_Fernández, http://dbpedia.org/resource/Paqui, http://dbpedia.org/resource/Miguel_Bañuz, http://dbpedia.org/resource/Rodrigo_Hernández_Cascante, http://dbpedia.org/resource/Kiko_Torres, http://dbpedia.org/resource/Jaume_Delgado, http://dbpedia.org/resource/Miquel_Martínez, http://dbpedia.org/resource/Gil_Muntadas, http://dbpedia.org/resource/Pere_Martínez, http://dbpedia.org/resource/Xavi_Moro, http://dbpedia.org/resource/Daniel_Fragoso, http://dbpedia.org/resource/Sebastián_Alabanda, http://dbpedia.org/resource/Rafael_García_García, http://dbpedia.org/resource/Rodrigo_Suárez_Peña, http://dbpedia.org/resource/Albert_Manteca, http://dbpedia.org/resource/David_Vázquez_Bardera, http://dbpedia.org/resource/David_Añón, http://dbpedia.org/resource/Juan_Pablo_Ruiz_Pérez, http://dbpedia.org/resource/Fábio_Marques, http://dbpedia.org/resource/Roberto_Sánchez_Piñas, http://dbpedia.org/resource/José_Jesús_Lanza, http://dbpedia.org/resource/Víctor_Aldana, http://dbpedia.org/resource/Bienvenido_Marañón_Morejón, http://dbpedia.org/resource/Pepelu_Vidal, http://dbpedia.org/resource/Sebastián_Herrera, http://dbpedia.org/resource/Al-Lal_Mohamed_Amar, http://dbpedia.org/resource/Chus_Hevia, http://dbpedia.org/resource/André_Onana, http://dbpedia.org/resource/Antonio_Sanabria, http://dbpedia.org/resource/Cristian_Tello, http://dbpedia.org/resource/Francisco_Rufete, http://dbpedia.org/resource/Héctor_Bellerín, http://dbpedia.org/resource/Iago_Falque, http://dbpedia.org/resource/Isaac_Cuenca, http://dbpedia.org/resource/Jean_Marie_Dongou, http://dbpedia.org/resource/Mikel_Arteta, http://dbpedia.org/resource/Pedro_Mosquera, http://dbpedia.org/resource/Roger_García_Junyent, http://dbpedia.org/resource/Samuel_Eto'o, http://dbpedia.org/resource/Sergio_Sánchez_Ortega, http://dbpedia.org/resource/Thiago_Alcântara, http://dbpedia.org/resource/Tito_Vilanova, http://dbpedia.org/resource/Jesé, http://dbpedia.org/resource/Agustín_Rodríguez_Santiago, http://dbpedia.org/resource/Andrés_Iniesta, http://dbpedia.org/resource/David_Rivas, http://dbpedia.org/resource/Denis_Cheryshev, http://dbpedia.org/resource/Francisco_José_Carrasco, http://dbpedia.org/resource/Iker_Casillas, http://dbpedia.org/resource/Joel_Santana, http://dbpedia.org/resource/José_Rodríguez_Martínez, http://dbpedia.org/resource/Lionel_Messi, http://dbpedia.org/resource/Santiago_Cañizares, http://dbpedia.org/resource/Carles_Rexach, http://dbpedia.org/resource/Javier_Camuñas, http://dbpedia.org/resource/Jeffrén_Suárez, http://dbpedia.org/resource/Luis_del_Sol, http://dbpedia.org/resource/Marc_Crosas, http://dbpedia.org/resource/Marcelo_Amado_Djaló_Taritolay, http://dbpedia.org/resource/Miguel_Ángel_Nieto, http://dbpedia.org/resource/Nacho_(footballer,_born_1990), http://dbpedia.org/resource/Pol_García, http://dbpedia.org/resource/Rubén_Miño, http://dbpedia.org/resource/Víctor_Valdés, http://dbpedia.org/resource/Álvaro_Morata, http://dbpedia.org/resource/Aleix_Vidal, http://dbpedia.org/resource/Joan_Campins, http://dbpedia.org/resource/Jordi_Cruyff, http://dbpedia.org/resource/Jordi_Masip, http://dbpedia.org/resource/Jordi_Roura, http://dbpedia.org/resource/José_Luis_Caminero, http://dbpedia.org/resource/Martín_Montoya, http://dbpedia.org/resource/Munir_El_Haddadi, http://dbpedia.org/resource/Rafael_Gordillo, http://dbpedia.org/resource/Sandro_Ramírez, http://dbpedia.org/resource/Óliver_Torres, http://dbpedia.org/resource/Borja_Gómez_Pérez, http://dbpedia.org/resource/Carlos_Peña_(footballer), http://dbpedia.org/resource/Carlos_Rodríguez_(footballer), http://dbpedia.org/resource/Cristian_Fernández_Conchuela, http://dbpedia.org/resource/Cristóbal_Parralo, http://dbpedia.org/resource/David_Fernández_Cortázar, http://dbpedia.org/resource/Enrique_Mateos, http://dbpedia.org/resource/Francisco_Antúnez, http://dbpedia.org/resource/Francisco_Javier_López_Castro, http://dbpedia.org/resource/Hipólito_Rincón, http://dbpedia.org/resource/Ian_Mackay_(footballer), http://dbpedia.org/resource/Joaquín_Caparrós, http://dbpedia.org/resource/Julio_Alberto_Moreno, http://dbpedia.org/resource/Juvenal_Edjogo-Owono, http://dbpedia.org/resource/Manu_Molina, http://dbpedia.org/resource/Manu_del_Moral, http://dbpedia.org/resource/Oriol_Romeu, http://dbpedia.org/resource/Roberto_Ríos, http://dbpedia.org/resource/Román_Golobart, http://dbpedia.org/resource/Sergi_Barjuán, http://dbpedia.org/resource/Toché_(footballer), http://dbpedia.org/resource/Alberto_Bueno, http://dbpedia.org/resource/David_Soria, http://dbpedia.org/resource/Alberto_Rivera_Pizarro, http://dbpedia.org/resource/Ernesto_Domínguez, http://dbpedia.org/resource/Joan_Verdú, http://dbpedia.org/resource/Jordi_Gonzalvo, http://dbpedia.org/resource/Jordi_Xumetra, http://dbpedia.org/resource/Ángel_Cuéllar, http://dbpedia.org/resource/Cedric_Omoigui, http://dbpedia.org/resource/Eduard_Campabadal, http://dbpedia.org/resource/Manuel_Arana_Rodríguez, http://dbpedia.org/resource/Pol_Roigé, http://dbpedia.org/resource/Alexandru_Maxim, http://dbpedia.org/resource/Andreas_Laudrup, http://dbpedia.org/resource/Francisco_Javier_Rodríguez_Vílchez, http://dbpedia.org/resource/Francisco_Rodríguez_García, http://dbpedia.org/resource/Jordi_López, http://dbpedia.org/resource/Carlos_García_Quesada_(footballer), http://dbpedia.org/resource/Francisco_Mateo, http://dbpedia.org/resource/Sergio_Cirio, http://dbpedia.org/resource/Fernando_Quesada, http://dbpedia.org/resource/Fernando_Morán_(footballer), http://dbpedia.org/resource/Francisco_José_Maldonado, http://dbpedia.org/resource/Antonio_Calle, http://dbpedia.org/resource/Jaume_Sobregrau, http://dbpedia.org/resource/José_Barril, http://dbpedia.org/resource/Noé_Acosta, http://dbpedia.org/resource/Álvaro_Brachi, http://dbpedia.org/resource/Lluís_Sastre, http://dbpedia.org/resource/Luis_Ruiz_Sayago, http://dbpedia.org/resource/Marc_Martínez_(footballer), http://dbpedia.org/resource/Pedro_Braojos, http://dbpedia.org/resource/Queco_Piña, http://dbpedia.org/resource/Sunny_Omoregie, http://dbpedia.org/resource/Arzu_(footballer), http://dbpedia.org/resource/Julián_Vara, http://dbpedia.org/resource/Andrés_del_Campo_Santos, http://dbpedia.org/resource/Jofre, http://dbpedia.org/resource/Carlos_Aranda, http://dbpedia.org/resource/David_Sánchez_(footballer), http://dbpedia.org/resource/Iván_Calero, http://dbpedia.org/resource/Daniel_García_(footballer,_born_1974), http://dbpedia.org/resource/Nagore_(footballer), http://dbpedia.org/resource/Gerardo_Bruna, http://dbpedia.org/resource/Samuel_Bayón, http://dbpedia.org/resource/Alex_Zahavi, http://dbpedia.org/resource/David_Cubillo, http://dbpedia.org/resource/David_Pirri, http://dbpedia.org/resource/Manu_Sánchez, http://dbpedia.org/resource/Quique_Martín, http://dbpedia.org/resource/Rubén_Pulido, http://dbpedia.org/resource/Vicente_Fernández_Pujante, http://dbpedia.org/resource/Óscar_Arpón, http://dbpedia.org/resource/Óscar_Sánchez_Fuentes, http://dbpedia.org/resource/Juanlu_(footballer,_born_1972), http://dbpedia.org/resource/Juan_Carlos_Moreno_Rodríguez, http://dbpedia.org/resource/Óscar_Álvarez_(footballer), http://dbpedia.org/resource/Akui_N'dekre_Elie_Moises, http://dbpedia.org/resource/Pablo_González_Juárez, http://dbpedia.org/resource/Stefan_Čolović_(Serbian_footballer), http://dbpedia.org/resource/Jesús_Ángel_Solana, http://dbpedia.org/resource/Julián_López_de_Lerma, http://dbpedia.org/resource/Jesjua_Angoy-Cruyff, http://dbpedia.org/resource/Maikel_Hermann, http://dbpedia.org/resource/Antonio_Reguero, http://dbpedia.org/resource/Francisco_Montañés, http://dbpedia.org/resource/David_Sierra_(footballer), http://dbpedia.org/resource/José_Antonio_García_Aparicio, http://dbpedia.org/resource/Iván_Gómez, http://dbpedia.org/resource/Lee_Seung-woo, http://dbpedia.org/resource/Juan_Baena, http://dbpedia.org/resource/Juan_Cruz_Armada, http://dbpedia.org/resource/Jacole_Turner, http://dbpedia.org/resource/Alejandro_Rodríguez_(footballer,_born_1991), http://dbpedia.org/resource/Jorge_Sáez_Carrillo, http://dbpedia.org/resource/José_Miguel_Bermúdez, http://dbpedia.org/resource/Borja_Valero, http://dbpedia.org/resource/Marcos_Alonso_Mendoza, http://dbpedia.org/resource/Albert_Ferrer, http://dbpedia.org/resource/Alberto_Botía, http://dbpedia.org/resource/Dani_Carvajal, http://dbpedia.org/resource/Dídac_Vilà, http://dbpedia.org/resource/Emilio_Butragueño, http://dbpedia.org/resource/Esteban_Granero, http://dbpedia.org/resource/Jesús_Fernández_Collado, http://dbpedia.org/resource/José_Manuel_Jurado, http://dbpedia.org/resource/Koke_(footballer,_born_1992), http://dbpedia.org/resource/Luis_García_(footballer,_born_1978), http://dbpedia.org/resource/Macky_Bagnack, http://dbpedia.org/resource/Manuel_Velázquez, http://dbpedia.org/resource/Marc_Bartra, http://dbpedia.org/resource/Raúl_(footballer), http://dbpedia.org/resource/Rubén_Iván_Martínez, http://dbpedia.org/resource/Santiago_Bernabéu_Yeste, http://dbpedia.org/resource/Sergio_Busquets, http://dbpedia.org/resource/Tomás_Mejías, http://dbpedia.org/resource/Xavi, http://dbpedia.org/resource/Xisco_(footballer,_born_1986), http://dbpedia.org/resource/Álvaro_Arbeloa, http://dbpedia.org/resource/Sergi_López_Segú, http://dbpedia.org/resource/Capi_(footballer,_born_1977), http://dbpedia.org/resource/Dani_Fernández, http://dbpedia.org/resource/Albert_Dorca, http://dbpedia.org/resource/Borja_González, http://dbpedia.org/resource/Francisco_Pavón, http://dbpedia.org/resource/Gabi_(footballer), http://dbpedia.org/resource/Isaac_Carcelén, http://dbpedia.org/resource/Javi_Moreno, http://dbpedia.org/resource/Javier_Manquillo, http://dbpedia.org/resource/José_Aurelio_Gay, http://dbpedia.org/resource/José_María_Movilla, http://dbpedia.org/resource/Juan_Antonio_Señor, http://dbpedia.org/resource/Juan_Carlos_Aguilera, http://dbpedia.org/resource/Juan_Manuel_Villa, http://dbpedia.org/resource/Juanfran_(footballer,_born_1985), http://dbpedia.org/resource/Juanfran_(footballer,_born_1988), http://dbpedia.org/resource/Lluís_Carreras, http://dbpedia.org/resource/Lucas_Hernández, http://dbpedia.org/resource/Léo_Baptistão, http://dbpedia.org/resource/Manu_Herrera, http://dbpedia.org/resource/Manuel_Lanzarote, http://dbpedia.org/resource/Marc_Bertrán, http://dbpedia.org/resource/Marcos_Alonso_Peña, http://dbpedia.org/resource/Marcos_Vales, http://dbpedia.org/resource/Mario_Suárez_(footballer), http://dbpedia.org/resource/Paco_Campos, http://dbpedia.org/resource/Rafael_García_Cortés, http://dbpedia.org/resource/Roberto_Solozábal, http://dbpedia.org/resource/Rubén_González_Rocha, http://dbpedia.org/resource/Samuel_Sáiz, http://dbpedia.org/resource/Saúl_Ñíguez, http://dbpedia.org/resource/Sekou_Keita_Souza, http://dbpedia.org/resource/Thomas_Partey, http://dbpedia.org/resource/Tomás_Reñones, http://dbpedia.org/resource/Carles_Busquets, http://dbpedia.org/resource/Antonio_Rodríguez_Dovale, http://dbpedia.org/resource/Marc_Valiente, http://dbpedia.org/resource/David_Babunski, http://dbpedia.org/resource/Juanmi_Callejón, http://dbpedia.org/resource/Pitu, http://dbpedia.org/resource/David_Fernández_Miramontes, http://dbpedia.org/resource/Jordi_Amat, http://dbpedia.org/resource/Sergi_Canós, http://dbpedia.org/resource/Ángel_Martínez, http://dbpedia.org/resource/Cedrick_Mabwati, http://dbpedia.org/resource/Javier_Espinosa, http://dbpedia.org/resource/Jordi_Quintillà, http://dbpedia.org/resource/Lucas_Ontivero, http://dbpedia.org/resource/Álex_Quillo, http://dbpedia.org/resource/Antonio_López_(footballer,_born_1981), http://dbpedia.org/resource/Albert_Serrán, http://dbpedia.org/resource/Braulio_Nóbrega, http://dbpedia.org/resource/Bruno_(footballer), http://dbpedia.org/resource/Daniel_Solsona, http://dbpedia.org/resource/Diego_Rivas, http://dbpedia.org/resource/Javi_Guerra, http://dbpedia.org/resource/Javi_Márquez, http://dbpedia.org/resource/Joan_Tomàs_Campasol, http://dbpedia.org/resource/Jonathan_Soriano, http://dbpedia.org/resource/Jordi_Lardín, http://dbpedia.org/resource/Juan_Manuel_López_(footballer), http://dbpedia.org/resource/Miguel_Palanca, http://dbpedia.org/resource/Miquel_Robusté, http://dbpedia.org/resource/Pedro_Mario_Álvarez, http://dbpedia.org/resource/Raúl_Baena, http://dbpedia.org/resource/Rubén_Pérez_del_Mármol, http://dbpedia.org/resource/Víctor_Vázquez, http://dbpedia.org/resource/Álvaro_Vázquez, http://dbpedia.org/resource/Rafa_Mir, http://dbpedia.org/resource/Fabrice_Ondoa, http://dbpedia.org/resource/Thievy_Bifouma, http://dbpedia.org/resource/Mariano_Díaz_Mejía, http://dbpedia.org/resource/Rony_Beard, http://dbpedia.org/resource/Tonet, http://dbpedia.org/resource/Ernesto_Galán, http://dbpedia.org/resource/José_Pedrosa_Galán, http://dbpedia.org/resource/Enric_Vallès, http://dbpedia.org/resource/Goran_Marić_(footballer), http://dbpedia.org/resource/Pau_Morer, http://dbpedia.org/resource/Miguel_Mora_Morales, http://dbpedia.org/resource/Sito_Riera, http://dbpedia.org/resource/Alejandro_López_de_Groot, http://dbpedia.org/resource/Aníbal_Zurdo, http://dbpedia.org/resource/Cristian_Lobato, http://dbpedia.org/resource/Iván_Aguilar, http://dbpedia.org/resource/Sergio_Tejera, http://dbpedia.org/resource/Stephane_Emaná, http://dbpedia.org/resource/David_Vázquez_González, http://dbpedia.org/resource/Tote_(footballer), http://dbpedia.org/resource/Manuel_Clares, http://dbpedia.org/resource/Narcís_Martí_Filosia, http://dbpedia.org/resource/Pedro_Martín, http://dbpedia.org/resource/Sígfrid_Gràcia, http://dbpedia.org/resource/Antonio_Casado_Ruiz, http://dbpedia.org/resource/Antonio_Lao, http://dbpedia.org/resource/Ernest_Forgàs, http://dbpedia.org/resource/Eugeni_Valderrama, http://dbpedia.org/resource/Francisco_No_Rodríguez, http://dbpedia.org/resource/Hipólito_Fernández_Serrano, http://dbpedia.org/resource/Jacinto_Elá, http://dbpedia.org/resource/Jordi_Vinyals, http://dbpedia.org/resource/José_Manuel_Alcañiz, http://dbpedia.org/resource/Juan_Antonio_Osado, http://dbpedia.org/resource/Juanjo_Ciércoles, http://dbpedia.org/resource/Lluís_Pujol, http://dbpedia.org/resource/Marc_Fernández_(footballer), http://dbpedia.org/resource/Marc_de_Val, http://dbpedia.org/resource/Pedro_Antonio_Núñez, http://dbpedia.org/resource/Pere_Valentí_Mora, http://dbpedia.org/resource/Xavier_Pelegrí, http://dbpedia.org/resource/Yeray_Sabariego, http://dbpedia.org/resource/Álex_García_(footballer,_born_1970), http://dbpedia.org/resource/Édgar_Badía, http://dbpedia.org/resource/Roberto_Merino, http://dbpedia.org/resource/Iván_Carril, http://dbpedia.org/resource/Abel_Molinero, http://dbpedia.org/resource/Antonio_Campillo, http://dbpedia.org/resource/Carlos_Pita_(footballer), http://dbpedia.org/resource/David_Gutiérrez_de_Coz, http://dbpedia.org/resource/Álvaro_Lemos, http://dbpedia.org/resource/Nakor_Bueno, http://dbpedia.org/resource/David_García_(footballer,_born_1981), http://dbpedia.org/resource/Raúl_Vates, http://dbpedia.org/resource/David_Cordón, http://dbpedia.org/resource/Javi_Ruiz, http://dbpedia.org/resource/Dani_Tortolero, http://dbpedia.org/resource/Geri_Malaj, http://dbpedia.org/resource/Jorge_Prado_Rus, http://dbpedia.org/resource/Jorge_Troiteiro, http://dbpedia.org/resource/Juan_Pedro_Pina, http://dbpedia.org/resource/Simón_Colina, http://dbpedia.org/resource/Carlos_de_la_Vega, http://dbpedia.org/resource/Juan_Aguilera_Núñez, http://dbpedia.org/resource/Julen_Arellano, http://dbpedia.org/resource/Gerardo_Berodia, http://dbpedia.org/resource/Angeliño_(footballer), http://dbpedia.org/resource/Adrián_Cañas, http://dbpedia.org/resource/Alejandro_Bernal_Carreras, http://dbpedia.org/resource/Manuel_Micó_Yébana, http://dbpedia.org/resource/Luís_Gustavo_Ledes, http://dbpedia.org/resource/Pedro_Astray, http://dbpedia.org/resource/Victor_Blasco, http://dbpedia.org/resource/Diego_Capel, http://dbpedia.org/resource/Domingo_Cisma, http://dbpedia.org/resource/Gerard_Piqué, http://dbpedia.org/resource/Marc_Torrejón, http://dbpedia.org/resource/Haruna_Babangida, http://dbpedia.org/resource/Nayim, http://dbpedia.org/resource/Gerard_López, http://dbpedia.org/resource/Luis_Milla, http://dbpedia.org/resource/Álvaro_Negredo, http://dbpedia.org/resource/Adama_Traoré_(footballer,_born_1996), http://dbpedia.org/resource/Adrián_(footballer), http://dbpedia.org/resource/Iván_Amaya, http://dbpedia.org/resource/Adrià_Carmona, http://dbpedia.org/resource/Alberto_García_(Spanish_footballer), http://dbpedia.org/resource/Dani_Mallo, http://dbpedia.org/resource/Diego_Benito_Rey, http://dbpedia.org/resource/Fernando_Navarro, http://dbpedia.org/resource/Iván_Cuéllar, http://dbpedia.org/resource/Jason_Remeseiro, http://dbpedia.org/resource/Jonathan_Rodríguez_Menéndez, http://dbpedia.org/resource/Jorge_Pulido, http://dbpedia.org/resource/Miguel_Ángel_Guerrero, http://dbpedia.org/resource/Rubén_Cruz_Gil, http://dbpedia.org/resource/Sergio_Cidoncha, http://dbpedia.org/resource/Víctor_Curto, http://dbpedia.org/resource/Gerard_Deulofeu, http://dbpedia.org/resource/Joel_Robles, http://dbpedia.org/resource/Antonio_Pinilla, http://dbpedia.org/resource/Jordi_Codina, http://dbpedia.org/resource/Álex_Grimaldo, http://dbpedia.org/resource/Max_Llovera, http://dbpedia.org/resource/Oriol_Rosell, http://dbpedia.org/resource/Alexander_Szymanowski, http://dbpedia.org/resource/Sergi_Darder, http://dbpedia.org/resource/Carlos_Julio_Martínez, http://dbpedia.org/resource/José_Luis_Loreto, http://dbpedia.org/resource/Ignacio_Conte, http://dbpedia.org/resource/Quique_Estebaranz, http://dbpedia.org/resource/Raúl_Cámara, http://dbpedia.org/resource/Santiago_Jiménez_Luque, http://dbpedia.org/resource/Víctor_García_Marín, http://dbpedia.org/resource/Dani_Cancela, http://dbpedia.org/resource/Jordi_Tarrés_(footballer), http://dbpedia.org/resource/Rufino_Segovia_del_Burgo, http://dbpedia.org/resource/Justo_Tejada, http://dbpedia.org/resource/Juande, http://dbpedia.org/resource/Martí_Riverola, http://dbpedia.org/resource/Miguel_Ángel_Maza, http://dbpedia.org/resource/Alberto_Manuel_Domínguez_Rivas, http://dbpedia.org/resource/Diego_Seoane_(Spanish_footballer), http://dbpedia.org/resource/Ignasi_Miquel, http://dbpedia.org/resource/Jesús_Berrocal, http://dbpedia.org/resource/Jorge_Casado, http://dbpedia.org/resource/Ramón_Montesinos, http://dbpedia.org/resource/Álvaro_del_Moral, http://dbpedia.org/resource/Julio_Prieto_Martín, http://dbpedia.org/resource/Fernando_Marqués, http://dbpedia.org/resource/Toni_Doblas, http://dbpedia.org/resource/Franco_Fasciana, http://dbpedia.org/resource/Javier_Torres_Gómez, http://dbpedia.org/resource/Rubén_Ramiro_Pastor, http://dbpedia.org/resource/Alberto_Manga, http://dbpedia.org/resource/Jon_Toral, http://dbpedia.org/resource/Israel_(footballer), http://dbpedia.org/resource/Javi_López, http://dbpedia.org/resource/Antonio_Reyes_González, http://dbpedia.org/resource/José_Ramón_Romo, http://dbpedia.org/resource/Moisés_Pereiro, http://dbpedia.org/resource/David_García_(footballer,_born_1980), http://dbpedia.org/resource/Xavier_Ginard_Torres, http://dbpedia.org/resource/Álvaro_Vega, http://dbpedia.org/resource/Antonio_Fernando_Díaz, http://dbpedia.org/resource/Eric_Reverter, http://dbpedia.org/resource/Manuel_Palancar_Belloso, http://dbpedia.org/resource/Giovani_dos_Santos, http://dbpedia.org/resource/Juan_Manuel_Ortiz, http://dbpedia.org/resource/Sergi_Gómez, http://dbpedia.org/resource/Luis_Carrión, http://dbpedia.org/resource/Alberto_Lopo, http://dbpedia.org/resource/Bicho_(footballer), http://dbpedia.org/resource/Diego_Tristán, http://dbpedia.org/resource/Fabricio_Agosto_Ramírez, http://dbpedia.org/resource/Fran_González, http://dbpedia.org/resource/Juan_Domínguez_Lamas, http://dbpedia.org/resource/Luis_Fernández_Teijeiro, http://dbpedia.org/resource/Oriol_Riera, http://dbpedia.org/resource/Pablo_Insua, http://dbpedia.org/resource/Álex_Bergantiños, http://dbpedia.org/resource/Ángel_Javier_Arizmendi, http://dbpedia.org/resource/Álex_Arias, http://dbpedia.org/resource/Alberto_Sansimena_Chamorro, http://dbpedia.org/resource/Rubén_Duarte, http://dbpedia.org/resource/Carlos_García_Badías, http://dbpedia.org/resource/Urko_Pardo, http://dbpedia.org/resource/Timur_Akmurzin, http://dbpedia.org/resource/Anselmo_Eyegue, http://dbpedia.org/resource/Joan_Ángel_Román, http://dbpedia.org/resource/Luis_García_(footballer,_born_1972), http://dbpedia.org/resource/David_López_(footballer,_born_1989), http://dbpedia.org/resource/Diego_Alberto_Gama_García, http://dbpedia.org/resource/Fernando_Varela_Ramos, http://dbpedia.org/resource/Maximiliano_Leonel_Rodríguez, http://dbpedia.org/resource/Yann_Songo'o, http://dbpedia.org/resource/Nano_(Spanish_footballer,_born_1980), http://dbpedia.org/resource/Nano_(Spanish_footballer,_born_1982), http://dbpedia.org/resource/Juan_Santisteban, http://dbpedia.org/resource/Hugo_Fraile, http://dbpedia.org/resource/Ilie_Sánchez, http://dbpedia.org/resource/José_Antonio_Delgado_Villar, http://dbpedia.org/resource/Pol_Freixanet, http://dbpedia.org/resource/Roberto_Casabella, http://dbpedia.org/resource/Sergio_León, http://dbpedia.org/resource/Andreu_Fontàs, http://dbpedia.org/resource/Joan_Jordán, http://dbpedia.org/resource/Pau_López, http://dbpedia.org/resource/Pere_Gratacós, http://dbpedia.org/resource/Sergi_Samper, http://dbpedia.org/resource/Sergio_García_(footballer,_born_1983), http://dbpedia.org/resource/Sergio_Juste, http://dbpedia.org/resource/Jesús_Glaría, http://dbpedia.org/resource/César_Ortiz, http://dbpedia.org/resource/Rubén_Ramos_Martínez, http://dbpedia.org/resource/Sergio_Santamaría, http://dbpedia.org/resource/Toni_Calvo, http://dbpedia.org/resource/Ángel_Bernabé, http://dbpedia.org/resource/Javi_Cantero, http://dbpedia.org/resource/Antonio_González_Álvarez, http://dbpedia.org/resource/Martí_Vergés, http://dbpedia.org/resource/Adrián_Sardinero, http://dbpedia.org/resource/David_Català, http://dbpedia.org/resource/Juan_Carricondo, http://dbpedia.org/resource/Germán_Pacheco, http://dbpedia.org/resource/Jorge_Perona, http://dbpedia.org/resource/Raúl_Medina_Zamora, http://dbpedia.org/resource/Óscar_López_(footballer), http://dbpedia.org/resource/Javito, http://dbpedia.org/resource/Armand_Ken_Ella, http://dbpedia.org/resource/Keidi_Bare, http://dbpedia.org/resource/Clemente_Villaverde, http://dbpedia.org/resource/Daniel_Lucas_Segovia, http://dbpedia.org/resource/Rubén_Rivera_Corral, http://dbpedia.org/resource/Dani_Olmo, http://dbpedia.org/resource/José_Manuel_Casado, http://dbpedia.org/resource/Elohor_Godswill, http://dbpedia.org/resource/José_Aurelio_Suárez, http://dbpedia.org/resource/Sergi_Palencia, http://dbpedia.org/resource/Wilfrid_Kaptoum, http://dbpedia.org/resource/Paik_Seung-ho, http://dbpedia.org/resource/Julio_Alberto_Rico_Pardillo, http://dbpedia.org/resource/Elías_Molina, http://dbpedia.org/resource/Lorenzo_Morón, http://dbpedia.org/resource/Jonathan_Carril, http://dbpedia.org/resource/Didac_Rodríguez, http://dbpedia.org/resource/Lauren_Egea, http://dbpedia.org/resource/Álvaro_Portilla_Suárez, http://dbpedia.org/resource/Óscar_Pinchi, http://dbpedia.org/resource/Albert_Celades, http://dbpedia.org/resource/Lucas_Gafarot, http://dbpedia.org/resource/Antonio_Martínez_Felipe, http://dbpedia.org/resource/Francisco_Regalón, http://dbpedia.org/resource/Julio_Álvarez, http://dbpedia.org/resource/Marc_Bernaus, http://dbpedia.org/resource/Marc_Pedraza, http://dbpedia.org/resource/Iván_de_la_Peña, http://dbpedia.org/resource/Álvaro_Baigorri, http://dbpedia.org/resource/Marc_Muniesa, http://dbpedia.org/resource/Saeid_Ezatolahi, http://dbpedia.org/resource/Javier_Chica, http://dbpedia.org/resource/Víctor_Pérez_Alonso, http://dbpedia.org/resource/Juan_Carlos_Ceballos, http://dbpedia.org/resource/Juan_Carlos_Martín_Corral, http://dbpedia.org/resource/Juan_Carlos_Real_Ruiz, http://dbpedia.org/resource/Juan_Carlos_Rojo, http://dbpedia.org/resource/Ignacio_Maganto, http://dbpedia.org/resource/Julio_José_Iglesias_Rouget, http://dbpedia.org/resource/Walter_Iglesias, http://dbpedia.org/resource/Albert_Yagüe, http://dbpedia.org/resource/José_Antonio_Llamas, http://dbpedia.org/resource/Miguel_Martínez_(footballer), http://dbpedia.org/resource/Cristian_Hidalgo, http://dbpedia.org/resource/Antonio_Soldevilla, http://dbpedia.org/resource/Eladio_Silvestre, http://dbpedia.org/resource/Hugo_López_Martínez, http://dbpedia.org/resource/Antoni_Torres, http://dbpedia.org/resource/Marc_Caballé, http://dbpedia.org/resource/Juan_García_Díaz, http://dbpedia.org/resource/José_Casado, http://dbpedia.org/resource/Miguel_Falcón_García-Ramos, http://dbpedia.org/resource/Chuli, http://dbpedia.org/resource/Jorge_Vilda, http://dbpedia.org/resource/Raúl_Caballero, http://dbpedia.org/resource/Ezequiel_Calvente, http://dbpedia.org/resource/Francisco_Tena, http://dbpedia.org/resource/Iván_López_Álvarez, http://dbpedia.org/resource/Ibán_Cuadrado, http://dbpedia.org/resource/Ian_González_Nieto, http://dbpedia.org/resource/Carlos_Calvo_Sobrado, http://dbpedia.org/resource/Joshua_Zapata, http://dbpedia.org/resource/Yuma_(footballer), http://dbpedia.org/resource/Joni_Montiel, http://dbpedia.org/resource/Mario_Gómez_Martín, http://dbpedia.org/resource/Pablo_Clavería, http://dbpedia.org/resource/Damián_Petkoff, http://dbpedia.org/resource/Uxío_Marcos, http://dbpedia.org/resource/Enric_Saborit, http://dbpedia.org/resource/Alejandro_Pérez_Navarro, http://dbpedia.org/resource/Daniel_Mejías, http://dbpedia.org/resource/Jason_Bli, http://dbpedia.org/resource/Carles_Coto, http://dbpedia.org/resource/Manuel_Alfaro, http://dbpedia.org/resource/Jesús_Rodríguez_(footballer), http://dbpedia.org/resource/José_Manuel_Rueda, http://dbpedia.org/resource/Rufo_(footballer), http://dbpedia.org/resource/Antonio_Megías, http://dbpedia.org/resource/Iván_Mateo, http://dbpedia.org/resource/Tomás_Girón, http://dbpedia.org/resource/Robert_Costa_(footballer), http://dbpedia.org/resource/Fernando_Núñez_Lázaro, http://dbpedia.org/resource/Eric_López, http://dbpedia.org/resource/Gabino_Rodríguez_Rodríguez, http://dbpedia.org/resource/Roberto_Suárez_Pier, http://dbpedia.org/resource/Javier_Oliva_González, http://dbpedia.org/resource/Roberto_Batres, http://dbpedia.org/resource/Francisco_Borrego, http://dbpedia.org/resource/Eduardo_Rodríguez_Fernández, http://dbpedia.org/resource/José_Ángel_Jurado_de_la_Torre, http://dbpedia.org/resource/Míchel_Zabaco, http://dbpedia.org/resource/Josué_Currais_Prieto, http://dbpedia.org/resource/Aitor_Cantalapiedra, http://dbpedia.org/resource/Albert_Miravent, http://dbpedia.org/resource/Nico_Zaldana, http://dbpedia.org/resource/Juanjo_Muko_Nsue, http://dbpedia.org/resource/Ismael_Santiago, http://dbpedia.org/resource/Migue, http://dbpedia.org/resource/Kilian_Grant, http://dbpedia.org/resource/Sergio_Buenacasa, http://dbpedia.org/resource/Tarsicio_Aguado, http://dbpedia.org/resource/Julio_García_Fernández, http://dbpedia.org/resource/Gaël_Etock, http://dbpedia.org/resource/David_González_Plata, http://dbpedia.org/resource/Alberto_Noguera, http://dbpedia.org/resource/Viktor_Rodríguez_Álvarez, http://dbpedia.org/resource/Pedro_(footballer,_born_1978), http://dbpedia.org/resource/Mario_Gómez_Atanet, http://dbpedia.org/resource/Arnal_Llibert, http://dbpedia.org/resource/Carlos_Indiano, http://dbpedia.org/resource/Juanito_(footballer,_born_1988), http://dbpedia.org/resource/José_Luis_Gómez_Hurtado, http://dbpedia.org/resource/Brian_Oliván, http://dbpedia.org/resource/Mingo_(footballer), http://dbpedia.org/resource/Xavier_Carmona, http://dbpedia.org/resource/Xavier_Roca, http://dbpedia.org/resource/José_María_Isaac_Béjar, http://dbpedia.org/resource/Iker_Muniain, http://dbpedia.org/resource/José_Antonio_Reyes, http://dbpedia.org/resource/José_María_Maguregui, http://dbpedia.org/resource/Ángel_Guirado, http://dbpedia.org/resource/Asier_Illarramendi, http://dbpedia.org/resource/Javier_Garrido, http://dbpedia.org/resource/Salva_Ballesta, http://dbpedia.org/resource/Sergio_Ramos, http://dbpedia.org/resource/Fernando_Muñoz, http://dbpedia.org/resource/Ricardo_Serna, http://dbpedia.org/resource/Aurélien_Chedjou, http://dbpedia.org/resource/Manuel_Redondo, http://dbpedia.org/resource/Dani_Quintana, http://dbpedia.org/resource/Marcos_García_Barreno, http://dbpedia.org/resource/Jesús_García_Pitarch, http://dbpedia.org/resource/Raúl_Fabiani, http://dbpedia.org/resource/Adrián_Ortolá, http://dbpedia.org/resource/Yordi, http://dbpedia.org/resource/Manuel_Gaspar_Haro, http://dbpedia.org/resource/Perico_(Spanish_footballer), http://dbpedia.org/resource/David_Villabona, http://dbpedia.org/resource/Aitor_López_Rekarte, http://dbpedia.org/resource/Ander_Alaña, http://dbpedia.org/resource/Ander_Capa, http://dbpedia.org/resource/Antonio_Luna_(footballer), http://dbpedia.org/resource/Asier_Etxaburu, http://dbpedia.org/resource/Daniel_García_(footballer,_born_1990), http://dbpedia.org/resource/Imanol_Corral, http://dbpedia.org/resource/Ion_Ansotegi, http://dbpedia.org/resource/Iñigo_Barrenetxea, http://dbpedia.org/resource/Lillo_(footballer), http://dbpedia.org/resource/Momo_(Spanish_footballer), http://dbpedia.org/resource/Jesuli, http://dbpedia.org/resource/Xisco_Nadal, http://dbpedia.org/resource/Jhonatan_Rojas, http://dbpedia.org/resource/Fernando_Barrachina, http://dbpedia.org/resource/Carlos_Bellvís, http://dbpedia.org/resource/Dani_Jiménez, http://dbpedia.org/resource/Fausto_Tienza, http://dbpedia.org/resource/Mikel_Iribas, http://dbpedia.org/resource/Natxo_Insa, http://dbpedia.org/resource/Abel_Moreno_Zorrero, http://dbpedia.org/resource/Ander_Lafuente_Aguado, http://dbpedia.org/resource/Antonio_Vera, http://dbpedia.org/resource/Apoño, http://dbpedia.org/resource/David_Omar_Rodríguez_Barrera, http://dbpedia.org/resource/Francisco_Javier_Castaño, http://dbpedia.org/resource/Francisco_Javier_Solar_González, http://dbpedia.org/resource/Jorge_Luque, http://dbpedia.org/resource/José_Javier_Rubio, http://dbpedia.org/resource/José_Manuel_Pérez_Rodríguez, http://dbpedia.org/resource/José_Ángel_Blanco_Hevia, http://dbpedia.org/resource/Modesto_Acosta_de_la_Rosa, http://dbpedia.org/resource/Moisés_García_Rosa, http://dbpedia.org/resource/Samuel_de_los_Reyes, http://dbpedia.org/resource/Álex_Rubio, http://dbpedia.org/resource/David_Karanka, http://dbpedia.org/resource/Guillermo_Eizaguirre, http://dbpedia.org/resource/Mikel_Kortina, http://dbpedia.org/resource/Mikel_Orbegozo, http://dbpedia.org/resource/Mikel_Roteta, http://dbpedia.org/resource/Gil_Blumstein]... [INFO] Predicted Answer: [http://dbpedia.org/resource/Deportivo_de_La_Coruña, http://dbpedia.org/resource/Getafe_CF, http://dbpedia.org/resource/1996–97_FC_Barcelona_season, http://dbpedia.org/resource/2005–06_FC_Barcelona_season, http://dbpedia.org/resource/2013–14_Real_Madrid_C.F._season, http://dbpedia.org/resource/2014–15_FC_Barcelona_season, http://dbpedia.org/resource/2015–16_Real_Sociedad_season, http://dbpedia.org/resource/1998–99_FC_Barcelona_season, http://dbpedia.org/resource/1999–2000_FC_Barcelona_season, http://dbpedia.org/resource/2003–04_FC_Barcelona_season, http://dbpedia.org/resource/2013–14_Valencia_CF_season, http://dbpedia.org/resource/1995–96_Atlético_Madrid_season, http://dbpedia.org/resource/2015–16_Deportivo_de_La_Coruña_season, http://dbpedia.org/resource/2015–16_Málaga_CF_season, http://dbpedia.org/resource/1993–94_FC_Barcelona_season, http://dbpedia.org/resource/1995–96_FC_Barcelona_season, http://dbpedia.org/resource/2003–04_Deportivo_de_La_Coruña_season, http://dbpedia.org/resource/1931–32_FC_Barcelona_season, http://dbpedia.org/resource/1932–33_FC_Barcelona_season, http://dbpedia.org/resource/2002–03_Real_Sociedad_season, http://dbpedia.org/resource/Atlético_Madrid, http://dbpedia.org/resource/FC_Barcelona, http://dbpedia.org/resource/2014–15_Real_Madrid_C.F._season, http://dbpedia.org/resource/Rayo_Vallecano, http://dbpedia.org/resource/2012–13_FC_Barcelona_season, http://dbpedia.org/resource/2013–14_FC_Barcelona_season, http://dbpedia.org/resource/2003–04_Atlético_Madrid_season, http://dbpedia.org/resource/2004–05_Atlético_Madrid_season, http://dbpedia.org/resource/2015–16_Celta_de_Vigo_season, http://dbpedia.org/resource/1928–29_FC_Barcelona_season, http://dbpedia.org/resource/1930–31_FC_Barcelona_season, http://dbpedia.org/resource/RCD_Espanyol, http://dbpedia.org/resource/Real_Betis, http://dbpedia.org/resource/Real_Madrid_C.F., http://dbpedia.org/resource/Real_Sociedad, http://dbpedia.org/resource/SD_Eibar, http://dbpedia.org/resource/2001–02_Real_Madrid_C.F._season, http://dbpedia.org/resource/2003–04_Real_Madrid_C.F._season, http://dbpedia.org/resource/2004–05_Real_Madrid_C.F._season, http://dbpedia.org/resource/Moghreb_Tétouan, http://dbpedia.org/resource/2015–16_FC_Barcelona_season, http://dbpedia.org/resource/2004–05_FC_Barcelona_season, http://dbpedia.org/resource/1950–51_UE_Lleida_season, http://dbpedia.org/resource/1998–99_Valencia_CF_season, http://dbpedia.org/resource/2014–15_Real_Sociedad_season, http://dbpedia.org/resource/Athletic_Bilbao, http://dbpedia.org/resource/Granada_CF, http://dbpedia.org/resource/Málaga_CF, http://dbpedia.org/resource/Sevilla_FC, http://dbpedia.org/resource/Sporting_de_Gijón, http://dbpedia.org/resource/Valencia_CF, http://dbpedia.org/resource/Villarreal_CF, http://dbpedia.org/resource/UD_Las_Palmas, http://dbpedia.org/resource/2013–14_Athletic_Bilbao_season, http://dbpedia.org/resource/2013–14_Real_Sociedad_season, http://dbpedia.org/resource/2007–08_Racing_de_Santander_season, http://dbpedia.org/resource/1935–36_Hércules_CF_season, http://dbpedia.org/resource/Levante_UD, http://dbpedia.org/resource/2009–10_Real_Madrid_C.F._season, http://dbpedia.org/resource/2010–11_Real_Madrid_C.F._season, http://dbpedia.org/resource/2011–12_Real_Madrid_C.F._season, http://dbpedia.org/resource/2012–13_Real_Madrid_C.F._season, http://dbpedia.org/resource/2015–16_Real_Madrid_C.F._season, http://dbpedia.org/resource/2004–05_Deportivo_de_La_Coruña_season, http://dbpedia.org/resource/2012–13_Málaga_CF_season, http://dbpedia.org/resource/2013–14_Málaga_CF_season, http://dbpedia.org/resource/2014–15_UD_Almería_season, http://dbpedia.org/resource/2013–14_Sevilla_FC_season, http://dbpedia.org/resource/2003–04_Celta_de_Vigo_season, http://dbpedia.org/resource/1996–97_Atlético_Madrid_season, http://dbpedia.org/resource/1997–98_Atlético_Madrid_season, http://dbpedia.org/resource/2003–04_Real_Betis_season, http://dbpedia.org/resource/1996–97_Real_Betis_season, http://dbpedia.org/resource/1998–99_RCD_Mallorca_season, http://dbpedia.org/resource/2002–03_CA_Osasuna_season, http://dbpedia.org/resource/1999–2000_Valencia_CF_season, http://dbpedia.org/resource/2001–02_Valencia_CF_season, http://dbpedia.org/resource/2002–03_Valencia_CF_season, http://dbpedia.org/resource/2003–04_Valencia_CF_season, http://dbpedia.org/resource/2006–07_Valencia_CF_season, http://dbpedia.org/resource/Celta_de_Vigo, http://dbpedia.org/resource/2002–03_FC_Barcelona_season, http://dbpedia.org/resource/1997–98_FC_Barcelona_season, http://dbpedia.org/resource/2014–15_Valencia_CF_season, http://dbpedia.org/resource/2015–16_Valencia_CF_season, http://dbpedia.org/resource/2013–14_UD_Almería_season, http://dbpedia.org/resource/2002–03_Real_Betis_season, http://dbpedia.org/resource/2015–16_Getafe_CF_season, http://dbpedia.org/resource/2015–16_Granada_CF_season, http://dbpedia.org/resource/2015–16_RCD_Espanyol_season, http://dbpedia.org/resource/1999–2000_Deportivo_de_La_Coruña_season, http://dbpedia.org/resource/2000–01_Deportivo_Alavés_season, http://dbpedia.org/resource/2005–06_Villarreal_CF_season, http://dbpedia.org/resource/2000–01_Rayo_Vallecano_season, http://dbpedia.org/resource/2005–06_CA_Osasuna_season, http://dbpedia.org/resource/2007–08_Villarreal_CF_season, http://dbpedia.org/resource/2015–16_Levante_UD_season, http://dbpedia.org/resource/2007–08_Valencia_CF_season, http://dbpedia.org/resource/2015–16_Sevilla_FC_season, http://dbpedia.org/resource/2015–16_UD_Las_Palmas_season, http://dbpedia.org/resource/1929–30_FC_Barcelona_season, http://dbpedia.org/resource/1933–34_FC_Barcelona_season, http://dbpedia.org/resource/1996–97_CD_Tenerife_season, http://dbpedia.org/resource/1998–99_CF_Extremadura_season, http://dbpedia.org/resource/1997–98_Real_Madrid_C.F._season, http://dbpedia.org/resource/1998–99_Real_Madrid_C.F._season, http://dbpedia.org/resource/2008–09_Valencia_CF_season, http://dbpedia.org/resource/2004–05_Real_Betis_season, http://dbpedia.org/resource/2005–06_Real_Betis_season, http://dbpedia.org/resource/2014–15_Sevilla_FC_season, http://dbpedia.org/resource/2015–16_Rayo_Vallecano_season, http://dbpedia.org/resource/2015–16_Real_Betis_season, http://dbpedia.org/resource/1928–29_Real_Madrid_C.F._season, http://dbpedia.org/resource/1929–30_Real_Madrid_C.F._season, http://dbpedia.org/resource/1930–31_Real_Madrid_C.F._season, http://dbpedia.org/resource/2012–13_CA_Osasuna_season, http://dbpedia.org/resource/2006–07_Real_Betis_season, http://dbpedia.org/resource/2007–08_Real_Betis_season, http://dbpedia.org/resource/José_María_García-Aranda, http://dbpedia.org/resource/Carlos_Clos_Gómez, http://dbpedia.org/resource/Luis_Medina_Cantalejo, http://dbpedia.org/resource/Antonio_Mateu_Lahoz, http://dbpedia.org/resource/David_Fernández_Borbalán, http://dbpedia.org/resource/Arturo_Daudén_Ibáñez, http://dbpedia.org/resource/Alfonso_Pérez_Burrull, http://dbpedia.org/resource/Manuel_Díaz_Vega, http://dbpedia.org/resource/Eduardo_Iturralde_González, http://dbpedia.org/resource/Javier_Estrada_Fernández, http://dbpedia.org/resource/Manuel_Mejuto_González, http://dbpedia.org/resource/Juan_Fernández_Marin, http://dbpedia.org/resource/Antonio_Jesús_López_Nieto, http://dbpedia.org/resource/Carlos_Velasco_Carballo, http://dbpedia.org/resource/Jesús_Gil_Manzano, http://dbpedia.org/resource/César_Muñiz_Fernández] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 406, P: 0.514, R: 0.570, macro F1: 0.521, macro F1*: 0.541 -[INFO] Current time: 2079 ms, average: 2861.11 ms/ques, entity detection: 448.74 ms/ques (linking tool: 357.97 ms/ques), relation detection: 933.29 ms/ques (similarity: 719.61 ms/ques), EDG generation: 55.87 ms/ques +[INFO] Cumulative metrics, sample: 406, P: 0.514, R: 0.570, macro F1: 0.541 +[INFO] Current question consumed time: 1870 ms, average: 2637.28 ms/ques, entity detection: 409.92 ms/ques (linking tool: 328.89 ms/ques), relation detection: 876.06 ms/ques (similarity: 702.06 ms/ques), EDG generation: 56.04 ms/ques [INFO] Entity Linking P: 0.7401477832512315, R: 0.7019704433497537, micro F1: 0.7138752052545155, macro F1: 0.7205537770336499 [INFO] Relation Linking P: 0.5952380952380952, R: 0.5554187192118226, micro F1: 0.5653530377668303, macro F1: 0.5746394169512258 [INFO] Type Linking P: 0.5320197044334976, R: 0.5320197044334976, micro F1: 0.5320197044334976, macro F1: 0.5320197044334976 @@ -5696,8 +5696,8 @@ Question 406: Which key perosn of the Elders is also the president of united Wor [INFO] Golden Answer: [http://dbpedia.org/resource/Nelson_Mandela] [INFO] Predicted Answer: [http://dbpedia.org/resource/Louis_Mountbatten,_1st_Earl_Mountbatten_of_Burma, http://dbpedia.org/resource/Nelson_Mandela, http://dbpedia.org/resource/Charles,_Prince_of_Wales, http://dbpedia.org/resource/Queen_Noor_of_Jordan, Queen Noor of Jordan@en] [INFO] P: 0.25, R: 1.0, F: 0.4 -[INFO] Cumulative metrics, sample: 407, P: 0.514, R: 0.571, macro F1: 0.520, macro F1*: 0.541 -[INFO] Current time: 2793 ms, average: 2860.95 ms/ques, entity detection: 448.58 ms/ques (linking tool: 357.48 ms/ques), relation detection: 933.57 ms/ques (similarity: 719.34 ms/ques), EDG generation: 55.97 ms/ques +[INFO] Cumulative metrics, sample: 407, P: 0.514, R: 0.571, macro F1: 0.541 +[INFO] Current question consumed time: 2628 ms, average: 2637.26 ms/ques, entity detection: 409.91 ms/ques (linking tool: 328.48 ms/ques), relation detection: 876.36 ms/ques (similarity: 701.77 ms/ques), EDG generation: 56.17 ms/ques [INFO] Entity Linking P: 0.7407862407862408, R: 0.7014742014742015, micro F1: 0.7137592137592138, macro F1: 0.7205944522809941 [INFO] Relation Linking P: 0.5962325962325962, R: 0.5552825552825553, micro F1: 0.565601965601965, macro F1: 0.575029445584211 [INFO] Type Linking P: 0.5331695331695332, R: 0.5331695331695332, micro F1: 0.5331695331695332, macro F1: 0.5331695331695332 @@ -5710,8 +5710,8 @@ Question 407: Who are the players whose current clubs have Ferran Soriano as cha [INFO] Golden Answer: [http://dbpedia.org/resource/Andrea_Pirlo, http://dbpedia.org/resource/R._J._Allen, http://dbpedia.org/resource/Josh_Saunders, http://dbpedia.org/resource/Patrick_Mullins, http://dbpedia.org/resource/Mehdi_Ballouchy, http://dbpedia.org/resource/Patrick_Vieira, http://dbpedia.org/resource/Diego_Martínez_(Argentinian_footballer), http://dbpedia.org/resource/Federico_Bravo, http://dbpedia.org/resource/Frédéric_Brillant, http://dbpedia.org/resource/Rónald_Matarrita, http://dbpedia.org/resource/Mix_Diskerud, http://dbpedia.org/resource/Frank_Lampard, http://dbpedia.org/resource/David_Villa, http://dbpedia.org/resource/Eirik_Johansen, http://dbpedia.org/resource/Andrew_Jacobson, http://dbpedia.org/resource/Andoni_Iraola, http://dbpedia.org/resource/Andre_Rawls, http://dbpedia.org/resource/Connor_Brandt, http://dbpedia.org/resource/Ethan_White, http://dbpedia.org/resource/Jack_Harrison_(footballer,_born_1996), http://dbpedia.org/resource/Jason_Hernandez, http://dbpedia.org/resource/Jefferson_Mena, http://dbpedia.org/resource/Khiry_Shelton, http://dbpedia.org/resource/Mikey_Lopez, http://dbpedia.org/resource/Thomas_McNamara_(soccer), http://dbpedia.org/resource/Tony_Taylor_(soccer), http://dbpedia.org/resource/Kwadwo_Poku_(footballer,_born_1992)] [INFO] Predicted Answer: [http://dbpedia.org/resource/2015_New_York_City_FC_season, http://dbpedia.org/resource/New_York_City_FC, http://dbpedia.org/resource/2016_New_York_City_FC_season] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 408, P: 0.512, R: 0.569, macro F1: 0.519, macro F1*: 0.539 -[INFO] Current time: 1012 ms, average: 2856.41 ms/ques, entity detection: 447.92 ms/ques (linking tool: 356.84 ms/ques), relation detection: 931.99 ms/ques (similarity: 718.29 ms/ques), EDG generation: 55.88 ms/ques +[INFO] Cumulative metrics, sample: 408, P: 0.512, R: 0.569, macro F1: 0.539 +[INFO] Current question consumed time: 919 ms, average: 2633.05 ms/ques, entity detection: 409.29 ms/ques (linking tool: 327.87 ms/ques), relation detection: 874.88 ms/ques (similarity: 700.77 ms/ques), EDG generation: 56.10 ms/ques [INFO] Entity Linking P: 0.741421568627451, R: 0.7022058823529411, micro F1: 0.7144607843137255, macro F1: 0.7212810845900329 [INFO] Relation Linking P: 0.5972222222222222, R: 0.5551470588235294, micro F1: 0.5658496732026138, macro F1: 0.575416518964906 [INFO] Type Linking P: 0.5318627450980392, R: 0.5318627450980392, micro F1: 0.5318627450980392, macro F1: 0.5318627450980392 @@ -5724,8 +5724,8 @@ Question 408: What are some parties which have people from Mangalore as a member [INFO] Golden Answer: [http://dbpedia.org/resource/Indian_National_Congress, http://dbpedia.org/resource/Bharatiya_Janata_Party] [INFO] Predicted Answer: [http://dbpedia.org/resource/B._Shiva_Rao, http://dbpedia.org/resource/Arjun_Kapikad] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 409, P: 0.511, R: 0.568, macro F1: 0.518, macro F1*: 0.538 -[INFO] Current time: 4038 ms, average: 2859.30 ms/ques, entity detection: 447.79 ms/ques (linking tool: 356.70 ms/ques), relation detection: 933.85 ms/ques (similarity: 721.44 ms/ques), EDG generation: 55.81 ms/ques +[INFO] Cumulative metrics, sample: 409, P: 0.511, R: 0.568, macro F1: 0.538 +[INFO] Current question consumed time: 3861 ms, average: 2636.05 ms/ques, entity detection: 409.14 ms/ques (linking tool: 327.69 ms/ques), relation detection: 876.65 ms/ques (similarity: 703.99 ms/ques), EDG generation: 56.02 ms/ques [INFO] Entity Linking P: 0.7420537897310513, R: 0.7029339853300733, micro F1: 0.715158924205379, macro F1: 0.7219643470310567 [INFO] Relation Linking P: 0.5957620211898941, R: 0.5537897310513448, micro F1: 0.564466177669111, macro F1: 0.5740096326104687 [INFO] Type Linking P: 0.5330073349633252, R: 0.5330073349633252, micro F1: 0.5330073349633252, macro F1: 0.5330073349633252 @@ -5738,8 +5738,8 @@ Question 409: Which employer of Michael M. Sears is also the producer of Delta I [INFO] Golden Answer: [http://dbpedia.org/resource/Boeing] [INFO] Predicted Answer: [http://dbpedia.org/resource/Boeing] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 410, P: 0.512, R: 0.569, macro F1: 0.519, macro F1*: 0.539 -[INFO] Current time: 2327 ms, average: 2858.00 ms/ques, entity detection: 447.87 ms/ques (linking tool: 356.42 ms/ques), relation detection: 933.43 ms/ques (similarity: 720.85 ms/ques), EDG generation: 55.89 ms/ques +[INFO] Cumulative metrics, sample: 410, P: 0.512, R: 0.569, macro F1: 0.539 +[INFO] Current question consumed time: 2231 ms, average: 2635.06 ms/ques, entity detection: 409.30 ms/ques (linking tool: 327.46 ms/ques), relation detection: 876.30 ms/ques (similarity: 703.43 ms/ques), EDG generation: 56.11 ms/ques [INFO] Entity Linking P: 0.7426829268292683, R: 0.7036585365853658, micro F1: 0.7158536585365853, macro F1: 0.7226442643853083 [INFO] Relation Linking P: 0.5967479674796747, R: 0.5548780487804879, micro F1: 0.5655284552845522, macro F1: 0.5750518712387971 [INFO] Type Linking P: 0.5341463414634147, R: 0.5341463414634147, micro F1: 0.5341463414634147, macro F1: 0.5341463414634147 @@ -5752,8 +5752,8 @@ Question 410: Name the largest city in the county where there is a Cy-Fair High [INFO] Golden Answer: [http://dbpedia.org/resource/Houston] [INFO] Predicted Answer: [http://dbpedia.org/resource/Houston] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 411, P: 0.514, R: 0.570, macro F1: 0.520, macro F1*: 0.540 -[INFO] Current time: 2343 ms, average: 2856.75 ms/ques, entity detection: 447.59 ms/ques (linking tool: 356.31 ms/ques), relation detection: 933.37 ms/ques (similarity: 720.76 ms/ques), EDG generation: 55.84 ms/ques +[INFO] Cumulative metrics, sample: 411, P: 0.514, R: 0.570, macro F1: 0.540 +[INFO] Current question consumed time: 2167 ms, average: 2633.92 ms/ques, entity detection: 409.06 ms/ques (linking tool: 327.37 ms/ques), relation detection: 876.24 ms/ques (similarity: 703.44 ms/ques), EDG generation: 56.06 ms/ques [INFO] Entity Linking P: 0.7433090024330901, R: 0.7043795620437956, micro F1: 0.7165450121654501, macro F1: 0.7233208611911919 [INFO] Relation Linking P: 0.5977291159772912, R: 0.5559610705596107, micro F1: 0.566585563665855, macro F1: 0.5760890108997302 [INFO] Type Linking P: 0.5352798053527981, R: 0.5352798053527981, micro F1: 0.5352798053527981, macro F1: 0.5352798053527981 @@ -5766,8 +5766,8 @@ Question 411: How many bacterias are there whose division is Firmicutes? [INFO] Golden Answer: [10] [INFO] Predicted Answer: [204, 2] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 412, P: 0.512, R: 0.569, macro F1: 0.519, macro F1*: 0.539 -[INFO] Current time: 845 ms, average: 2851.87 ms/ques, entity detection: 446.73 ms/ques (linking tool: 355.60 ms/ques), relation detection: 931.37 ms/ques (similarity: 719.16 ms/ques), EDG generation: 55.74 ms/ques +[INFO] Cumulative metrics, sample: 412, P: 0.512, R: 0.569, macro F1: 0.539 +[INFO] Current question consumed time: 820 ms, average: 2629.52 ms/ques, entity detection: 408.29 ms/ques (linking tool: 326.73 ms/ques), relation detection: 874.41 ms/ques (similarity: 701.88 ms/ques), EDG generation: 55.96 ms/ques [INFO] Entity Linking P: 0.7439320388349514, R: 0.7050970873786407, micro F1: 0.7172330097087378, macro F1: 0.7239941617472475 [INFO] Relation Linking P: 0.598705501618123, R: 0.5570388349514563, micro F1: 0.5676375404530738, macro F1: 0.577121089064162 [INFO] Type Linking P: 0.5339805825242718, R: 0.5339805825242718, micro F1: 0.5339805825242718, macro F1: 0.5339805825242718 @@ -5780,8 +5780,8 @@ Question 412: To which country does Peter Blay owe his allegiance? [INFO] Golden Answer: [http://dbpedia.org/resource/Ghana] [INFO] Predicted Answer: [http://dbpedia.org/resource/Ghana] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 413, P: 0.513, R: 0.570, macro F1: 0.520, macro F1*: 0.540 -[INFO] Current time: 927 ms, average: 2847.21 ms/ques, entity detection: 446.08 ms/ques (linking tool: 354.94 ms/ques), relation detection: 929.54 ms/ques (similarity: 717.83 ms/ques), EDG generation: 55.63 ms/ques +[INFO] Cumulative metrics, sample: 413, P: 0.513, R: 0.570, macro F1: 0.540 +[INFO] Current question consumed time: 844 ms, average: 2625.20 ms/ques, entity detection: 407.68 ms/ques (linking tool: 326.13 ms/ques), relation detection: 872.74 ms/ques (similarity: 700.60 ms/ques), EDG generation: 55.82 ms/ques [INFO] Entity Linking P: 0.7445520581113801, R: 0.7058111380145279, micro F1: 0.7179176755447942, macro F1: 0.7246641901150829 [INFO] Relation Linking P: 0.5996771589991928, R: 0.5581113801452785, micro F1: 0.5686844229217105, macro F1: 0.5781481428343561 [INFO] Type Linking P: 0.5326876513317191, R: 0.5326876513317191, micro F1: 0.5326876513317191, macro F1: 0.5326876513317191 @@ -5794,8 +5794,8 @@ Question 413: Is jim harbaugh currently in the Michigan Wolverines? [INFO] Golden Answer: [true] [INFO] Predicted Answer: [true] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 414, P: 0.515, R: 0.571, macro F1: 0.521, macro F1*: 0.541 -[INFO] Current time: 2007 ms, average: 2845.18 ms/ques, entity detection: 445.79 ms/ques (linking tool: 354.60 ms/ques), relation detection: 929.90 ms/ques (similarity: 717.66 ms/ques), EDG generation: 55.53 ms/ques +[INFO] Cumulative metrics, sample: 414, P: 0.515, R: 0.571, macro F1: 0.541 +[INFO] Current question consumed time: 1967 ms, average: 2623.61 ms/ques, entity detection: 407.61 ms/ques (linking tool: 325.94 ms/ques), relation detection: 873.12 ms/ques (similarity: 700.44 ms/ques), EDG generation: 55.72 ms/ques [INFO] Entity Linking P: 0.7451690821256038, R: 0.7065217391304348, micro F1: 0.7185990338164251, macro F1: 0.72533097012226 [INFO] Relation Linking P: 0.6006441223832528, R: 0.5591787439613527, micro F1: 0.5697262479871169, macro F1: 0.5791702089485167 [INFO] Type Linking P: 0.533816425120773, R: 0.533816425120773, micro F1: 0.533816425120773, macro F1: 0.533816425120773 @@ -5808,8 +5808,8 @@ Question 414: Name the television show whose subsequent work is Crusade and Comp [INFO] Golden Answer: [http://dbpedia.org/resource/Babylon_5] [INFO] Predicted Answer: [http://dbpedia.org/resource/Jesse_Stone:_Death_in_Paradise, http://dbpedia.org/resource/Jesse_Stone:_Thin_Ice, http://dbpedia.org/resource/Crusade_(TV_series)] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 415, P: 0.513, R: 0.569, macro F1: 0.520, macro F1*: 0.540 -[INFO] Current time: 5392 ms, average: 2851.32 ms/ques, entity detection: 446.26 ms/ques (linking tool: 354.69 ms/ques), relation detection: 932.92 ms/ques (similarity: 720.31 ms/ques), EDG generation: 55.52 ms/ques +[INFO] Cumulative metrics, sample: 415, P: 0.513, R: 0.569, macro F1: 0.540 +[INFO] Current question consumed time: 5207 ms, average: 2629.83 ms/ques, entity detection: 408.03 ms/ques (linking tool: 326.01 ms/ques), relation detection: 876.27 ms/ques (similarity: 703.37 ms/ques), EDG generation: 55.67 ms/ques [INFO] Entity Linking P: 0.7457831325301205, R: 0.7060240963855422, micro F1: 0.7184738955823293, macro F1: 0.725359196120582 [INFO] Relation Linking P: 0.6016064257028112, R: 0.5602409638554217, micro F1: 0.5707630522088347, macro F1: 0.5801873237852898 [INFO] Type Linking P: 0.5325301204819277, R: 0.5325301204819277, micro F1: 0.5325301204819277, macro F1: 0.5325301204819277 @@ -5822,8 +5822,8 @@ Question 415: What are the current music bands of Chester Bennington? [INFO] Golden Answer: [http://dbpedia.org/resource/Linkin_Park, http://dbpedia.org/resource/Dead_by_Sunrise] [INFO] Predicted Answer: [http://dbpedia.org/resource/Linkin_Park, http://dbpedia.org/resource/Dead_by_Sunrise] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 416, P: 0.515, R: 0.570, macro F1: 0.521, macro F1*: 0.541 -[INFO] Current time: 851 ms, average: 2846.51 ms/ques, entity detection: 445.66 ms/ques (linking tool: 354.12 ms/ques), relation detection: 931.22 ms/ques (similarity: 718.93 ms/ques), EDG generation: 55.42 ms/ques +[INFO] Cumulative metrics, sample: 416, P: 0.515, R: 0.570, macro F1: 0.541 +[INFO] Current question consumed time: 778 ms, average: 2625.38 ms/ques, entity detection: 407.48 ms/ques (linking tool: 325.48 ms/ques), relation detection: 874.72 ms/ques (similarity: 702.06 ms/ques), EDG generation: 55.57 ms/ques [INFO] Entity Linking P: 0.7463942307692307, R: 0.7067307692307693, micro F1: 0.7191506410256411, macro F1: 0.7260211872494751 [INFO] Relation Linking P: 0.6025641025641025, R: 0.5612980769230769, micro F1: 0.5717948717948712, macro F1: 0.581199523368198 [INFO] Type Linking P: 0.5336538461538461, R: 0.5336538461538461, micro F1: 0.5336538461538461, macro F1: 0.5336538461538461 @@ -5836,8 +5836,8 @@ Question 416: Who was the prime minister under which the predecessor of Derick H [INFO] Golden Answer: [http://dbpedia.org/resource/Clement_Attlee, http://dbpedia.org/resource/Harold_Macmillan, http://dbpedia.org/resource/Winston_Churchill, Anthony Eden@en, Sir Alec Douglas-Home@en] [INFO] Predicted Answer: [http://dbpedia.org/resource/Harold_Macmillan, http://dbpedia.org/resource/Winston_Churchill, http://dbpedia.org/resource/Anthony_Eden] [INFO] P: 1.0, R: 0.6, F: 0.7499999999999999 -[INFO] Cumulative metrics, sample: 417, P: 0.516, R: 0.570, macro F1: 0.522, macro F1*: 0.542 -[INFO] Current time: 1279 ms, average: 2842.75 ms/ques, entity detection: 445.03 ms/ques (linking tool: 353.55 ms/ques), relation detection: 929.91 ms/ques (similarity: 718.26 ms/ques), EDG generation: 55.35 ms/ques +[INFO] Cumulative metrics, sample: 417, P: 0.516, R: 0.570, macro F1: 0.542 +[INFO] Current question consumed time: 1189 ms, average: 2621.94 ms/ques, entity detection: 406.90 ms/ques (linking tool: 324.95 ms/ques), relation detection: 873.50 ms/ques (similarity: 701.39 ms/ques), EDG generation: 55.51 ms/ques [INFO] Entity Linking P: 0.7470023980815348, R: 0.7074340527577938, micro F1: 0.71982414068745, macro F1: 0.7266799915385087 [INFO] Relation Linking P: 0.601119104716227, R: 0.5599520383693045, micro F1: 0.5704236610711425, macro F1: 0.5798057595231904 [INFO] Type Linking P: 0.5347721822541966, R: 0.5347721822541966, micro F1: 0.5347721822541966, macro F1: 0.5347721822541966 @@ -5850,8 +5850,8 @@ Question 417: Which person designed the cars which has been designed by ASC Crea [INFO] Golden Answer: [http://dbpedia.org/resource/Michigan, http://dbpedia.org/resource/Warren,_Michigan, http://dbpedia.org/resource/ASC_Creative_Services] [INFO] Predicted Answer: [http://dbpedia.org/resource/Chairman] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 418, P: 0.514, R: 0.569, macro F1: 0.520, macro F1*: 0.540 -[INFO] Current time: 1636 ms, average: 2839.86 ms/ques, entity detection: 444.26 ms/ques (linking tool: 352.90 ms/ques), relation detection: 929.11 ms/ques (similarity: 717.93 ms/ques), EDG generation: 55.26 ms/ques +[INFO] Cumulative metrics, sample: 418, P: 0.514, R: 0.569, macro F1: 0.540 +[INFO] Current question consumed time: 1484 ms, average: 2619.21 ms/ques, entity detection: 406.16 ms/ques (linking tool: 324.33 ms/ques), relation detection: 872.68 ms/ques (similarity: 701.00 ms/ques), EDG generation: 55.41 ms/ques [INFO] Entity Linking P: 0.7452153110047847, R: 0.7057416267942583, micro F1: 0.7181020733652314, macro F1: 0.72494152265923 [INFO] Relation Linking P: 0.5996810207336523, R: 0.5586124401913876, micro F1: 0.5690590111642737, macro F1: 0.5784186644047138 [INFO] Type Linking P: 0.5334928229665071, R: 0.5334928229665071, micro F1: 0.5334928229665071, macro F1: 0.5334928229665071 @@ -5864,8 +5864,8 @@ Question 418: What are some orthonychidae birds? [INFO] Golden Answer: [http://dbpedia.org/resource/Orthonyx_hypsilophus, http://dbpedia.org/resource/Papuan_logrunner, http://dbpedia.org/resource/Orthonyx_kaldowinyeri, http://dbpedia.org/resource/Australian_logrunner, http://dbpedia.org/resource/Chowchilla, http://dbpedia.org/resource/Logrunners] [INFO] Predicted Answer: [http://dbpedia.org/resource/Orthonyx_hypsilophus, http://dbpedia.org/resource/Papuan_logrunner, http://dbpedia.org/resource/Orthonyx_kaldowinyeri, http://dbpedia.org/resource/Australian_logrunner, http://dbpedia.org/resource/Chowchilla, http://dbpedia.org/resource/Logrunners, Orthonyx@en] [INFO] P: 0.8571428571428571, R: 1.0, F: 0.923076923076923 -[INFO] Cumulative metrics, sample: 419, P: 0.515, R: 0.570, macro F1: 0.521, macro F1*: 0.541 -[INFO] Current time: 1113 ms, average: 2835.74 ms/ques, entity detection: 443.32 ms/ques (linking tool: 352.15 ms/ques), relation detection: 927.24 ms/ques (similarity: 716.42 ms/ques), EDG generation: 55.15 ms/ques +[INFO] Cumulative metrics, sample: 419, P: 0.515, R: 0.570, macro F1: 0.541 +[INFO] Current question consumed time: 985 ms, average: 2615.31 ms/ques, entity detection: 405.29 ms/ques (linking tool: 323.64 ms/ques), relation detection: 870.90 ms/ques (similarity: 699.55 ms/ques), EDG generation: 55.28 ms/ques [INFO] Entity Linking P: 0.7458233890214797, R: 0.7064439140811456, micro F1: 0.718774860779634, macro F1: 0.7255997474128447 [INFO] Relation Linking P: 0.5982498011137629, R: 0.5572792362768496, micro F1: 0.5677008750994426, macro F1: 0.5770381902653232 [INFO] Type Linking P: 0.5322195704057279, R: 0.5322195704057279, micro F1: 0.5322195704057279, macro F1: 0.5322195704057279 @@ -5878,8 +5878,8 @@ Question 419: Which driver had the pole position in 1994 Spanish Grand Prix ? [INFO] Golden Answer: [http://dbpedia.org/resource/Michael_Schumacher] [INFO] Predicted Answer: [http://dbpedia.org/resource/Michael_Schumacher] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 420, P: 0.516, R: 0.571, macro F1: 0.523, macro F1*: 0.542 -[INFO] Current time: 1290 ms, average: 2832.06 ms/ques, entity detection: 442.70 ms/ques (linking tool: 351.56 ms/ques), relation detection: 926.21 ms/ques (similarity: 715.49 ms/ques), EDG generation: 55.21 ms/ques +[INFO] Cumulative metrics, sample: 420, P: 0.516, R: 0.571, macro F1: 0.542 +[INFO] Current question consumed time: 1281 ms, average: 2612.14 ms/ques, entity detection: 404.73 ms/ques (linking tool: 323.09 ms/ques), relation detection: 869.98 ms/ques (similarity: 698.63 ms/ques), EDG generation: 55.41 ms/ques [INFO] Entity Linking P: 0.7464285714285714, R: 0.7071428571428572, micro F1: 0.7194444444444444, macro F1: 0.7262548262548262 [INFO] Relation Linking P: 0.5992063492063492, R: 0.5583333333333333, micro F1: 0.5687301587301581, macro F1: 0.5780482230602217 [INFO] Type Linking P: 0.5333333333333333, R: 0.5333333333333333, micro F1: 0.5333333333333333, macro F1: 0.5333333333333333 @@ -5892,8 +5892,8 @@ Question 420: Does the US route 281 have a junction at San Antonio? [INFO] Golden Answer: [true] [INFO] Predicted Answer: [true] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 421, P: 0.518, R: 0.572, macro F1: 0.524, macro F1*: 0.543 -[INFO] Current time: 3160 ms, average: 2832.84 ms/ques, entity detection: 443.35 ms/ques (linking tool: 352.16 ms/ques), relation detection: 927.83 ms/ques (similarity: 715.87 ms/ques), EDG generation: 55.14 ms/ques +[INFO] Cumulative metrics, sample: 421, P: 0.518, R: 0.572, macro F1: 0.543 +[INFO] Current question consumed time: 2671 ms, average: 2612.28 ms/ques, entity detection: 405.22 ms/ques (linking tool: 323.55 ms/ques), relation detection: 871.36 ms/ques (similarity: 699.01 ms/ques), EDG generation: 55.33 ms/ques [INFO] Entity Linking P: 0.7470308788598575, R: 0.7066508313539193, micro F1: 0.7193190815518608, macro F1: 0.7262800211137503 [INFO] Relation Linking P: 0.5977830562153602, R: 0.5570071258907363, micro F1: 0.5673792557403002, macro F1: 0.5766751869009337 [INFO] Type Linking P: 0.5344418052256532, R: 0.5344418052256532, micro F1: 0.5344418052256532, macro F1: 0.5344418052256532 @@ -5906,8 +5906,8 @@ Question 421: How many awards have been given to the founder of Grameen Bank? [INFO] Golden Answer: [9] [INFO] Predicted Answer: [9] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 422, P: 0.519, R: 0.573, macro F1: 0.525, macro F1*: 0.545 -[INFO] Current time: 1860 ms, average: 2830.54 ms/ques, entity detection: 442.61 ms/ques (linking tool: 351.53 ms/ques), relation detection: 927.12 ms/ques (similarity: 715.49 ms/ques), EDG generation: 55.25 ms/ques +[INFO] Cumulative metrics, sample: 422, P: 0.519, R: 0.573, macro F1: 0.545 +[INFO] Current question consumed time: 1714 ms, average: 2610.15 ms/ques, entity detection: 404.56 ms/ques (linking tool: 322.97 ms/ques), relation detection: 870.66 ms/ques (similarity: 698.65 ms/ques), EDG generation: 55.42 ms/ques [INFO] Entity Linking P: 0.7476303317535545, R: 0.707345971563981, micro F1: 0.7199842022116905, macro F1: 0.7269304691480001 [INFO] Relation Linking P: 0.5975513428120063, R: 0.556872037914692, micro F1: 0.5672195892575034, macro F1: 0.5764949663803824 [INFO] Type Linking P: 0.533175355450237, R: 0.533175355450237, micro F1: 0.533175355450237, macro F1: 0.533175355450237 @@ -5920,8 +5920,8 @@ Question 422: Was Queen Latifah in the U.N.I.T.Y. group? [INFO] Golden Answer: [true] [INFO] Predicted Answer: [false] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 423, P: 0.518, R: 0.572, macro F1: 0.524, macro F1*: 0.543 -[INFO] Current time: 2847 ms, average: 2830.57 ms/ques, entity detection: 442.43 ms/ques (linking tool: 351.28 ms/ques), relation detection: 925.82 ms/ques (similarity: 714.31 ms/ques), EDG generation: 55.14 ms/ques +[INFO] Cumulative metrics, sample: 423, P: 0.518, R: 0.572, macro F1: 0.543 +[INFO] Current question consumed time: 2396 ms, average: 2609.64 ms/ques, entity detection: 404.34 ms/ques (linking tool: 322.72 ms/ques), relation detection: 869.41 ms/ques (similarity: 697.48 ms/ques), EDG generation: 55.32 ms/ques [INFO] Entity Linking P: 0.7458628841607565, R: 0.7056737588652482, micro F1: 0.7182821118991333, macro F1: 0.7252119574006052 [INFO] Relation Linking P: 0.5961386918833728, R: 0.5555555555555556, micro F1: 0.5658786446020483, macro F1: 0.5751320941194358 [INFO] Type Linking P: 0.5342789598108747, R: 0.5342789598108747, micro F1: 0.5342789598108747, macro F1: 0.5342789598108747 @@ -5934,8 +5934,8 @@ Question 423: Count the number of important works done by the writers of The Sec [INFO] Golden Answer: [9] [INFO] Predicted Answer: [5, 4] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 424, P: 0.516, R: 0.570, macro F1: 0.522, macro F1*: 0.542 -[INFO] Current time: 2102 ms, average: 2828.86 ms/ques, entity detection: 441.86 ms/ques (linking tool: 350.68 ms/ques), relation detection: 924.99 ms/ques (similarity: 713.83 ms/ques), EDG generation: 55.07 ms/ques +[INFO] Cumulative metrics, sample: 424, P: 0.516, R: 0.570, macro F1: 0.542 +[INFO] Current question consumed time: 1765 ms, average: 2607.65 ms/ques, entity detection: 403.75 ms/ques (linking tool: 322.15 ms/ques), relation detection: 868.61 ms/ques (similarity: 697.03 ms/ques), EDG generation: 55.22 ms/ques [INFO] Entity Linking P: 0.7441037735849056, R: 0.7040094339622641, micro F1: 0.7165880503144655, macro F1: 0.7235015518406982 [INFO] Relation Linking P: 0.5947327044025157, R: 0.5542452830188679, micro F1: 0.5645440251572321, macro F1: 0.5737756505012296 [INFO] Type Linking P: 0.535377358490566, R: 0.535377358490566, micro F1: 0.535377358490566, macro F1: 0.535377358490566 @@ -5948,8 +5948,8 @@ Question 424: Who all are known to play the Gibson Guitar Corporation? [INFO] Golden Answer: [http://dbpedia.org/resource/George_Cole_(musician), http://dbpedia.org/resource/Lindsey_Buckingham, http://dbpedia.org/resource/Joanne_Shaw_Taylor, http://dbpedia.org/resource/Chuck_Brown, http://dbpedia.org/resource/Woody_Guthrie, http://dbpedia.org/resource/Chris_Thile, http://dbpedia.org/resource/Kevin_Rudolf, http://dbpedia.org/resource/Brent_Fitz, http://dbpedia.org/resource/Willie_Brown_(musician), http://dbpedia.org/resource/J._D._Simo, http://dbpedia.org/resource/Kian_Egan, http://dbpedia.org/resource/Noel_Gallagher, http://dbpedia.org/resource/Tim_Wildsmith, http://dbpedia.org/resource/Gordie_Johnson, http://dbpedia.org/resource/Adam_Jensen_(musician), http://dbpedia.org/resource/Kris_Delmhorst, http://dbpedia.org/resource/Jamie_Walters, http://dbpedia.org/resource/Tomas_Kalnoky, http://dbpedia.org/resource/Jim_Couza, http://dbpedia.org/resource/George_Karafotis, http://dbpedia.org/resource/Jake_Bugg, http://dbpedia.org/resource/Linsey_Alexander, http://dbpedia.org/resource/Scott_Page, http://dbpedia.org/resource/Roberto_Diana, http://dbpedia.org/resource/Aaron_Weistrop, http://dbpedia.org/resource/Mirah, http://dbpedia.org/resource/Joel_Silbersher, http://dbpedia.org/resource/Ira_Cruz] [INFO] Predicted Answer: [http://dbpedia.org/resource/Neil_Diamond, http://dbpedia.org/resource/George_Cole_(musician), http://dbpedia.org/resource/Lindsey_Buckingham, http://dbpedia.org/resource/Jon_Schaffer, http://dbpedia.org/resource/Gary_Grainger, http://dbpedia.org/resource/John_Campbell_(blues_guitarist), http://dbpedia.org/resource/Chuck_Brown, http://dbpedia.org/resource/George_Shuffler, http://dbpedia.org/resource/Woody_Guthrie, http://dbpedia.org/resource/Chris_Thile, http://dbpedia.org/resource/Kevin_Rudolf, http://dbpedia.org/resource/Brent_Fitz, http://dbpedia.org/resource/Gillian_Welch, http://dbpedia.org/resource/Willie_Brown_(musician), http://dbpedia.org/resource/Ged_Peck, http://dbpedia.org/resource/J._D._Simo, http://dbpedia.org/resource/Felix_Pappalardi, http://dbpedia.org/resource/Kian_Egan, http://dbpedia.org/resource/Noel_Gallagher, http://dbpedia.org/resource/Tim_Wildsmith, http://dbpedia.org/resource/Neko_Case, http://dbpedia.org/resource/Goher_Mumtaz, http://dbpedia.org/resource/Gordie_Johnson, http://dbpedia.org/resource/Adam_Jensen_(musician), http://dbpedia.org/resource/Don_Henley, http://dbpedia.org/resource/Kris_Delmhorst, http://dbpedia.org/resource/Jamie_Walters, http://dbpedia.org/resource/Tomas_Kalnoky, http://dbpedia.org/resource/Sylvia_Juncosa, http://dbpedia.org/resource/Jim_Couza, http://dbpedia.org/resource/Lanie_Lane, http://dbpedia.org/resource/Michael_Anthony_(musician), http://dbpedia.org/resource/Johnny_Smith, http://dbpedia.org/resource/George_Karafotis, http://dbpedia.org/resource/Jake_Bugg, http://dbpedia.org/resource/Diana_Jones_(singer-songwriter), http://dbpedia.org/resource/Linsey_Alexander, http://dbpedia.org/resource/Todd_Sharpville, http://dbpedia.org/resource/Scott_Page, http://dbpedia.org/resource/Jens_Fredrik_Ryland, http://dbpedia.org/resource/Roberto_Diana, http://dbpedia.org/resource/JJ_Appleton, http://dbpedia.org/resource/Aaron_Weistrop, http://dbpedia.org/resource/Captain_Kirk_Douglas, http://dbpedia.org/resource/Mirah, http://dbpedia.org/resource/Cherry_Filter, http://dbpedia.org/resource/Emilio_Cueto, http://dbpedia.org/resource/Joel_Silbersher, http://dbpedia.org/resource/Ira_Cruz, http://dbpedia.org/resource/Jeffrey_Luck_Lucas] [INFO] P: 0.54, R: 0.9642857142857143, F: 0.6923076923076923 -[INFO] Cumulative metrics, sample: 425, P: 0.516, R: 0.571, macro F1: 0.523, macro F1*: 0.542 -[INFO] Current time: 2087 ms, average: 2827.11 ms/ques, entity detection: 442.57 ms/ques (linking tool: 351.42 ms/ques), relation detection: 923.58 ms/ques (similarity: 712.58 ms/ques), EDG generation: 54.97 ms/ques +[INFO] Cumulative metrics, sample: 425, P: 0.516, R: 0.571, macro F1: 0.542 +[INFO] Current question consumed time: 1863 ms, average: 2605.90 ms/ques, entity detection: 404.37 ms/ques (linking tool: 322.81 ms/ques), relation detection: 867.26 ms/ques (similarity: 695.83 ms/ques), EDG generation: 55.13 ms/ques [INFO] Entity Linking P: 0.7447058823529412, R: 0.7047058823529412, micro F1: 0.7172549019607845, macro F1: 0.7241539343009932 [INFO] Relation Linking P: 0.5933333333333333, R: 0.5529411764705883, micro F1: 0.5632156862745092, macro F1: 0.5724255901471091 [INFO] Type Linking P: 0.5364705882352941, R: 0.5364705882352941, micro F1: 0.5364705882352941, macro F1: 0.5364705882352941 @@ -5962,8 +5962,8 @@ Question 425: Which child of Jack Ferguson became a Prime Minister? [INFO] Golden Answer: [http://dbpedia.org/resource/Julia_Gillard, http://dbpedia.org/resource/Kevin_Rudd] [INFO] Predicted Answer: [http://dbpedia.org/resource/Martin_Ferguson, http://dbpedia.org/resource/Laurie_Ferguson] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 426, P: 0.515, R: 0.570, macro F1: 0.521, macro F1*: 0.541 -[INFO] Current time: 2386 ms, average: 2826.08 ms/ques, entity detection: 442.30 ms/ques (linking tool: 351.11 ms/ques), relation detection: 923.78 ms/ques (similarity: 712.22 ms/ques), EDG generation: 54.88 ms/ques +[INFO] Cumulative metrics, sample: 426, P: 0.515, R: 0.570, macro F1: 0.541 +[INFO] Current question consumed time: 2043 ms, average: 2604.58 ms/ques, entity detection: 404.08 ms/ques (linking tool: 322.51 ms/ques), relation detection: 867.35 ms/ques (similarity: 695.51 ms/ques), EDG generation: 55.04 ms/ques [INFO] Entity Linking P: 0.7453051643192489, R: 0.7053990610328639, micro F1: 0.7179186228482004, macro F1: 0.7248032423234119 [INFO] Relation Linking P: 0.5942879499217527, R: 0.5528169014084507, micro F1: 0.5634585289514861, macro F1: 0.5728027784716485 [INFO] Type Linking P: 0.5375586854460094, R: 0.5375586854460094, micro F1: 0.5375586854460094, macro F1: 0.5375586854460094 @@ -5976,8 +5976,8 @@ Question 426: Did Annie Leibovitz do the cover of The Road Ahead? [INFO] Golden Answer: [true] [INFO] Predicted Answer: [true] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 427, P: 0.516, R: 0.571, macro F1: 0.523, macro F1*: 0.542 -[INFO] Current time: 378 ms, average: 2820.34 ms/ques, entity detection: 441.26 ms/ques (linking tool: 350.29 ms/ques), relation detection: 921.62 ms/ques (similarity: 710.56 ms/ques), EDG generation: 54.94 ms/ques +[INFO] Cumulative metrics, sample: 427, P: 0.516, R: 0.571, macro F1: 0.542 +[INFO] Current question consumed time: 312 ms, average: 2599.21 ms/ques, entity detection: 403.13 ms/ques (linking tool: 321.76 ms/ques), relation detection: 865.32 ms/ques (similarity: 693.89 ms/ques), EDG generation: 55.10 ms/ques [INFO] Entity Linking P: 0.7459016393442623, R: 0.7049180327868853, micro F1: 0.7177985948477753, macro F1: 0.7248309715661757 [INFO] Relation Linking P: 0.5952380952380952, R: 0.5538641686182669, micro F1: 0.5644808743169393, macro F1: 0.5738062888198757 [INFO] Type Linking P: 0.5386416861826698, R: 0.5386416861826698, micro F1: 0.5386416861826698, macro F1: 0.5386416861826698 @@ -5990,8 +5990,8 @@ Question 427: What are the things whose mascots are from the class Eutheria? [INFO] Golden Answer: [http://dbpedia.org/resource/North_Side_High_School_(Fort_Worth,_Texas), http://dbpedia.org/resource/Holderness_School, http://dbpedia.org/resource/Chungbuk_National_University, http://dbpedia.org/resource/Sonoma_Academy, http://dbpedia.org/resource/La_Joya_High_School, http://dbpedia.org/resource/Wichita_Falls_High_School, http://dbpedia.org/resource/Buena_Park_High_School, http://dbpedia.org/resource/École_Évangéline, http://dbpedia.org/resource/Centennial_High_School_(Peoria,_Arizona), http://dbpedia.org/resource/Cerro_Coso_Community_College, http://dbpedia.org/resource/Cuyamaca_College, http://dbpedia.org/resource/Arizona_Lutheran_Academy, http://dbpedia.org/resource/College_of_Idaho, http://dbpedia.org/resource/Clear_Horizons_Early_College_High_School, http://dbpedia.org/resource/Chandler–Gilbert_Community_College, http://dbpedia.org/resource/Cheverus_High_School, http://dbpedia.org/resource/Ridgely_Middle_School, http://dbpedia.org/resource/Richmond,_The_American_International_University_in_London, http://dbpedia.org/resource/Deer_High_School, http://dbpedia.org/resource/Sultan_Kudarat_State_University, http://dbpedia.org/resource/13th_Mechanized_Brigade_(Netherlands), http://dbpedia.org/resource/Howard_University, http://dbpedia.org/resource/Buffalo_Independent_School_District, http://dbpedia.org/resource/Gallaudet_University, http://dbpedia.org/resource/Buffalo_High_School_(Buffalo,_Missouri), http://dbpedia.org/resource/Great_Falls_High_School, http://dbpedia.org/resource/Buffalo_Grove_High_School, http://dbpedia.org/resource/Carlisle_High_School_(Arkansas), http://dbpedia.org/resource/Carlisle_School_District_(Arkansas), http://dbpedia.org/resource/Buffalo_High_School_(Minnesota), http://dbpedia.org/resource/President_of_Bucknell_University, http://dbpedia.org/resource/501_Squadron_(Portugal), http://dbpedia.org/resource/Wayne_Community_College, http://dbpedia.org/resource/Shaker_High_School, http://dbpedia.org/resource/43rd_Mechanized_Brigade_(Netherlands), http://dbpedia.org/resource/Buffalo_Gap_High_School, http://dbpedia.org/resource/Benton_Central_Junior-Senior_High_School, http://dbpedia.org/resource/Capital_City_Alternative_School] [INFO] Predicted Answer: [http://dbpedia.org/resource/Baiji, http://dbpedia.org/resource/Dusky_dolphin, http://dbpedia.org/resource/Ground_sloth, http://dbpedia.org/resource/Heaviside's_dolphin, http://dbpedia.org/resource/Shrew, http://dbpedia.org/resource/Spider_monkey, http://dbpedia.org/resource/Indus_river_dolphin, http://dbpedia.org/resource/Palaeomastodon, http://dbpedia.org/resource/Condylarth, http://dbpedia.org/resource/Mylodontidae, http://dbpedia.org/resource/Perognathus, http://dbpedia.org/resource/Ashoroa, http://dbpedia.org/resource/Cornwallius, http://dbpedia.org/resource/Desmostylidae, http://dbpedia.org/resource/Kronokotherium, http://dbpedia.org/resource/Mesonychidae, http://dbpedia.org/resource/Afrodon, http://dbpedia.org/resource/Alloeumyarion, http://dbpedia.org/resource/Lestobradys, http://dbpedia.org/resource/Turiasorex, http://dbpedia.org/resource/Cattle, http://dbpedia.org/resource/Coyote, http://dbpedia.org/resource/Deer, http://dbpedia.org/resource/Elephant_shrew, http://dbpedia.org/resource/Fin_whale, http://dbpedia.org/resource/Humpback_whale, http://dbpedia.org/resource/Manatee, http://dbpedia.org/resource/Pig, http://dbpedia.org/resource/Plesiorycteropus, http://dbpedia.org/resource/Ptolemaiida, http://dbpedia.org/resource/Hopping_mouse, http://dbpedia.org/resource/Sahyadris_forest_rat, http://dbpedia.org/resource/Bubal_hartebeest, http://dbpedia.org/resource/Dinofelis_paleoonca, http://dbpedia.org/resource/Ord's_kangaroo_rat, http://dbpedia.org/resource/List_of_Chinese_terrestrial_ungulates, http://dbpedia.org/resource/Scelidotheriinae, http://dbpedia.org/resource/Tritemnodon, http://dbpedia.org/resource/Buxolestes_minor, http://dbpedia.org/resource/Cohen's_horseshoe_bat, http://dbpedia.org/resource/Amazon_river_dolphin, http://dbpedia.org/resource/Rhinoceros, http://dbpedia.org/resource/Antarctic_minke_whale, http://dbpedia.org/resource/Araguaian_river_dolphin, http://dbpedia.org/resource/Atlantic_spotted_dolphin, http://dbpedia.org/resource/Cephalorhynchus, http://dbpedia.org/resource/Chilean_dolphin, http://dbpedia.org/resource/Commerson's_dolphin, http://dbpedia.org/resource/Common_minke_whale, http://dbpedia.org/resource/Fraser's_dolphin, http://dbpedia.org/resource/Hector's_dolphin, http://dbpedia.org/resource/Inia, http://dbpedia.org/resource/Iniidae, http://dbpedia.org/resource/La_Plata_dolphin, http://dbpedia.org/resource/Pantropical_spotted_dolphin, http://dbpedia.org/resource/Peale's_dolphin, http://dbpedia.org/resource/Pygmy_sperm_whale, http://dbpedia.org/resource/Rough-toothed_dolphin, http://dbpedia.org/resource/Sciuromorpha, http://dbpedia.org/resource/Spinner_dolphin, http://dbpedia.org/resource/Stenella, http://dbpedia.org/resource/Striped_dolphin, http://dbpedia.org/resource/White-beaked_dolphin, http://dbpedia.org/resource/Meridiungulata, http://dbpedia.org/resource/Odd-toed_ungulate, http://dbpedia.org/resource/Propalaeotherium, http://dbpedia.org/resource/Sturdee's_pipistrelle, http://dbpedia.org/resource/Adapisoriculidae, http://dbpedia.org/resource/Zalambdalestes, http://dbpedia.org/resource/Astrapotheriidae, http://dbpedia.org/resource/Mylagaulidae, http://dbpedia.org/resource/Pantolestidae, http://dbpedia.org/resource/Plesiadapidae, http://dbpedia.org/resource/Pyrotheriidae, http://dbpedia.org/resource/Simplicidentata, http://dbpedia.org/resource/Xenungulata, http://dbpedia.org/resource/Antarctodon, http://dbpedia.org/resource/Palaeosinopa, http://dbpedia.org/resource/Albertogaudrya, http://dbpedia.org/resource/Comahuetherium, http://dbpedia.org/resource/Parastrapotherium, http://dbpedia.org/resource/Bison, http://dbpedia.org/resource/Mole_(animal), http://dbpedia.org/resource/Sloth, http://dbpedia.org/resource/Glires, http://dbpedia.org/resource/Mesonychid, http://dbpedia.org/resource/Morro_Bay_kangaroo_rat, http://dbpedia.org/resource/Pyrotheria, http://dbpedia.org/resource/Black_rhinoceros, http://dbpedia.org/resource/Pseudoungulata, http://dbpedia.org/resource/Southern_tree_hyrax, http://dbpedia.org/resource/Chaetodipus, http://dbpedia.org/resource/Giant_kangaroo_rat, http://dbpedia.org/resource/Sagmatias, http://dbpedia.org/resource/Carpolestidae, http://dbpedia.org/resource/Alopecocyon, http://dbpedia.org/resource/Baiyankamys, http://dbpedia.org/resource/Flores_long-nosed_rat, http://dbpedia.org/resource/Leptomys_paulus, http://dbpedia.org/resource/Pithecheirops, http://dbpedia.org/resource/Tonkinomys, http://dbpedia.org/resource/Yamdena_mosaic-tailed_rat, http://dbpedia.org/resource/Ectocion, http://dbpedia.org/resource/Paschatherium, http://dbpedia.org/resource/Ailuravus, http://dbpedia.org/resource/Buxolestes, http://dbpedia.org/resource/Hyotheridium, http://dbpedia.org/resource/Long-nosed_paramelomys, http://dbpedia.org/resource/Hipposideros_rotalis, http://dbpedia.org/resource/Atlantic_white-sided_dolphin, http://dbpedia.org/resource/Bemalambdidae, http://dbpedia.org/resource/Dinocerata, http://dbpedia.org/resource/Minke_whale, http://dbpedia.org/resource/Pacific_white-sided_dolphin, http://dbpedia.org/resource/Pantodonta, http://dbpedia.org/resource/South_Asian_river_dolphin, http://dbpedia.org/resource/Megistotherium, http://dbpedia.org/resource/Tokudaia, http://dbpedia.org/resource/Human_taxonomy, http://dbpedia.org/resource/Scaglia_(genus), http://dbpedia.org/resource/Bryde's_whale, http://dbpedia.org/resource/Cottontail_rabbit, http://dbpedia.org/resource/Even-toed_ungulate, http://dbpedia.org/resource/Omura's_whale, http://dbpedia.org/resource/Tremarctinae, http://dbpedia.org/resource/Ursinae, http://dbpedia.org/resource/Verhoeven's_giant_tree_rat, http://dbpedia.org/resource/Cricetidae, http://dbpedia.org/resource/Myomorpha, http://dbpedia.org/resource/Treeshrew, http://dbpedia.org/resource/Astrapotheria, http://dbpedia.org/resource/Amphechinus, http://dbpedia.org/resource/Dichobunidae, http://dbpedia.org/resource/Erinaceidae, http://dbpedia.org/resource/Proterotheriidae, http://dbpedia.org/resource/Cape_warthog, http://dbpedia.org/resource/Flores_cave_rat, http://dbpedia.org/resource/Texas_kangaroo_rat, http://dbpedia.org/resource/Gomphos, http://dbpedia.org/resource/Afrosoricida, http://dbpedia.org/resource/Apodemus, http://dbpedia.org/resource/Asioryctitheria, http://dbpedia.org/resource/Atlantogenata, http://dbpedia.org/resource/Colombian_weasel, http://dbpedia.org/resource/Epitheria, http://dbpedia.org/resource/Lagomorpha, http://dbpedia.org/resource/Laurasiatheria, http://dbpedia.org/resource/Protungulatum_donnae, http://dbpedia.org/resource/Soricomorpha, http://dbpedia.org/resource/Elasmotherium, http://dbpedia.org/resource/Litopterna, http://dbpedia.org/resource/Notoungulata, http://dbpedia.org/resource/Radinskya, http://dbpedia.org/resource/Montanalestes, http://dbpedia.org/resource/Alcidedorbignya, http://dbpedia.org/resource/Papagomys, http://dbpedia.org/resource/Deccanolestes, http://dbpedia.org/resource/Sahnitherium, http://dbpedia.org/resource/Carodnia, http://dbpedia.org/resource/Hyopsodus, http://dbpedia.org/resource/Mongolestes, http://dbpedia.org/resource/Protungulatum, http://dbpedia.org/resource/Thalassocnus, http://dbpedia.org/resource/Dipodinae, http://dbpedia.org/resource/Spinifex_hopping_mouse, http://dbpedia.org/resource/Zapodinae, http://dbpedia.org/resource/Dusky_hopping_mouse, http://dbpedia.org/resource/Eastern_coyote, http://dbpedia.org/resource/Fawn_hopping_mouse, http://dbpedia.org/resource/Short-tailed_hopping_mouse, http://dbpedia.org/resource/Eobalaenoptera, http://dbpedia.org/resource/Tree_hyrax, http://dbpedia.org/resource/Anomaluromorpha, http://dbpedia.org/resource/Sciurognathi, http://dbpedia.org/resource/Oriental_giant_squirrel, http://dbpedia.org/resource/Simiolus, http://dbpedia.org/resource/Guilestes, http://dbpedia.org/resource/Harpagolestes, http://dbpedia.org/resource/Hukoutherium, http://dbpedia.org/resource/Mongolonyx, http://dbpedia.org/resource/Horolodectes_sunae, http://dbpedia.org/resource/Parapontoporia] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 428, P: 0.515, R: 0.570, macro F1: 0.521, macro F1*: 0.541 -[INFO] Current time: 912 ms, average: 2815.88 ms/ques, entity detection: 440.55 ms/ques (linking tool: 349.77 ms/ques), relation detection: 920.11 ms/ques (similarity: 709.50 ms/ques), EDG generation: 54.86 ms/ques +[INFO] Cumulative metrics, sample: 428, P: 0.515, R: 0.570, macro F1: 0.541 +[INFO] Current question consumed time: 800 ms, average: 2595.00 ms/ques, entity detection: 402.46 ms/ques (linking tool: 321.26 ms/ques), relation detection: 863.91 ms/ques (similarity: 692.86 ms/ques), EDG generation: 55.02 ms/ques [INFO] Entity Linking P: 0.7464953271028038, R: 0.705607476635514, micro F1: 0.7184579439252338, macro F1: 0.7254757483026444 [INFO] Relation Linking P: 0.596183800623053, R: 0.5537383177570093, micro F1: 0.5647196261682237, macro F1: 0.5741776935225068 [INFO] Type Linking P: 0.5397196261682243, R: 0.5397196261682243, micro F1: 0.5397196261682243, macro F1: 0.5397196261682243 @@ -6004,8 +6004,8 @@ Question 428: For whom did Michael Springer palyed for which is also had a forme [INFO] Golden Answer: [http://dbpedia.org/resource/Syracuse_University] [INFO] Predicted Answer: [http://dbpedia.org/resource/Syracuse_University] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 429, P: 0.516, R: 0.571, macro F1: 0.522, macro F1*: 0.542 -[INFO] Current time: 1881 ms, average: 2813.70 ms/ques, entity detection: 440.69 ms/ques (linking tool: 349.63 ms/ques), relation detection: 918.75 ms/ques (similarity: 708.32 ms/ques), EDG generation: 54.85 ms/ques +[INFO] Cumulative metrics, sample: 429, P: 0.516, R: 0.571, macro F1: 0.542 +[INFO] Current question consumed time: 1656 ms, average: 2592.81 ms/ques, entity detection: 402.66 ms/ques (linking tool: 321.25 ms/ques), relation detection: 862.60 ms/ques (similarity: 691.73 ms/ques), EDG generation: 55.03 ms/ques [INFO] Entity Linking P: 0.747086247086247, R: 0.7062937062937062, micro F1: 0.7191142191142192, macro F1: 0.7261175071920862 [INFO] Relation Linking P: 0.597125097125097, R: 0.5547785547785548, micro F1: 0.5657342657342651, macro F1: 0.5751734493724374 [INFO] Type Linking P: 0.5407925407925408, R: 0.5407925407925408, micro F1: 0.5407925407925408, macro F1: 0.5407925407925408 @@ -6018,8 +6018,8 @@ Question 429: Name the common nationality of Georgina Theodora Wood and akwasi O [INFO] Golden Answer: [http://dbpedia.org/resource/Ghana] [INFO] Predicted Answer: [] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 430, P: 0.515, R: 0.569, macro F1: 0.521, macro F1*: 0.541 -[INFO] Current time: 457 ms, average: 2808.22 ms/ques, entity detection: 439.66 ms/ques (linking tool: 348.82 ms/ques), relation detection: 916.80 ms/ques (similarity: 706.80 ms/ques), EDG generation: 54.76 ms/ques +[INFO] Cumulative metrics, sample: 430, P: 0.515, R: 0.569, macro F1: 0.541 +[INFO] Current question consumed time: 429 ms, average: 2587.78 ms/ques, entity detection: 401.72 ms/ques (linking tool: 320.50 ms/ques), relation detection: 860.76 ms/ques (similarity: 690.24 ms/ques), EDG generation: 54.93 ms/ques [INFO] Entity Linking P: 0.7453488372093023, R: 0.7046511627906977, micro F1: 0.7174418604651164, macro F1: 0.7244288618265232 [INFO] Relation Linking P: 0.5957364341085271, R: 0.5534883720930233, micro F1: 0.5644186046511621, macro F1: 0.5738358366994785 [INFO] Type Linking P: 0.541860465116279, R: 0.541860465116279, micro F1: 0.541860465116279, macro F1: 0.541860465116279 @@ -6032,8 +6032,8 @@ Question 430: Name the movie written by Monty Python and distributed by Cinema I [INFO] Golden Answer: [http://dbpedia.org/resource/Monty_Python's_Life_of_Brian] [INFO] Predicted Answer: [http://dbpedia.org/resource/Monty_Python's_Life_of_Brian] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 431, P: 0.516, R: 0.570, macro F1: 0.522, macro F1*: 0.542 -[INFO] Current time: 2839 ms, average: 2808.29 ms/ques, entity detection: 439.60 ms/ques (linking tool: 348.60 ms/ques), relation detection: 916.68 ms/ques (similarity: 707.27 ms/ques), EDG generation: 54.70 ms/ques +[INFO] Cumulative metrics, sample: 431, P: 0.516, R: 0.570, macro F1: 0.542 +[INFO] Current question consumed time: 2468 ms, average: 2587.50 ms/ques, entity detection: 401.59 ms/ques (linking tool: 320.27 ms/ques), relation detection: 860.55 ms/ques (similarity: 690.75 ms/ques), EDG generation: 54.85 ms/ques [INFO] Entity Linking P: 0.7459396751740139, R: 0.7053364269141531, micro F1: 0.7180974477958239, macro F1: 0.7250700599613116 [INFO] Relation Linking P: 0.5966744006187161, R: 0.5545243619489559, micro F1: 0.5654292343387465, macro F1: 0.5748277396622364 [INFO] Type Linking P: 0.5406032482598608, R: 0.5406032482598608, micro F1: 0.5406032482598608, macro F1: 0.5406032482598608 @@ -6046,8 +6046,8 @@ Question 431: Is Pittsburgh Steelers the former team of Marv McFadden? [INFO] Golden Answer: [true] [INFO] Predicted Answer: [true] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 432, P: 0.517, R: 0.571, macro F1: 0.523, macro F1*: 0.543 -[INFO] Current time: 1688 ms, average: 2805.70 ms/ques, entity detection: 440.18 ms/ques (linking tool: 348.93 ms/ques), relation detection: 915.05 ms/ques (similarity: 705.93 ms/ques), EDG generation: 54.61 ms/ques +[INFO] Cumulative metrics, sample: 432, P: 0.517, R: 0.571, macro F1: 0.543 +[INFO] Current question consumed time: 1473 ms, average: 2584.92 ms/ques, entity detection: 402.06 ms/ques (linking tool: 320.56 ms/ques), relation detection: 859.01 ms/ques (similarity: 689.45 ms/ques), EDG generation: 54.79 ms/ques [INFO] Entity Linking P: 0.7465277777777778, R: 0.7060185185185185, micro F1: 0.7187500000000001, macro F1: 0.7257082779991146 [INFO] Relation Linking P: 0.5952932098765431, R: 0.5532407407407407, micro F1: 0.5641203703703698, macro F1: 0.5734971198944997 [INFO] Type Linking P: 0.5416666666666666, R: 0.5416666666666666, micro F1: 0.5416666666666666, macro F1: 0.5416666666666666 @@ -6060,8 +6060,8 @@ Question 432: Which series was written by Eric Jendresen and John Orloff? [INFO] Golden Answer: [http://dbpedia.org/resource/Band_of_Brothers_(miniseries)] [INFO] Predicted Answer: [http://dbpedia.org/resource/Band_of_Brothers_(miniseries)] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 433, P: 0.518, R: 0.572, macro F1: 0.525, macro F1*: 0.544 -[INFO] Current time: 2901 ms, average: 2805.92 ms/ques, entity detection: 439.84 ms/ques (linking tool: 348.50 ms/ques), relation detection: 915.21 ms/ques (similarity: 706.91 ms/ques), EDG generation: 54.56 ms/ques +[INFO] Cumulative metrics, sample: 433, P: 0.518, R: 0.572, macro F1: 0.544 +[INFO] Current question consumed time: 2630 ms, average: 2585.03 ms/ques, entity detection: 401.71 ms/ques (linking tool: 320.14 ms/ques), relation detection: 859.19 ms/ques (similarity: 690.52 ms/ques), EDG generation: 54.73 ms/ques [INFO] Entity Linking P: 0.7471131639722863, R: 0.7066974595842956, micro F1: 0.7193995381062357, macro F1: 0.7263435366974412 [INFO] Relation Linking P: 0.5962278675904541, R: 0.5542725173210161, micro F1: 0.5651270207852188, macro F1: 0.5744851986150913 [INFO] Type Linking P: 0.5404157043879908, R: 0.5404157043879908, micro F1: 0.5404157043879908, macro F1: 0.5404157043879908 @@ -6074,8 +6074,8 @@ Question 433: In which city can I find both the office of Elasticsearch and the [INFO] Golden Answer: [http://dbpedia.org/resource/Amsterdam] [INFO] Predicted Answer: [Elasticsearch commercial solutions@en] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 434, P: 0.517, R: 0.571, macro F1: 0.523, macro F1*: 0.543 -[INFO] Current time: 1377 ms, average: 2802.63 ms/ques, entity detection: 439.29 ms/ques (linking tool: 348.08 ms/ques), relation detection: 913.79 ms/ques (similarity: 706.00 ms/ques), EDG generation: 54.59 ms/ques +[INFO] Cumulative metrics, sample: 434, P: 0.517, R: 0.571, macro F1: 0.543 +[INFO] Current question consumed time: 1180 ms, average: 2581.79 ms/ques, entity detection: 401.19 ms/ques (linking tool: 319.74 ms/ques), relation detection: 857.80 ms/ques (similarity: 689.63 ms/ques), EDG generation: 54.75 ms/ques [INFO] Entity Linking P: 0.7476958525345622, R: 0.706221198156682, micro F1: 0.719278033794163, macro F1: 0.7263669692609932 [INFO] Relation Linking P: 0.5948540706605222, R: 0.5529953917050692, micro F1: 0.5638248847926262, macro F1: 0.5731615000007708 [INFO] Type Linking P: 0.5414746543778802, R: 0.5414746543778802, micro F1: 0.5414746543778802, macro F1: 0.5414746543778802 @@ -6088,8 +6088,8 @@ Question 434: Name the partners of Kelly Brook? [INFO] Golden Answer: [http://dbpedia.org/resource/Thom_Evans, http://dbpedia.org/resource/David_McIntosh_(Gladiators), http://dbpedia.org/resource/Billy_Zane, http://dbpedia.org/resource/Jason_Statham] [INFO] Predicted Answer: [http://dbpedia.org/resource/Thom_Evans, http://dbpedia.org/resource/David_McIntosh_(Gladiators), http://dbpedia.org/resource/Billy_Zane, http://dbpedia.org/resource/Jason_Statham, Thom Evans@en, Billy Zane@en, Jason Statham@en] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 435, P: 0.518, R: 0.572, macro F1: 0.524, macro F1*: 0.544 -[INFO] Current time: 1035 ms, average: 2798.57 ms/ques, entity detection: 438.88 ms/ques (linking tool: 347.70 ms/ques), relation detection: 912.59 ms/ques (similarity: 704.91 ms/ques), EDG generation: 54.48 ms/ques +[INFO] Cumulative metrics, sample: 435, P: 0.518, R: 0.572, macro F1: 0.544 +[INFO] Current question consumed time: 906 ms, average: 2577.94 ms/ques, entity detection: 400.78 ms/ques (linking tool: 319.36 ms/ques), relation detection: 856.65 ms/ques (similarity: 688.57 ms/ques), EDG generation: 54.64 ms/ques [INFO] Entity Linking P: 0.7482758620689656, R: 0.7068965517241379, micro F1: 0.7199233716475097, macro F1: 0.726997875469848 [INFO] Relation Linking P: 0.5957854406130267, R: 0.5540229885057472, micro F1: 0.564827586206896, macro F1: 0.57414578282335 [INFO] Type Linking P: 0.542528735632184, R: 0.542528735632184, micro F1: 0.542528735632184, macro F1: 0.542528735632184 @@ -6102,8 +6102,8 @@ Question 435: What is the area of the broadcast network which has the affiliatio [INFO] Golden Answer: [http://dbpedia.org/resource/Philippines] [INFO] Predicted Answer: [http://dbpedia.org/resource/Philippines] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 436, P: 0.519, R: 0.573, macro F1: 0.526, macro F1*: 0.545 -[INFO] Current time: 1532 ms, average: 2795.66 ms/ques, entity detection: 437.95 ms/ques (linking tool: 346.95 ms/ques), relation detection: 912.14 ms/ques (similarity: 704.69 ms/ques), EDG generation: 54.45 ms/ques +[INFO] Cumulative metrics, sample: 436, P: 0.519, R: 0.573, macro F1: 0.545 +[INFO] Current question consumed time: 1326 ms, average: 2575.07 ms/ques, entity detection: 399.92 ms/ques (linking tool: 318.67 ms/ques), relation detection: 856.15 ms/ques (similarity: 688.39 ms/ques), EDG generation: 54.59 ms/ques [INFO] Entity Linking P: 0.7488532110091743, R: 0.7075688073394495, micro F1: 0.7205657492354742, macro F1: 0.7276258758939537 [INFO] Relation Linking P: 0.5967125382262997, R: 0.555045871559633, micro F1: 0.5658256880733938, macro F1: 0.575125526388707 [INFO] Type Linking P: 0.5412844036697247, R: 0.5412844036697247, micro F1: 0.5412844036697247, macro F1: 0.5412844036697247 @@ -6116,8 +6116,8 @@ Question 436: Who is married to Gerard Piqu? [INFO] Golden Answer: [http://dbpedia.org/resource/Shakira] [INFO] Predicted Answer: [http://dbpedia.org/resource/Shakira] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 437, P: 0.520, R: 0.574, macro F1: 0.527, macro F1*: 0.546 -[INFO] Current time: 894 ms, average: 2791.31 ms/ques, entity detection: 437.38 ms/ques (linking tool: 346.36 ms/ques), relation detection: 910.75 ms/ques (similarity: 703.48 ms/ques), EDG generation: 54.34 ms/ques +[INFO] Cumulative metrics, sample: 437, P: 0.520, R: 0.574, macro F1: 0.546 +[INFO] Current question consumed time: 769 ms, average: 2570.93 ms/ques, entity detection: 399.37 ms/ques (linking tool: 318.11 ms/ques), relation detection: 854.84 ms/ques (similarity: 687.22 ms/ques), EDG generation: 54.49 ms/ques [INFO] Entity Linking P: 0.7494279176201373, R: 0.7082379862700229, micro F1: 0.7212051868802443, macro F1: 0.7282509905916248 [INFO] Relation Linking P: 0.5976353928299007, R: 0.5560640732265446, micro F1: 0.5668192219679627, macro F1: 0.5761007621461126 [INFO] Type Linking P: 0.5423340961098398, R: 0.5423340961098398, micro F1: 0.5423340961098398, macro F1: 0.5423340961098398 @@ -6130,8 +6130,8 @@ Question 437: Which sports are played in schools affiliated with the Harvest Chr [INFO] Golden Answer: [http://dbpedia.org/resource/Badminton, http://dbpedia.org/resource/Basketball, http://dbpedia.org/resource/Chess, http://dbpedia.org/resource/Tennis, http://dbpedia.org/resource/Volleyball] [INFO] Predicted Answer: [] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 438, P: 0.519, R: 0.573, macro F1: 0.525, macro F1*: 0.545 -[INFO] Current time: 4402 ms, average: 2794.99 ms/ques, entity detection: 436.67 ms/ques (linking tool: 345.74 ms/ques), relation detection: 912.88 ms/ques (similarity: 706.19 ms/ques), EDG generation: 54.25 ms/ques +[INFO] Cumulative metrics, sample: 438, P: 0.519, R: 0.573, macro F1: 0.545 +[INFO] Current question consumed time: 3860 ms, average: 2573.88 ms/ques, entity detection: 398.68 ms/ques (linking tool: 317.53 ms/ques), relation detection: 856.70 ms/ques (similarity: 689.98 ms/ques), EDG generation: 54.41 ms/ques [INFO] Entity Linking P: 0.747716894977169, R: 0.70662100456621, micro F1: 0.7195585996955862, macro F1: 0.7265883170971233 [INFO] Relation Linking P: 0.5962709284627092, R: 0.5547945205479452, micro F1: 0.5655251141552505, macro F1: 0.5747854636024 [INFO] Type Linking P: 0.541095890410959, R: 0.541095890410959, micro F1: 0.541095890410959, macro F1: 0.541095890410959 @@ -6144,8 +6144,8 @@ Question 438: Who debuted their career in a team coached by Chip Kelly? [INFO] Golden Answer: [http://dbpedia.org/resource/Roger_Craig_(American_football), http://dbpedia.org/resource/Joe_Perry_(American_football), http://dbpedia.org/resource/Ernie_Hughes, http://dbpedia.org/resource/Doug_DuBose, http://dbpedia.org/resource/Jamie_Winborn, http://dbpedia.org/resource/Quinton_Dial, http://dbpedia.org/resource/Wayne_Trimble, http://dbpedia.org/resource/Joe_Looney_(offensive_lineman), http://dbpedia.org/resource/Parys_Haralson, http://dbpedia.org/resource/Roy_Williams_(defensive_tackle), http://dbpedia.org/resource/Brandon_Moore_(linebacker), http://dbpedia.org/resource/Ahmed_Plummer, http://dbpedia.org/resource/Alex_Boone, http://dbpedia.org/resource/Andy_Lee_(American_football), http://dbpedia.org/resource/Dominique_Zeigler, http://dbpedia.org/resource/Kendall_Hunter, http://dbpedia.org/resource/LaMichael_James, http://dbpedia.org/resource/Ronnie_Heard, http://dbpedia.org/resource/Mike_Magac, http://dbpedia.org/resource/Jaquiski_Tartt, http://dbpedia.org/resource/Eddie_Lewis_(American_football), http://dbpedia.org/resource/Jason_Webster, http://dbpedia.org/resource/Brian_Jennings, http://dbpedia.org/resource/Andrew_Williams_(American_football), http://dbpedia.org/resource/Garrett_Celek, http://dbpedia.org/resource/Ronald_Fields, http://dbpedia.org/resource/Mike_Burns_(American_football), http://dbpedia.org/resource/Jimmy_Williams_(cornerback,_born_1979), http://dbpedia.org/resource/Iheanyi_Uwaezuoke, http://dbpedia.org/resource/Kevin_Mitchell_(linebacker), http://dbpedia.org/resource/Hugh_McElhenny, http://dbpedia.org/resource/Renaldo_Nehemiah, http://dbpedia.org/resource/Howard_Mudd, http://dbpedia.org/resource/J._J._Stokes, http://dbpedia.org/resource/Keith_Henderson, http://dbpedia.org/resource/Charles_Johnson_(defensive_back), http://dbpedia.org/resource/Chris_Patrick, http://dbpedia.org/resource/Jim_Looney, http://dbpedia.org/resource/Julian_Peterson, http://dbpedia.org/resource/David_Baas, http://dbpedia.org/resource/Harvey_Dahl, http://dbpedia.org/resource/Tony_Peterson, http://dbpedia.org/resource/Tramaine_Brock, http://dbpedia.org/resource/Jim_Strong_(running_back), http://dbpedia.org/resource/Lance_Schulters, http://dbpedia.org/resource/Bob_Jury, http://dbpedia.org/resource/Marcus_Martin, http://dbpedia.org/resource/Ian_Williams_(American_football), http://dbpedia.org/resource/Pete_Kugler, http://dbpedia.org/resource/Mike_Salmon_(American_football), http://dbpedia.org/resource/Josh_Shaw, http://dbpedia.org/resource/Chuck_Jacobs, http://dbpedia.org/resource/Bill_Romanowski, http://dbpedia.org/resource/John_Brodie, http://dbpedia.org/resource/John_Henry_Johnson, http://dbpedia.org/resource/Barry_Helton, http://dbpedia.org/resource/Dexter_Carter, http://dbpedia.org/resource/Kevin_Fagan_(American_football), http://dbpedia.org/resource/Merton_Hanks, http://dbpedia.org/resource/Michael_Carter_(athlete), http://dbpedia.org/resource/Steve_Wallace_(American_football), http://dbpedia.org/resource/Terrence_Flagler, http://dbpedia.org/resource/Wesley_Walls, http://dbpedia.org/resource/Scott_Hilton, http://dbpedia.org/resource/Howard_Stidham, http://dbpedia.org/resource/Chris_Dalman, http://dbpedia.org/resource/Jason_Hill_(American_football), http://dbpedia.org/resource/Mike_Simpson_(American_football), http://dbpedia.org/resource/Otis_Amey, http://dbpedia.org/resource/Eric_Johnson_(tight_end), http://dbpedia.org/resource/Jason_Baker, http://dbpedia.org/resource/Dave_Parks, http://dbpedia.org/resource/Tom_Seabron, http://dbpedia.org/resource/R._W._McQuarters, http://dbpedia.org/resource/Adam_Snyder, http://dbpedia.org/resource/Saleem_Rasheed, http://dbpedia.org/resource/Derrick_Johnson_(cornerback), http://dbpedia.org/resource/Dana_Hall, http://dbpedia.org/resource/Mark_Thomas_(American_football), http://dbpedia.org/resource/Matt_LaBounty, http://dbpedia.org/resource/Rod_Milstead, http://dbpedia.org/resource/Kentwan_Balmer, http://dbpedia.org/resource/DiAndre_Campbell, http://dbpedia.org/resource/Bob_St._Clair, http://dbpedia.org/resource/Earl_Morrall, http://dbpedia.org/resource/Ricky_Watters, http://dbpedia.org/resource/Ronnie_Lott, http://dbpedia.org/resource/Kyle_Kosier, http://dbpedia.org/resource/Joe_Hastings_(American_football), http://dbpedia.org/resource/Kwame_Harris, http://dbpedia.org/resource/Shawntae_Spencer, http://dbpedia.org/resource/Tony_Brown_(defensive_tackle), http://dbpedia.org/resource/Billy_Bajema, http://dbpedia.org/resource/John_Macaulay_(American_football), http://dbpedia.org/resource/Adrian_Hardy, http://dbpedia.org/resource/Darryl_Morris_(American_football), http://dbpedia.org/resource/A._J._Jenkins, http://dbpedia.org/resource/Greg_Cox_(American_football), http://dbpedia.org/resource/Brit_Miller, http://dbpedia.org/resource/Forrest_Blue, http://dbpedia.org/resource/Manny_Lawson, http://dbpedia.org/resource/Vernon_Davis, http://dbpedia.org/resource/Doug_Brien, http://dbpedia.org/resource/David_Kopay, http://dbpedia.org/resource/Chike_Okeafor, http://dbpedia.org/resource/Cody_Pickett, http://dbpedia.org/resource/John_Johnson_(linebacker), http://dbpedia.org/resource/Mark_Anelli, http://dbpedia.org/resource/Ron_Heller_(tight_end), http://dbpedia.org/resource/Tai_Streets, http://dbpedia.org/resource/Odell_Haggins, http://dbpedia.org/resource/Karl_Rubke, http://dbpedia.org/resource/Tony_Jerod-Eddie, http://dbpedia.org/resource/Vance_McDonald, http://dbpedia.org/resource/Keith_Lewis, http://dbpedia.org/resource/Carlos_Hyde, http://dbpedia.org/resource/Wade_Richey, http://dbpedia.org/resource/Eric_Heitmann, http://dbpedia.org/resource/John_Sutro_(American_football), http://dbpedia.org/resource/Fred_Beasley, http://dbpedia.org/resource/Jeff_Bregel, http://dbpedia.org/resource/Tim_Long_(American_football), http://dbpedia.org/resource/Tim_Washington, http://dbpedia.org/resource/Kevin_Lilly, http://dbpedia.org/resource/Chase_Thomas, http://dbpedia.org/resource/Verl_Lillywhite, http://dbpedia.org/resource/Michael_Barber_(wide_receiver), http://dbpedia.org/resource/Charles_Haley, http://dbpedia.org/resource/Billy_Kilmer, http://dbpedia.org/resource/Dana_Stubblefield, http://dbpedia.org/resource/Charlie_Krueger, http://dbpedia.org/resource/Delvin_Williams, http://dbpedia.org/resource/Jimmy_Johnson_(cornerback), http://dbpedia.org/resource/Paul_Smith_(fullback), http://dbpedia.org/resource/Terry_Beasley, http://dbpedia.org/resource/Nate_Singleton, http://dbpedia.org/resource/Tarell_Brown, http://dbpedia.org/resource/Todd_Shell, http://dbpedia.org/resource/Cam_Johnson, http://dbpedia.org/resource/Damien_Russell, http://dbpedia.org/resource/Patrick_Estes, http://dbpedia.org/resource/Rashaun_Woods, http://dbpedia.org/resource/Calvin_Nicholas, http://dbpedia.org/resource/Marcus_Hudson, http://dbpedia.org/resource/Brandon_Williams_(wide_receiver), http://dbpedia.org/resource/Marcus_Maxwell, http://dbpedia.org/resource/Bobby_Hoppe, http://dbpedia.org/resource/Glen_Coffee, http://dbpedia.org/resource/Marc_Edwards_(American_football), http://dbpedia.org/resource/Bruce_Ellington, http://dbpedia.org/resource/Derrick_Hamilton, http://dbpedia.org/resource/Scott_McKillop, http://dbpedia.org/resource/Gail_Bruce, http://dbpedia.org/resource/Chad_Stanley, http://dbpedia.org/resource/Bill_Johnson_(center), http://dbpedia.org/resource/Tony_Blevins, http://dbpedia.org/resource/Isaac_Sopoaga, http://dbpedia.org/resource/Andrew_Tiller, http://dbpedia.org/resource/Martin_Harrison_(American_football), http://dbpedia.org/resource/Justin_Smiley, http://dbpedia.org/resource/Demarcus_Dobbs, http://dbpedia.org/resource/Ted_Washington, http://dbpedia.org/resource/Keaton_Kristick, http://dbpedia.org/resource/Jim_Powers_(American_football), http://dbpedia.org/resource/Aaron_Lynch_(American_football), http://dbpedia.org/resource/Dave_Atkins_(American_football), http://dbpedia.org/resource/Reggie_Smith_(defensive_back), http://dbpedia.org/resource/Shonn_Bell, http://dbpedia.org/resource/Richard_Seigler, http://dbpedia.org/resource/Dan_Colchico] [INFO] Predicted Answer: [http://dbpedia.org/resource/Roger_Craig_(American_football), http://dbpedia.org/resource/Joe_Perry_(American_football), http://dbpedia.org/resource/Ernie_Hughes, http://dbpedia.org/resource/Doug_DuBose, http://dbpedia.org/resource/Jamie_Winborn, http://dbpedia.org/resource/Quinton_Dial, http://dbpedia.org/resource/Wayne_Trimble, http://dbpedia.org/resource/Joe_Looney_(offensive_lineman), http://dbpedia.org/resource/Parys_Haralson, http://dbpedia.org/resource/Roy_Williams_(defensive_tackle), http://dbpedia.org/resource/Brandon_Moore_(linebacker), http://dbpedia.org/resource/Ahmed_Plummer, http://dbpedia.org/resource/Alex_Boone, http://dbpedia.org/resource/Andy_Lee_(American_football), http://dbpedia.org/resource/Dominique_Zeigler, http://dbpedia.org/resource/Kendall_Hunter, http://dbpedia.org/resource/LaMichael_James, http://dbpedia.org/resource/Ronnie_Heard, http://dbpedia.org/resource/Mike_Magac, http://dbpedia.org/resource/Jaquiski_Tartt, http://dbpedia.org/resource/Eddie_Lewis_(American_football), http://dbpedia.org/resource/Jason_Webster, http://dbpedia.org/resource/Brian_Jennings, http://dbpedia.org/resource/Andrew_Williams_(American_football), http://dbpedia.org/resource/Garrett_Celek, http://dbpedia.org/resource/Ronald_Fields, http://dbpedia.org/resource/Mike_Burns_(American_football), http://dbpedia.org/resource/Jimmy_Williams_(cornerback,_born_1979), http://dbpedia.org/resource/Iheanyi_Uwaezuoke, http://dbpedia.org/resource/Kevin_Mitchell_(linebacker), http://dbpedia.org/resource/Hugh_McElhenny, http://dbpedia.org/resource/Renaldo_Nehemiah, http://dbpedia.org/resource/Howard_Mudd, http://dbpedia.org/resource/J._J._Stokes, http://dbpedia.org/resource/Keith_Henderson, http://dbpedia.org/resource/Charles_Johnson_(defensive_back), http://dbpedia.org/resource/Chris_Patrick, http://dbpedia.org/resource/Jim_Looney, http://dbpedia.org/resource/Julian_Peterson, http://dbpedia.org/resource/David_Baas, http://dbpedia.org/resource/Harvey_Dahl, http://dbpedia.org/resource/Tony_Peterson, http://dbpedia.org/resource/Tramaine_Brock, http://dbpedia.org/resource/Jim_Strong_(running_back), http://dbpedia.org/resource/Lance_Schulters, http://dbpedia.org/resource/Bob_Jury, http://dbpedia.org/resource/Marcus_Martin, http://dbpedia.org/resource/Ian_Williams_(American_football), http://dbpedia.org/resource/Pete_Kugler, http://dbpedia.org/resource/Mike_Salmon_(American_football), http://dbpedia.org/resource/Josh_Shaw, http://dbpedia.org/resource/Chuck_Jacobs, http://dbpedia.org/resource/Bill_Romanowski, http://dbpedia.org/resource/John_Brodie, http://dbpedia.org/resource/John_Henry_Johnson, http://dbpedia.org/resource/Barry_Helton, http://dbpedia.org/resource/Dexter_Carter, http://dbpedia.org/resource/Kevin_Fagan_(American_football), http://dbpedia.org/resource/Merton_Hanks, http://dbpedia.org/resource/Michael_Carter_(athlete), http://dbpedia.org/resource/Steve_Wallace_(American_football), http://dbpedia.org/resource/Terrence_Flagler, http://dbpedia.org/resource/Wesley_Walls, http://dbpedia.org/resource/Scott_Hilton, http://dbpedia.org/resource/Howard_Stidham, http://dbpedia.org/resource/Chris_Dalman, http://dbpedia.org/resource/Jason_Hill_(American_football), http://dbpedia.org/resource/Mike_Simpson_(American_football), http://dbpedia.org/resource/Otis_Amey, http://dbpedia.org/resource/Eric_Johnson_(tight_end), http://dbpedia.org/resource/Jason_Baker, http://dbpedia.org/resource/Dave_Parks, http://dbpedia.org/resource/Tom_Seabron, http://dbpedia.org/resource/R._W._McQuarters, http://dbpedia.org/resource/Adam_Snyder, http://dbpedia.org/resource/Saleem_Rasheed, http://dbpedia.org/resource/Derrick_Johnson_(cornerback), http://dbpedia.org/resource/Dana_Hall, http://dbpedia.org/resource/Mark_Thomas_(American_football), http://dbpedia.org/resource/Matt_LaBounty, http://dbpedia.org/resource/Rod_Milstead, http://dbpedia.org/resource/Kentwan_Balmer, http://dbpedia.org/resource/DiAndre_Campbell, http://dbpedia.org/resource/Bob_St._Clair, http://dbpedia.org/resource/Earl_Morrall, http://dbpedia.org/resource/Ricky_Watters, http://dbpedia.org/resource/Ronnie_Lott, http://dbpedia.org/resource/Kyle_Kosier, http://dbpedia.org/resource/Joe_Hastings_(American_football), http://dbpedia.org/resource/Kwame_Harris, http://dbpedia.org/resource/Shawntae_Spencer, http://dbpedia.org/resource/Tony_Brown_(defensive_tackle), http://dbpedia.org/resource/Billy_Bajema, http://dbpedia.org/resource/John_Macaulay_(American_football), http://dbpedia.org/resource/Adrian_Hardy, http://dbpedia.org/resource/Darryl_Morris_(American_football), http://dbpedia.org/resource/A._J._Jenkins, http://dbpedia.org/resource/Greg_Cox_(American_football), http://dbpedia.org/resource/Brit_Miller, http://dbpedia.org/resource/Forrest_Blue, http://dbpedia.org/resource/Manny_Lawson, http://dbpedia.org/resource/Vernon_Davis, http://dbpedia.org/resource/Doug_Brien, http://dbpedia.org/resource/David_Kopay, http://dbpedia.org/resource/Chike_Okeafor, http://dbpedia.org/resource/Cody_Pickett, http://dbpedia.org/resource/John_Johnson_(linebacker), http://dbpedia.org/resource/Mark_Anelli, http://dbpedia.org/resource/Ron_Heller_(tight_end), http://dbpedia.org/resource/Tai_Streets, http://dbpedia.org/resource/Odell_Haggins, http://dbpedia.org/resource/Karl_Rubke, http://dbpedia.org/resource/Tony_Jerod-Eddie, http://dbpedia.org/resource/Vance_McDonald, http://dbpedia.org/resource/Keith_Lewis, http://dbpedia.org/resource/Carlos_Hyde, http://dbpedia.org/resource/Wade_Richey, http://dbpedia.org/resource/Eric_Heitmann, http://dbpedia.org/resource/John_Sutro_(American_football), http://dbpedia.org/resource/Fred_Beasley, http://dbpedia.org/resource/Jeff_Bregel, http://dbpedia.org/resource/Tim_Long_(American_football), http://dbpedia.org/resource/Tim_Washington, http://dbpedia.org/resource/Kevin_Lilly, http://dbpedia.org/resource/Chase_Thomas, http://dbpedia.org/resource/Verl_Lillywhite, http://dbpedia.org/resource/Michael_Barber_(wide_receiver), http://dbpedia.org/resource/Charles_Haley, http://dbpedia.org/resource/Billy_Kilmer, http://dbpedia.org/resource/Dana_Stubblefield, http://dbpedia.org/resource/Charlie_Krueger, http://dbpedia.org/resource/Delvin_Williams, http://dbpedia.org/resource/Jimmy_Johnson_(cornerback), http://dbpedia.org/resource/Paul_Smith_(fullback), http://dbpedia.org/resource/Terry_Beasley, http://dbpedia.org/resource/Nate_Singleton, http://dbpedia.org/resource/Tarell_Brown, http://dbpedia.org/resource/Todd_Shell, http://dbpedia.org/resource/Cam_Johnson, http://dbpedia.org/resource/Damien_Russell, http://dbpedia.org/resource/Patrick_Estes, http://dbpedia.org/resource/Rashaun_Woods, http://dbpedia.org/resource/Calvin_Nicholas, http://dbpedia.org/resource/Marcus_Hudson, http://dbpedia.org/resource/Brandon_Williams_(wide_receiver), http://dbpedia.org/resource/Marcus_Maxwell, http://dbpedia.org/resource/Bobby_Hoppe, http://dbpedia.org/resource/Glen_Coffee, http://dbpedia.org/resource/Marc_Edwards_(American_football), http://dbpedia.org/resource/Bruce_Ellington, http://dbpedia.org/resource/Derrick_Hamilton, http://dbpedia.org/resource/Scott_McKillop, http://dbpedia.org/resource/Gail_Bruce, http://dbpedia.org/resource/Chad_Stanley, http://dbpedia.org/resource/Bill_Johnson_(center), http://dbpedia.org/resource/Tony_Blevins, http://dbpedia.org/resource/Isaac_Sopoaga, http://dbpedia.org/resource/Andrew_Tiller, http://dbpedia.org/resource/Martin_Harrison_(American_football), http://dbpedia.org/resource/Justin_Smiley, http://dbpedia.org/resource/Demarcus_Dobbs, http://dbpedia.org/resource/Ted_Washington, http://dbpedia.org/resource/Keaton_Kristick, http://dbpedia.org/resource/Jim_Powers_(American_football), http://dbpedia.org/resource/Aaron_Lynch_(American_football), http://dbpedia.org/resource/Dave_Atkins_(American_football), http://dbpedia.org/resource/Reggie_Smith_(defensive_back), http://dbpedia.org/resource/Shonn_Bell, http://dbpedia.org/resource/Richard_Seigler, http://dbpedia.org/resource/Dan_Colchico] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 439, P: 0.520, R: 0.574, macro F1: 0.527, macro F1*: 0.546 -[INFO] Current time: 3877 ms, average: 2797.45 ms/ques, entity detection: 436.19 ms/ques (linking tool: 345.28 ms/ques), relation detection: 915.51 ms/ques (similarity: 707.58 ms/ques), EDG generation: 54.18 ms/ques +[INFO] Cumulative metrics, sample: 439, P: 0.520, R: 0.574, macro F1: 0.546 +[INFO] Current question consumed time: 3203 ms, average: 2575.31 ms/ques, entity detection: 398.14 ms/ques (linking tool: 317.03 ms/ques), relation detection: 858.80 ms/ques (similarity: 691.42 ms/ques), EDG generation: 54.33 ms/ques [INFO] Entity Linking P: 0.7482915717539863, R: 0.7072892938496583, micro F1: 0.7201974183750951, macro F1: 0.7272129359299304 [INFO] Relation Linking P: 0.5971905846621107, R: 0.5558086560364465, micro F1: 0.5665148063781316, macro F1: 0.5757570075372606 [INFO] Type Linking P: 0.5421412300683371, R: 0.5421412300683371, micro F1: 0.5421412300683371, macro F1: 0.5421412300683371 @@ -6158,8 +6158,8 @@ Question 439: What are the nicknames given to the characters of The Wolves of th [INFO] Golden Answer: [http://dbpedia.org/resource/Romulus_and_Remus, http://dbpedia.org/resource/Ringo_Starr, Sam@en, The Boy@en, NARKOTIKA@en, The Pissy One@en, The White One@en, White She-Wolf@en] [INFO] Predicted Answer: [http://dbpedia.org/resource/Romulus_and_Remus, http://dbpedia.org/resource/Ringo_Starr, Sam@en, The Boy@en, NARKOTIKA@en, The Pissy One@en, The White One@en, White She-Wolf@en] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 440, P: 0.521, R: 0.575, macro F1: 0.528, macro F1*: 0.547 -[INFO] Current time: 1364 ms, average: 2794.20 ms/ques, entity detection: 435.60 ms/ques (linking tool: 344.79 ms/ques), relation detection: 914.00 ms/ques (similarity: 706.33 ms/ques), EDG generation: 54.40 ms/ques +[INFO] Cumulative metrics, sample: 440, P: 0.521, R: 0.575, macro F1: 0.547 +[INFO] Current question consumed time: 1214 ms, average: 2572.22 ms/ques, entity detection: 397.56 ms/ques (linking tool: 316.56 ms/ques), relation detection: 857.39 ms/ques (similarity: 690.23 ms/ques), EDG generation: 54.61 ms/ques [INFO] Entity Linking P: 0.7465909090909091, R: 0.7056818181818182, micro F1: 0.7185606060606062, macro F1: 0.7255601792573624 [INFO] Relation Linking P: 0.5969696969696969, R: 0.5568181818181818, micro F1: 0.5667424242424236, macro F1: 0.5761953083029904 [INFO] Type Linking P: 0.5431818181818182, R: 0.5431818181818182, micro F1: 0.5431818181818182, macro F1: 0.5431818181818182 @@ -6172,8 +6172,8 @@ Question 440: Name the river with source country in Zambia and source region as [INFO] Golden Answer: [http://dbpedia.org/resource/Zambezi] [INFO] Predicted Answer: [http://dbpedia.org/resource/Zambezi] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 441, P: 0.523, R: 0.576, macro F1: 0.529, macro F1*: 0.548 -[INFO] Current time: 9096 ms, average: 2808.49 ms/ques, entity detection: 436.97 ms/ques (linking tool: 346.00 ms/ques), relation detection: 919.87 ms/ques (similarity: 713.68 ms/ques), EDG generation: 54.46 ms/ques +[INFO] Cumulative metrics, sample: 441, P: 0.523, R: 0.576, macro F1: 0.548 +[INFO] Current question consumed time: 8179 ms, average: 2584.93 ms/ques, entity detection: 398.66 ms/ques (linking tool: 317.57 ms/ques), relation detection: 862.62 ms/ques (similarity: 697.87 ms/ques), EDG generation: 54.71 ms/ques [INFO] Entity Linking P: 0.7471655328798186, R: 0.7063492063492064, micro F1: 0.7191987906273622, macro F1: 0.7261842854666569 [INFO] Relation Linking P: 0.5978835978835978, R: 0.5578231292517006, micro F1: 0.5677248677248671, macro F1: 0.5771590519791958 [INFO] Type Linking P: 0.5419501133786848, R: 0.5419501133786848, micro F1: 0.5419501133786848, macro F1: 0.5419501133786848 @@ -6186,8 +6186,8 @@ Question 441: Members of how many political parties have died in Delhi? [INFO] Golden Answer: [3] [INFO] Predicted Answer: [1, 9] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 442, P: 0.521, R: 0.574, macro F1: 0.527, macro F1*: 0.547 -[INFO] Current time: 4820 ms, average: 2813.04 ms/ques, entity detection: 437.03 ms/ques (linking tool: 346.05 ms/ques), relation detection: 922.04 ms/ques (similarity: 716.57 ms/ques), EDG generation: 54.40 ms/ques +[INFO] Cumulative metrics, sample: 442, P: 0.521, R: 0.574, macro F1: 0.547 +[INFO] Current question consumed time: 4359 ms, average: 2588.95 ms/ques, entity detection: 398.64 ms/ques (linking tool: 317.55 ms/ques), relation detection: 864.52 ms/ques (similarity: 700.89 ms/ques), EDG generation: 54.66 ms/ques [INFO] Entity Linking P: 0.747737556561086, R: 0.7070135746606335, micro F1: 0.7198340874811465, macro F1: 0.7268055565329373 [INFO] Relation Linking P: 0.5965309200603317, R: 0.5565610859728507, micro F1: 0.5664404223227746, macro F1: 0.5758532622688357 [INFO] Type Linking P: 0.5429864253393665, R: 0.5429864253393665, micro F1: 0.5429864253393665, macro F1: 0.5429864253393665 @@ -6200,8 +6200,8 @@ Question 442: What is the birthplace of Adugna Deyas and the place where Fiq is [INFO] Golden Answer: [http://dbpedia.org/resource/Ethiopia] [INFO] Predicted Answer: [http://dbpedia.org/resource/Ethiopia] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 443, P: 0.522, R: 0.575, macro F1: 0.529, macro F1*: 0.548 -[INFO] Current time: 2037 ms, average: 2811.29 ms/ques, entity detection: 436.42 ms/ques (linking tool: 345.48 ms/ques), relation detection: 921.62 ms/ques (similarity: 716.80 ms/ques), EDG generation: 54.39 ms/ques +[INFO] Cumulative metrics, sample: 443, P: 0.522, R: 0.575, macro F1: 0.548 +[INFO] Current question consumed time: 1806 ms, average: 2587.18 ms/ques, entity detection: 398.02 ms/ques (linking tool: 317.00 ms/ques), relation detection: 864.07 ms/ques (similarity: 701.14 ms/ques), EDG generation: 54.64 ms/ques [INFO] Entity Linking P: 0.7483069977426636, R: 0.7065462753950339, micro F1: 0.7197140707298723, macro F1: 0.7268272778695227 [INFO] Relation Linking P: 0.5974416854778027, R: 0.5564334085778782, micro F1: 0.566666666666666, macro F1: 0.5762088378361072 [INFO] Type Linking P: 0.5417607223476298, R: 0.5417607223476298, micro F1: 0.5417607223476298, macro F1: 0.5417607223476298 @@ -6214,8 +6214,8 @@ Question 443: what is the headquarters of Juan Dalmau Ramerez? [INFO] Golden Answer: [http://dbpedia.org/resource/Puerto_Rico, http://dbpedia.org/resource/San_Juan,_Puerto_Rico] [INFO] Predicted Answer: [http://dbpedia.org/resource/Puerto_Rico, http://dbpedia.org/resource/San_Juan,_Puerto_Rico] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 444, P: 0.524, R: 0.576, macro F1: 0.530, macro F1*: 0.549 -[INFO] Current time: 945 ms, average: 2807.08 ms/ques, entity detection: 435.76 ms/ques (linking tool: 344.89 ms/ques), relation detection: 920.34 ms/ques (similarity: 715.65 ms/ques), EDG generation: 54.29 ms/ques +[INFO] Cumulative metrics, sample: 444, P: 0.524, R: 0.576, macro F1: 0.549 +[INFO] Current question consumed time: 772 ms, average: 2583.09 ms/ques, entity detection: 397.38 ms/ques (linking tool: 316.47 ms/ques), relation detection: 862.82 ms/ques (similarity: 700.00 ms/ques), EDG generation: 54.53 ms/ques [INFO] Entity Linking P: 0.7488738738738738, R: 0.7072072072072072, micro F1: 0.7203453453453456, macro F1: 0.727444381736725 [INFO] Relation Linking P: 0.5983483483483483, R: 0.5574324324324325, micro F1: 0.567642642642642, macro F1: 0.5771661560878839 [INFO] Type Linking P: 0.5427927927927928, R: 0.5427927927927928, micro F1: 0.5427927927927928, macro F1: 0.5427927927927928 @@ -6228,8 +6228,8 @@ Question 444: The president who was in office when Lee Hyun Jae was the PM follo [INFO] Golden Answer: [http://dbpedia.org/resource/Buddhism] [INFO] Predicted Answer: [http://dbpedia.org/resource/Roh_Tae-woo] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 445, P: 0.522, R: 0.575, macro F1: 0.528, macro F1*: 0.547 -[INFO] Current time: 1523 ms, average: 2804.20 ms/ques, entity detection: 435.08 ms/ques (linking tool: 344.31 ms/ques), relation detection: 919.37 ms/ques (similarity: 715.51 ms/ques), EDG generation: 54.24 ms/ques +[INFO] Cumulative metrics, sample: 445, P: 0.522, R: 0.575, macro F1: 0.547 +[INFO] Current question consumed time: 1329 ms, average: 2580.27 ms/ques, entity detection: 396.71 ms/ques (linking tool: 315.90 ms/ques), relation detection: 861.85 ms/ques (similarity: 699.90 ms/ques), EDG generation: 54.48 ms/ques [INFO] Entity Linking P: 0.749438202247191, R: 0.7078651685393258, micro F1: 0.7209737827715358, macro F1: 0.7280587007181656 [INFO] Relation Linking P: 0.597003745318352, R: 0.5561797752808989, micro F1: 0.5663670411985012, macro F1: 0.5758691534899336 [INFO] Type Linking P: 0.5415730337078651, R: 0.5415730337078651, micro F1: 0.5415730337078651, macro F1: 0.5415730337078651 @@ -6242,8 +6242,8 @@ Question 445: Who is the maker of the engine of Ford F-Series first generation ? [INFO] Golden Answer: [http://dbpedia.org/resource/Ford_Motor_Company] [INFO] Predicted Answer: [http://dbpedia.org/resource/Ford_F-Series_third_generation, http://dbpedia.org/resource/Ford_F-Series_seventh_generation, http://dbpedia.org/resource/Ford_F-Series_sixth_generation, http://dbpedia.org/resource/Ford_Crestline, http://dbpedia.org/resource/Ford_Parklane, http://dbpedia.org/resource/Ford_Ranchero, http://dbpedia.org/resource/Hero_Karizma_ZMR, http://dbpedia.org/resource/Mercury_Meteor, http://dbpedia.org/resource/Sisu_K-50SS, http://dbpedia.org/resource/Ford_F-Series_second_generation, http://dbpedia.org/resource/Yamaha_Scorpio_Z, http://dbpedia.org/resource/Ford_F-Series_fourth_generation, http://dbpedia.org/resource/1957_Ford, http://dbpedia.org/resource/Ford_Galaxie, http://dbpedia.org/resource/1955_Ford, http://dbpedia.org/resource/Yamaha_XT_225, http://dbpedia.org/resource/Hero_Honda_Karizma_ZMR, http://dbpedia.org/resource/1960_Ford, http://dbpedia.org/resource/Chevrolet_C/K] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 446, P: 0.521, R: 0.574, macro F1: 0.527, macro F1*: 0.546 -[INFO] Current time: 1004 ms, average: 2800.16 ms/ques, entity detection: 434.47 ms/ques (linking tool: 343.75 ms/ques), relation detection: 917.93 ms/ques (similarity: 714.23 ms/ques), EDG generation: 54.20 ms/ques +[INFO] Cumulative metrics, sample: 446, P: 0.521, R: 0.574, macro F1: 0.546 +[INFO] Current question consumed time: 800 ms, average: 2576.28 ms/ques, entity detection: 396.09 ms/ques (linking tool: 315.36 ms/ques), relation detection: 860.46 ms/ques (similarity: 698.64 ms/ques), EDG generation: 54.43 ms/ques [INFO] Entity Linking P: 0.7477578475336323, R: 0.7062780269058296, micro F1: 0.7193572496263081, macro F1: 0.7264262821066898 [INFO] Relation Linking P: 0.5979073243647234, R: 0.5560538116591929, micro F1: 0.5665919282511204, macro F1: 0.5762215664862101 [INFO] Type Linking P: 0.5426008968609866, R: 0.5426008968609866, micro F1: 0.5426008968609866, macro F1: 0.5426008968609866 @@ -6256,8 +6256,8 @@ Question 446: What is the religious affiliation of Neungin High School? [INFO] Golden Answer: [http://dbpedia.org/resource/Buddhism] [INFO] Predicted Answer: [http://dbpedia.org/resource/Buddhism] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 447, P: 0.522, R: 0.574, macro F1: 0.528, macro F1*: 0.547 -[INFO] Current time: 1130 ms, average: 2796.43 ms/ques, entity detection: 434.47 ms/ques (linking tool: 343.73 ms/ques), relation detection: 916.12 ms/ques (similarity: 712.79 ms/ques), EDG generation: 54.10 ms/ques +[INFO] Cumulative metrics, sample: 447, P: 0.522, R: 0.574, macro F1: 0.547 +[INFO] Current question consumed time: 935 ms, average: 2572.61 ms/ques, entity detection: 396.00 ms/ques (linking tool: 315.27 ms/ques), relation detection: 858.78 ms/ques (similarity: 697.26 ms/ques), EDG generation: 54.33 ms/ques [INFO] Entity Linking P: 0.7483221476510067, R: 0.7069351230425056, micro F1: 0.719985085756898, macro F1: 0.7270401188553978 [INFO] Relation Linking P: 0.5988068605518269, R: 0.5570469798657718, micro F1: 0.5675615212527958, macro F1: 0.5771725481705996 [INFO] Type Linking P: 0.5436241610738255, R: 0.5436241610738255, micro F1: 0.5436241610738255, macro F1: 0.5436241610738255 @@ -6270,8 +6270,8 @@ Question 447: Is Maria Sharapova the silver medalist of Tennis at the 2012 Summe [INFO] Golden Answer: [true] [INFO] Predicted Answer: [true] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 448, P: 0.523, R: 0.575, macro F1: 0.529, macro F1*: 0.548 -[INFO] Current time: 3047 ms, average: 2796.98 ms/ques, entity detection: 435.74 ms/ques (linking tool: 344.81 ms/ques), relation detection: 915.02 ms/ques (similarity: 711.75 ms/ques), EDG generation: 54.09 ms/ques +[INFO] Cumulative metrics, sample: 448, P: 0.523, R: 0.575, macro F1: 0.548 +[INFO] Current question consumed time: 2510 ms, average: 2572.47 ms/ques, entity detection: 396.92 ms/ques (linking tool: 316.07 ms/ques), relation detection: 857.72 ms/ques (similarity: 696.22 ms/ques), EDG generation: 54.31 ms/ques [INFO] Entity Linking P: 0.7477678571428571, R: 0.7064732142857143, micro F1: 0.7194940476190478, macro F1: 0.7265342341848482 [INFO] Relation Linking P: 0.597470238095238, R: 0.5558035714285714, micro F1: 0.5662946428571421, macro F1: 0.5758842165898618 [INFO] Type Linking P: 0.5446428571428571, R: 0.5446428571428571, micro F1: 0.5446428571428571, macro F1: 0.5446428571428571 @@ -6284,8 +6284,8 @@ Question 448: Name the office holder with successor as Adrian A Basora and child [INFO] Golden Answer: [http://dbpedia.org/resource/Shirley_Temple] [INFO] Predicted Answer: [http://dbpedia.org/resource/Shirley_Temple, http://dbpedia.org/resource/Jenonne_R._Walker] [INFO] P: 0.5, R: 1.0, F: 0.6666666666666666 -[INFO] Cumulative metrics, sample: 449, P: 0.523, R: 0.576, macro F1: 0.530, macro F1*: 0.549 -[INFO] Current time: 4807 ms, average: 2801.46 ms/ques, entity detection: 435.99 ms/ques (linking tool: 344.89 ms/ques), relation detection: 915.74 ms/ques (similarity: 714.69 ms/ques), EDG generation: 54.11 ms/ques +[INFO] Cumulative metrics, sample: 449, P: 0.523, R: 0.576, macro F1: 0.549 +[INFO] Current question consumed time: 4302 ms, average: 2576.32 ms/ques, entity detection: 397.05 ms/ques (linking tool: 316.07 ms/ques), relation detection: 858.40 ms/ques (similarity: 699.25 ms/ques), EDG generation: 54.34 ms/ques [INFO] Entity Linking P: 0.7483296213808464, R: 0.7060133630289532, micro F1: 0.7193763919821828, macro F1: 0.7265558651691526 [INFO] Relation Linking P: 0.5983667409057163, R: 0.5556792873051225, micro F1: 0.5665181885671856, macro F1: 0.5762335227634979 [INFO] Type Linking P: 0.5434298440979956, R: 0.5434298440979956, micro F1: 0.5434298440979956, macro F1: 0.5434298440979956 @@ -6298,8 +6298,8 @@ Question 449: Who is the owner of the South end ? [INFO] Golden Answer: [http://dbpedia.org/resource/Wayne_State_University] [INFO] Predicted Answer: [http://dbpedia.org/resource/Tony_Rougier] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 450, P: 0.522, R: 0.575, macro F1: 0.528, macro F1*: 0.547 -[INFO] Current time: 795 ms, average: 2797.00 ms/ques, entity detection: 435.41 ms/ques (linking tool: 344.31 ms/ques), relation detection: 914.30 ms/ques (similarity: 713.46 ms/ques), EDG generation: 54.01 ms/ques +[INFO] Cumulative metrics, sample: 450, P: 0.522, R: 0.575, macro F1: 0.547 +[INFO] Current question consumed time: 715 ms, average: 2572.18 ms/ques, entity detection: 396.52 ms/ques (linking tool: 315.55 ms/ques), relation detection: 857.06 ms/ques (similarity: 698.05 ms/ques), EDG generation: 54.26 ms/ques [INFO] Entity Linking P: 0.7466666666666667, R: 0.7044444444444444, micro F1: 0.7177777777777781, macro F1: 0.7249412965798878 [INFO] Relation Linking P: 0.5970370370370369, R: 0.5544444444444444, micro F1: 0.5652592592592586, macro F1: 0.5749530038240234 [INFO] Type Linking P: 0.5444444444444444, R: 0.5444444444444444, micro F1: 0.5444444444444444, macro F1: 0.5444444444444444 @@ -6312,8 +6312,8 @@ Question 450: What is the colour of Xocolatlite ? [INFO] Golden Answer: [http://dbpedia.org/resource/Chocolate] [INFO] Predicted Answer: [http://dbpedia.org/resource/Chocolate] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 451, P: 0.523, R: 0.576, macro F1: 0.530, macro F1*: 0.548 -[INFO] Current time: 265 ms, average: 2791.39 ms/ques, entity detection: 434.51 ms/ques (linking tool: 343.59 ms/ques), relation detection: 912.57 ms/ques (similarity: 712.07 ms/ques), EDG generation: 53.91 ms/ques +[INFO] Cumulative metrics, sample: 451, P: 0.523, R: 0.576, macro F1: 0.548 +[INFO] Current question consumed time: 238 ms, average: 2567.01 ms/ques, entity detection: 395.70 ms/ques (linking tool: 314.89 ms/ques), relation detection: 855.43 ms/ques (similarity: 696.69 ms/ques), EDG generation: 54.16 ms/ques [INFO] Entity Linking P: 0.7472283813747228, R: 0.70509977827051, micro F1: 0.7184035476718406, macro F1: 0.7255530542814104 [INFO] Relation Linking P: 0.5979305247597929, R: 0.5554323725055432, micro F1: 0.5662232076866216, macro F1: 0.5758984804318927 [INFO] Type Linking P: 0.5454545454545454, R: 0.5454545454545454, micro F1: 0.5454545454545454, macro F1: 0.5454545454545454 @@ -6326,8 +6326,8 @@ Question 451: For how many other teams have the members of Atlant Moscow Oblast [INFO] Golden Answer: [149] [INFO] Predicted Answer: [180] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 452, P: 0.522, R: 0.575, macro F1: 0.528, macro F1*: 0.547 -[INFO] Current time: 2945 ms, average: 2791.73 ms/ques, entity detection: 434.00 ms/ques (linking tool: 343.12 ms/ques), relation detection: 912.30 ms/ques (similarity: 711.92 ms/ques), EDG generation: 53.84 ms/ques +[INFO] Cumulative metrics, sample: 452, P: 0.522, R: 0.575, macro F1: 0.547 +[INFO] Current question consumed time: 2618 ms, average: 2567.12 ms/ques, entity detection: 395.24 ms/ques (linking tool: 314.48 ms/ques), relation detection: 855.17 ms/ques (similarity: 696.65 ms/ques), EDG generation: 54.10 ms/ques [INFO] Entity Linking P: 0.7477876106194691, R: 0.7057522123893806, micro F1: 0.7190265486725667, macro F1: 0.7261620937217981 [INFO] Relation Linking P: 0.59882005899705, R: 0.5564159292035398, micro F1: 0.5671828908554565, macro F1: 0.5768397504159263 [INFO] Type Linking P: 0.5464601769911505, R: 0.5464601769911505, micro F1: 0.5464601769911505, macro F1: 0.5464601769911505 @@ -6340,8 +6340,8 @@ Question 452: What is the state of origin of the president whose alma mater is H [INFO] Golden Answer: [http://dbpedia.org/resource/Greece] [INFO] Predicted Answer: [http://dbpedia.org/resource/Greece] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 453, P: 0.523, R: 0.576, macro F1: 0.529, macro F1*: 0.548 -[INFO] Current time: 1566 ms, average: 2789.02 ms/ques, entity detection: 433.41 ms/ques (linking tool: 342.56 ms/ques), relation detection: 911.68 ms/ques (similarity: 711.39 ms/ques), EDG generation: 53.78 ms/ques +[INFO] Cumulative metrics, sample: 453, P: 0.523, R: 0.576, macro F1: 0.548 +[INFO] Current question consumed time: 1428 ms, average: 2564.61 ms/ques, entity detection: 394.65 ms/ques (linking tool: 313.94 ms/ques), relation detection: 854.69 ms/ques (similarity: 696.49 ms/ques), EDG generation: 54.06 ms/ques [INFO] Entity Linking P: 0.7483443708609272, R: 0.7064017660044151, micro F1: 0.719646799116998, macro F1: 0.7267684330060599 [INFO] Relation Linking P: 0.5997056659308314, R: 0.5573951434878587, micro F1: 0.5681383370125085, macro F1: 0.5777768418983845 [INFO] Type Linking P: 0.5452538631346578, R: 0.5452538631346578, micro F1: 0.5452538631346578, macro F1: 0.5452538631346578 @@ -6354,8 +6354,8 @@ Question 453: What is the common ethnicity of the Maike Evers and Francisco Lach [INFO] Golden Answer: [http://dbpedia.org/resource/Germans] [INFO] Predicted Answer: [http://dbpedia.org/resource/Germans] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 454, P: 0.524, R: 0.577, macro F1: 0.530, macro F1*: 0.549 -[INFO] Current time: 1384 ms, average: 2785.93 ms/ques, entity detection: 433.21 ms/ques (linking tool: 342.20 ms/ques), relation detection: 910.32 ms/ques (similarity: 710.20 ms/ques), EDG generation: 53.77 ms/ques +[INFO] Cumulative metrics, sample: 454, P: 0.524, R: 0.577, macro F1: 0.549 +[INFO] Current question consumed time: 1203 ms, average: 2561.61 ms/ques, entity detection: 394.42 ms/ques (linking tool: 313.58 ms/ques), relation detection: 853.40 ms/ques (similarity: 695.33 ms/ques), EDG generation: 54.04 ms/ques [INFO] Entity Linking P: 0.748898678414097, R: 0.7070484581497798, micro F1: 0.720264317180617, macro F1: 0.7273720900784421 [INFO] Relation Linking P: 0.6005873715124815, R: 0.5583700440528634, micro F1: 0.5690895741556528, macro F1: 0.5787097827497526 [INFO] Type Linking P: 0.5462555066079295, R: 0.5462555066079295, micro F1: 0.5462555066079295, macro F1: 0.5462555066079295 @@ -6368,8 +6368,8 @@ Question 454: In which places do companies founded in Newcastle operate? [INFO] Golden Answer: [http://dbpedia.org/resource/Australia, http://dbpedia.org/resource/New_Zealand, http://dbpedia.org/resource/Asia, http://dbpedia.org/resource/Pacific_Ocean, http://dbpedia.org/resource/Africa] [INFO] Predicted Answer: [http://dbpedia.org/resource/Allen_Raine, http://dbpedia.org/resource/Gareth_Thomas_(rugby_union_born_1993), http://dbpedia.org/resource/Dill_Jones] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 455, P: 0.523, R: 0.575, macro F1: 0.529, macro F1*: 0.548 -[INFO] Current time: 1543 ms, average: 2783.20 ms/ques, entity detection: 433.25 ms/ques (linking tool: 342.24 ms/ques), relation detection: 909.14 ms/ques (similarity: 709.53 ms/ques), EDG generation: 53.68 ms/ques +[INFO] Cumulative metrics, sample: 455, P: 0.523, R: 0.575, macro F1: 0.548 +[INFO] Current question consumed time: 1361 ms, average: 2558.97 ms/ques, entity detection: 394.42 ms/ques (linking tool: 313.61 ms/ques), relation detection: 852.31 ms/ques (similarity: 694.73 ms/ques), EDG generation: 53.96 ms/ques [INFO] Entity Linking P: 0.7472527472527473, R: 0.7054945054945055, micro F1: 0.7186813186813189, macro F1: 0.7257734701002476 [INFO] Relation Linking P: 0.5992673992673991, R: 0.5571428571428572, micro F1: 0.5678388278388271, macro F1: 0.5774378931173356 [INFO] Type Linking P: 0.545054945054945, R: 0.545054945054945, micro F1: 0.545054945054945, macro F1: 0.545054945054945 @@ -6382,8 +6382,8 @@ Question 455: Which senator served in military units VMF-155 and VMF-218? [INFO] Golden Answer: [http://dbpedia.org/resource/John_Glenn] [INFO] Predicted Answer: [http://dbpedia.org/resource/John_Glenn] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 456, P: 0.524, R: 0.576, macro F1: 0.530, macro F1*: 0.549 -[INFO] Current time: 1580 ms, average: 2780.56 ms/ques, entity detection: 433.06 ms/ques (linking tool: 341.85 ms/ques), relation detection: 908.06 ms/ques (similarity: 708.93 ms/ques), EDG generation: 53.64 ms/ques +[INFO] Cumulative metrics, sample: 456, P: 0.524, R: 0.576, macro F1: 0.549 +[INFO] Current question consumed time: 1405 ms, average: 2556.44 ms/ques, entity detection: 394.17 ms/ques (linking tool: 313.23 ms/ques), relation detection: 851.27 ms/ques (similarity: 694.20 ms/ques), EDG generation: 53.91 ms/ques [INFO] Entity Linking P: 0.7478070175438597, R: 0.706140350877193, micro F1: 0.7192982456140353, macro F1: 0.7263766505252574 [INFO] Relation Linking P: 0.6001461988304093, R: 0.5581140350877193, micro F1: 0.5687865497076016, macro F1: 0.5783674633095834 [INFO] Type Linking P: 0.543859649122807, R: 0.543859649122807, micro F1: 0.543859649122807, macro F1: 0.543859649122807 @@ -6396,8 +6396,8 @@ Question 456: What kind of games are made by Interplay Entertainment? [INFO] Golden Answer: [http://dbpedia.org/resource/Space_flight_simulator_game, http://dbpedia.org/resource/Fighting_game, http://dbpedia.org/resource/Platform_game, http://dbpedia.org/resource/Third-person_shooter, http://dbpedia.org/resource/Real-time_strategy, http://dbpedia.org/resource/Real-time_tactics, http://dbpedia.org/resource/Adventure_game, http://dbpedia.org/resource/First-person_shooter, http://dbpedia.org/resource/Sports_game, http://dbpedia.org/resource/Cue_sports, http://dbpedia.org/resource/Simulation_video_game, http://dbpedia.org/resource/Turn-based_strategy, Role-playing video games@en, Role-playing game@en, Game compilation@en, http://dbpedia.org/resource/Pinball, http://dbpedia.org/resource/Action-adventure_game, http://dbpedia.org/resource/Arcade_game, http://dbpedia.org/resource/Action_game, http://dbpedia.org/resource/Puzzle_video_game, http://dbpedia.org/resource/Shoot_'em_up, http://dbpedia.org/resource/Strategy_game, http://dbpedia.org/resource/Role-playing_video_game, http://dbpedia.org/resource/Techno, http://dbpedia.org/resource/Breakbeat_hardcore, http://dbpedia.org/resource/Rave_music, http://dbpedia.org/resource/Tactical_role-playing_game, http://dbpedia.org/resource/Hack_and_slash, http://dbpedia.org/resource/Action_role-playing_game, 4, http://dbpedia.org/resource/Shooter_game, Simulation@en, http://dbpedia.org/resource/Racing_video_game, http://dbpedia.org/resource/Chess, http://dbpedia.org/resource/Strategy_video_game, http://dbpedia.org/resource/Side-scrolling_video_game, http://dbpedia.org/resource/Construction_and_management_simulation, http://dbpedia.org/resource/Massively_multiplayer_online_role-playing_game, Full Motion Video game@en, http://dbpedia.org/resource/Beat_'em_up, Sports simulation – pool@en, http://dbpedia.org/resource/List_of_vehicular_combat_games, http://dbpedia.org/resource/Gambling, Chess game@en, Futuristic flight simulation@en, Vertical scrolling shooter@en, Action@en, Compilation@en, http://dbpedia.org/resource/Interactive_movie] [INFO] Predicted Answer: [http://dbpedia.org/resource/Wasteland_(video_game), http://dbpedia.org/resource/Fallout_(series), http://dbpedia.org/resource/ClayFighter_(series), http://dbpedia.org/resource/Descent_(video_game), http://dbpedia.org/resource/Earthworm_Jim_(series), http://dbpedia.org/resource/Baldur's_Gate_(series), http://dbpedia.org/resource/Icewind_Dale_(series), http://dbpedia.org/resource/Battle_Chess, http://dbpedia.org/resource/Planescape:_Torment, http://dbpedia.org/resource/MDK_(series), http://dbpedia.org/resource/Stonekeep] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 457, P: 0.523, R: 0.575, macro F1: 0.529, macro F1*: 0.548 -[INFO] Current time: 1273 ms, average: 2777.26 ms/ques, entity detection: 432.52 ms/ques (linking tool: 341.35 ms/ques), relation detection: 906.81 ms/ques (similarity: 708.10 ms/ques), EDG generation: 53.54 ms/ques +[INFO] Cumulative metrics, sample: 457, P: 0.523, R: 0.575, macro F1: 0.548 +[INFO] Current question consumed time: 1131 ms, average: 2553.32 ms/ques, entity detection: 393.63 ms/ques (linking tool: 312.75 ms/ques), relation detection: 850.11 ms/ques (similarity: 693.42 ms/ques), EDG generation: 53.79 ms/ques [INFO] Entity Linking P: 0.7483588621444202, R: 0.7067833698030634, micro F1: 0.7199124726477026, macro F1: 0.7269771803688653 [INFO] Relation Linking P: 0.598832968636032, R: 0.5568927789934355, micro F1: 0.5675419401896419, macro F1: 0.5771018889916193 [INFO] Type Linking P: 0.5426695842450766, R: 0.5426695842450766, micro F1: 0.5426695842450766, macro F1: 0.5426695842450766 @@ -6410,8 +6410,8 @@ Question 457: Which producer of Purvis Eureka engine is also the owner of Ford T [INFO] Golden Answer: [http://dbpedia.org/resource/Ford_Motor_Company] [INFO] Predicted Answer: [1974] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 458, P: 0.522, R: 0.574, macro F1: 0.528, macro F1*: 0.547 -[INFO] Current time: 2309 ms, average: 2776.24 ms/ques, entity detection: 432.97 ms/ques (linking tool: 341.66 ms/ques), relation detection: 906.02 ms/ques (similarity: 707.25 ms/ques), EDG generation: 53.64 ms/ques +[INFO] Cumulative metrics, sample: 458, P: 0.522, R: 0.574, macro F1: 0.547 +[INFO] Current question consumed time: 1975 ms, average: 2552.06 ms/ques, entity detection: 393.91 ms/ques (linking tool: 312.97 ms/ques), relation detection: 849.34 ms/ques (similarity: 692.61 ms/ques), EDG generation: 53.89 ms/ques [INFO] Entity Linking P: 0.7489082969432315, R: 0.7063318777292577, micro F1: 0.7197962154294035, macro F1: 0.7269972514962801 [INFO] Relation Linking P: 0.5975254730713245, R: 0.5556768558951966, micro F1: 0.5663027656477431, macro F1: 0.5758418411990613 [INFO] Type Linking P: 0.5436681222707423, R: 0.5436681222707423, micro F1: 0.5436681222707423, macro F1: 0.5436681222707423 @@ -6424,8 +6424,8 @@ Question 458: What developed EMMS? [INFO] Golden Answer: [http://dbpedia.org/resource/GNU] [INFO] Predicted Answer: [http://dbpedia.org/resource/GNU] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 459, P: 0.523, R: 0.575, macro F1: 0.529, macro F1*: 0.548 -[INFO] Current time: 735 ms, average: 2771.79 ms/ques, entity detection: 432.28 ms/ques (linking tool: 341.01 ms/ques), relation detection: 904.47 ms/ques (similarity: 705.95 ms/ques), EDG generation: 53.53 ms/ques +[INFO] Cumulative metrics, sample: 459, P: 0.523, R: 0.575, macro F1: 0.548 +[INFO] Current question consumed time: 611 ms, average: 2547.83 ms/ques, entity detection: 393.26 ms/ques (linking tool: 312.37 ms/ques), relation detection: 847.88 ms/ques (similarity: 691.34 ms/ques), EDG generation: 53.78 ms/ques [INFO] Entity Linking P: 0.7494553376906318, R: 0.7069716775599129, micro F1: 0.7204066811909952, macro F1: 0.7275938880496934 [INFO] Relation Linking P: 0.5984023238925199, R: 0.5566448801742919, micro F1: 0.5672476397966587, macro F1: 0.5767687912777322 [INFO] Type Linking P: 0.5446623093681917, R: 0.5446623093681917, micro F1: 0.5446623093681917, macro F1: 0.5446623093681917 @@ -6438,8 +6438,8 @@ Question 459: Which music band made Take Me There, and to which Tedd Riley was a [INFO] Golden Answer: [http://dbpedia.org/resource/Blackstreet] [INFO] Predicted Answer: [http://dbpedia.org/resource/Rascal_Flatts] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 460, P: 0.522, R: 0.573, macro F1: 0.528, macro F1*: 0.546 -[INFO] Current time: 2918 ms, average: 2772.11 ms/ques, entity detection: 431.80 ms/ques (linking tool: 340.52 ms/ques), relation detection: 904.40 ms/ques (similarity: 706.47 ms/ques), EDG generation: 53.96 ms/ques +[INFO] Cumulative metrics, sample: 460, P: 0.522, R: 0.573, macro F1: 0.546 +[INFO] Current question consumed time: 2399 ms, average: 2547.50 ms/ques, entity detection: 392.81 ms/ques (linking tool: 311.93 ms/ques), relation detection: 847.72 ms/ques (similarity: 691.89 ms/ques), EDG generation: 53.77 ms/ques [INFO] Entity Linking P: 0.7478260869565218, R: 0.7054347826086956, micro F1: 0.7188405797101453, macro F1: 0.7260121622061071 [INFO] Relation Linking P: 0.5992753623188405, R: 0.5565217391304348, micro F1: 0.5674637681159413, macro F1: 0.5771078097314978 [INFO] Type Linking P: 0.5456521739130434, R: 0.5456521739130434, micro F1: 0.5456521739130434, macro F1: 0.5456521739130434 @@ -6452,8 +6452,8 @@ Question 460: In which time zone is Pong Pha? [INFO] Golden Answer: [http://dbpedia.org/resource/Thailand] [INFO] Predicted Answer: [http://dbpedia.org/resource/Thailand] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 461, P: 0.523, R: 0.574, macro F1: 0.529, macro F1*: 0.547 -[INFO] Current time: 912 ms, average: 2768.07 ms/ques, entity detection: 431.23 ms/ques (linking tool: 340.00 ms/ques), relation detection: 903.03 ms/ques (similarity: 705.49 ms/ques), EDG generation: 53.86 ms/ques +[INFO] Cumulative metrics, sample: 461, P: 0.523, R: 0.574, macro F1: 0.547 +[INFO] Current question consumed time: 790 ms, average: 2543.69 ms/ques, entity detection: 392.25 ms/ques (linking tool: 311.42 ms/ques), relation detection: 846.43 ms/ques (similarity: 690.94 ms/ques), EDG generation: 53.67 ms/ques [INFO] Entity Linking P: 0.7483731019522777, R: 0.7060737527114967, micro F1: 0.7194504699927696, macro F1: 0.726608336123688 [INFO] Relation Linking P: 0.6001446131597975, R: 0.5574837310195228, micro F1: 0.5684020245842365, macro F1: 0.5780281033681497 [INFO] Type Linking P: 0.5466377440347071, R: 0.5466377440347071, micro F1: 0.5466377440347071, macro F1: 0.5466377440347071 @@ -6466,8 +6466,8 @@ Question 461: How many fictional characters are there in a series in which Craig [INFO] Golden Answer: [19] [INFO] Predicted Answer: [86, 89, 80, 82] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 462, P: 0.522, R: 0.573, macro F1: 0.528, macro F1*: 0.546 -[INFO] Current time: 4274 ms, average: 2771.33 ms/ques, entity detection: 430.72 ms/ques (linking tool: 339.56 ms/ques), relation detection: 904.17 ms/ques (similarity: 707.12 ms/ques), EDG generation: 53.82 ms/ques +[INFO] Cumulative metrics, sample: 462, P: 0.522, R: 0.573, macro F1: 0.546 +[INFO] Current question consumed time: 3775 ms, average: 2546.36 ms/ques, entity detection: 391.74 ms/ques (linking tool: 310.99 ms/ques), relation detection: 847.45 ms/ques (similarity: 692.57 ms/ques), EDG generation: 53.63 ms/ques [INFO] Entity Linking P: 0.7489177489177489, R: 0.7067099567099567, micro F1: 0.7200577200577204, macro F1: 0.7272019182799851 [INFO] Relation Linking P: 0.6010101010101009, R: 0.5573593073593074, micro F1: 0.568614718614718, macro F1: 0.5783622585242019 [INFO] Type Linking P: 0.5454545454545454, R: 0.5454545454545454, micro F1: 0.5454545454545454, macro F1: 0.5454545454545454 @@ -6480,8 +6480,8 @@ Question 462: Which political party of Chandra Shekhar is also the political par [INFO] Golden Answer: [http://dbpedia.org/resource/Indian_National_Congress] [INFO] Predicted Answer: [http://dbpedia.org/resource/Indian_National_Congress] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 463, P: 0.523, R: 0.574, macro F1: 0.529, macro F1*: 0.547 -[INFO] Current time: 1986 ms, average: 2769.64 ms/ques, entity detection: 430.27 ms/ques (linking tool: 338.98 ms/ques), relation detection: 903.35 ms/ques (similarity: 706.83 ms/ques), EDG generation: 53.93 ms/ques +[INFO] Cumulative metrics, sample: 463, P: 0.523, R: 0.574, macro F1: 0.547 +[INFO] Current question consumed time: 1778 ms, average: 2544.70 ms/ques, entity detection: 391.31 ms/ques (linking tool: 310.46 ms/ques), relation detection: 846.68 ms/ques (similarity: 692.31 ms/ques), EDG generation: 53.74 ms/ques [INFO] Entity Linking P: 0.7494600431965442, R: 0.7062634989200864, micro F1: 0.7199424046076317, macro F1: 0.7272208727752818 [INFO] Relation Linking P: 0.6018718502519798, R: 0.5572354211663066, micro F1: 0.568826493880489, macro F1: 0.5786941765155682 [INFO] Type Linking P: 0.5442764578833693, R: 0.5442764578833693, micro F1: 0.5442764578833693, macro F1: 0.5442764578833693 @@ -6494,8 +6494,8 @@ Question 463: Who owns the airport in Paphos? [INFO] Golden Answer: [http://dbpedia.org/resource/Cyprus] [INFO] Predicted Answer: [http://dbpedia.org/resource/Paphos] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 464, P: 0.521, R: 0.573, macro F1: 0.528, macro F1*: 0.546 -[INFO] Current time: 2470 ms, average: 2768.99 ms/ques, entity detection: 429.81 ms/ques (linking tool: 338.54 ms/ques), relation detection: 903.95 ms/ques (similarity: 706.93 ms/ques), EDG generation: 53.97 ms/ques +[INFO] Cumulative metrics, sample: 464, P: 0.521, R: 0.573, macro F1: 0.546 +[INFO] Current question consumed time: 2315 ms, average: 2544.20 ms/ques, entity detection: 390.88 ms/ques (linking tool: 310.04 ms/ques), relation detection: 847.38 ms/ques (similarity: 692.38 ms/ques), EDG generation: 53.80 ms/ques [INFO] Entity Linking P: 0.7478448275862069, R: 0.7047413793103449, micro F1: 0.7183908045977014, macro F1: 0.7256535864115421 [INFO] Relation Linking P: 0.6027298850574712, R: 0.5571120689655172, micro F1: 0.5690373563218385, macro F1: 0.5790238784293205 [INFO] Type Linking P: 0.5431034482758621, R: 0.5431034482758621, micro F1: 0.5431034482758621, macro F1: 0.5431034482758621 @@ -6508,8 +6508,8 @@ Question 464: What is the baseball team whose club manager is Chip Hale? [INFO] Golden Answer: [http://dbpedia.org/resource/Arizona_Diamondbacks] [INFO] Predicted Answer: [http://dbpedia.org/resource/Arizona_Diamondbacks] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 465, P: 0.522, R: 0.574, macro F1: 0.529, macro F1*: 0.547 -[INFO] Current time: 1328 ms, average: 2765.89 ms/ques, entity detection: 429.35 ms/ques (linking tool: 338.10 ms/ques), relation detection: 902.92 ms/ques (similarity: 706.40 ms/ques), EDG generation: 53.90 ms/ques +[INFO] Cumulative metrics, sample: 465, P: 0.522, R: 0.574, macro F1: 0.547 +[INFO] Current question consumed time: 1177 ms, average: 2541.26 ms/ques, entity detection: 390.45 ms/ques (linking tool: 309.61 ms/ques), relation detection: 846.41 ms/ques (similarity: 691.86 ms/ques), EDG generation: 53.72 ms/ques [INFO] Entity Linking P: 0.7483870967741936, R: 0.7053763440860215, micro F1: 0.7189964157706097, macro F1: 0.7262454666921169 [INFO] Relation Linking P: 0.6035842293906809, R: 0.5580645161290323, micro F1: 0.5699641577060927, macro F1: 0.5799325178409691 [INFO] Type Linking P: 0.5419354838709678, R: 0.5419354838709678, micro F1: 0.5419354838709678, macro F1: 0.5419354838709678 @@ -6522,8 +6522,8 @@ Question 465: Which bacteria's taxonomic class is Bacilli? [INFO] Golden Answer: [http://dbpedia.org/resource/Sporosarcina_pasteurii, http://dbpedia.org/resource/Lactococcus_lactis, http://dbpedia.org/resource/Alicyclobacillus, http://dbpedia.org/resource/Bacillus_thuringiensis, http://dbpedia.org/resource/Methicillin-resistant_Staphylococcus_aureus, http://dbpedia.org/resource/Lactobacillus_acidophilus, http://dbpedia.org/resource/Macrococcus_brunensis, http://dbpedia.org/resource/Macrococcus_hajekii, http://dbpedia.org/resource/Macrococcus_lamae, http://dbpedia.org/resource/Sporosarcina_aquimarina, http://dbpedia.org/resource/Acetoanaerobium, http://dbpedia.org/resource/Aeribacillus, http://dbpedia.org/resource/Alkalibacter, http://dbpedia.org/resource/Lysinibacillus_fusiformis, http://dbpedia.org/resource/Bacillus_anthracis, http://dbpedia.org/resource/Brevibacillus_borstelensis, http://dbpedia.org/resource/ST8:USA300, http://dbpedia.org/resource/Bacillus_subtilis, http://dbpedia.org/resource/Aerococcus, http://dbpedia.org/resource/Sporosarcina_ureae, http://dbpedia.org/resource/Thermicanus_aegyptius, http://dbpedia.org/resource/Bacillus, http://dbpedia.org/resource/Bacillus_azotoformans, http://dbpedia.org/resource/Bacillus_circulans, http://dbpedia.org/resource/Bacillus_firmus, http://dbpedia.org/resource/Virgibacillus, http://dbpedia.org/resource/Alkalibacillus, http://dbpedia.org/resource/Filobacillus, http://dbpedia.org/resource/Gracilibacillus, http://dbpedia.org/resource/Sporosarcina, http://dbpedia.org/resource/Lactobacillus_pontis, http://dbpedia.org/resource/Abiotrophia, http://dbpedia.org/resource/Paenibacillus_alvei, http://dbpedia.org/resource/Macrococcus, http://dbpedia.org/resource/Nosocomiicoccus, http://dbpedia.org/resource/Salinicoccus, http://dbpedia.org/resource/Alicyclobacillus_acidiphilus, http://dbpedia.org/resource/Streptococcus_canis, http://dbpedia.org/resource/Streptococcus_oralis, http://dbpedia.org/resource/Streptococcus_pseudopneumoniae, http://dbpedia.org/resource/Agitococcus, http://dbpedia.org/resource/Salinicoccus_jeotgali, http://dbpedia.org/resource/Halobacillus_karajensis, http://dbpedia.org/resource/Halobacillus_litoralis, http://dbpedia.org/resource/Halobacillus_trueperi, http://dbpedia.org/resource/Amphibacillus_xylanus, http://dbpedia.org/resource/Jeotgalicoccus_aerolatus, http://dbpedia.org/resource/Jeotgalicoccus_coquinae, http://dbpedia.org/resource/Jeotgalicoccus_halophilus, http://dbpedia.org/resource/Jeotgalicoccus_halotolerans, http://dbpedia.org/resource/Jeotgalicoccus_huakuii, http://dbpedia.org/resource/Jeotgalicoccus_marinus, http://dbpedia.org/resource/Jeotgalicoccus_nanhaiensis, http://dbpedia.org/resource/Jeotgalicoccus_pinnipedialis, http://dbpedia.org/resource/Jeotgalicoccus_psychrophilus, http://dbpedia.org/resource/Cohnella_hongkongensis, http://dbpedia.org/resource/Cohnella_thermotolerans] [INFO] Predicted Answer: [http://dbpedia.org/resource/Sporosarcina_pasteurii, http://dbpedia.org/resource/Lactococcus_lactis, http://dbpedia.org/resource/Alicyclobacillus, http://dbpedia.org/resource/Bacillus_thuringiensis, http://dbpedia.org/resource/Methicillin-resistant_Staphylococcus_aureus, http://dbpedia.org/resource/Lactobacillus_acidophilus, http://dbpedia.org/resource/Macrococcus_brunensis, http://dbpedia.org/resource/Macrococcus_hajekii, http://dbpedia.org/resource/Macrococcus_lamae, http://dbpedia.org/resource/Sporosarcina_aquimarina, http://dbpedia.org/resource/Acetoanaerobium, http://dbpedia.org/resource/Aeribacillus, http://dbpedia.org/resource/Alkalibacter, http://dbpedia.org/resource/Lysinibacillus_fusiformis, http://dbpedia.org/resource/Bacillus_anthracis, http://dbpedia.org/resource/Brevibacillus_borstelensis, http://dbpedia.org/resource/ST8:USA300, http://dbpedia.org/resource/Bacillus_subtilis, http://dbpedia.org/resource/Aerococcus, http://dbpedia.org/resource/Sporosarcina_ureae, http://dbpedia.org/resource/Thermicanus_aegyptius, http://dbpedia.org/resource/Bacillus, http://dbpedia.org/resource/Bacillus_azotoformans, http://dbpedia.org/resource/Bacillus_circulans, http://dbpedia.org/resource/Bacillus_firmus, http://dbpedia.org/resource/Virgibacillus, http://dbpedia.org/resource/Alkalibacillus, http://dbpedia.org/resource/Filobacillus, http://dbpedia.org/resource/Gracilibacillus, http://dbpedia.org/resource/Sporosarcina, http://dbpedia.org/resource/Lactobacillus_pontis, http://dbpedia.org/resource/Abiotrophia, http://dbpedia.org/resource/Paenibacillus_alvei, http://dbpedia.org/resource/Macrococcus, http://dbpedia.org/resource/Nosocomiicoccus, http://dbpedia.org/resource/Salinicoccus, http://dbpedia.org/resource/Alicyclobacillus_acidiphilus, http://dbpedia.org/resource/Streptococcus_canis, http://dbpedia.org/resource/Streptococcus_oralis, http://dbpedia.org/resource/Streptococcus_pseudopneumoniae, http://dbpedia.org/resource/Agitococcus, http://dbpedia.org/resource/Salinicoccus_jeotgali, http://dbpedia.org/resource/Halobacillus_karajensis, http://dbpedia.org/resource/Halobacillus_litoralis, http://dbpedia.org/resource/Halobacillus_trueperi, http://dbpedia.org/resource/Amphibacillus_xylanus, http://dbpedia.org/resource/Jeotgalicoccus_aerolatus, http://dbpedia.org/resource/Jeotgalicoccus_coquinae, http://dbpedia.org/resource/Jeotgalicoccus_halophilus, http://dbpedia.org/resource/Jeotgalicoccus_halotolerans, http://dbpedia.org/resource/Jeotgalicoccus_huakuii, http://dbpedia.org/resource/Jeotgalicoccus_marinus, http://dbpedia.org/resource/Jeotgalicoccus_nanhaiensis, http://dbpedia.org/resource/Jeotgalicoccus_pinnipedialis, http://dbpedia.org/resource/Jeotgalicoccus_psychrophilus, http://dbpedia.org/resource/Cohnella_hongkongensis, http://dbpedia.org/resource/Cohnella_thermotolerans] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 466, P: 0.523, R: 0.575, macro F1: 0.530, macro F1*: 0.548 -[INFO] Current time: 500 ms, average: 2761.03 ms/ques, entity detection: 428.66 ms/ques (linking tool: 337.53 ms/ques), relation detection: 901.32 ms/ques (similarity: 705.20 ms/ques), EDG generation: 53.81 ms/ques +[INFO] Cumulative metrics, sample: 466, P: 0.523, R: 0.575, macro F1: 0.548 +[INFO] Current question consumed time: 456 ms, average: 2536.79 ms/ques, entity detection: 389.85 ms/ques (linking tool: 309.09 ms/ques), relation detection: 844.90 ms/ques (similarity: 690.71 ms/ques), EDG generation: 53.63 ms/ques [INFO] Entity Linking P: 0.7489270386266095, R: 0.7060085836909872, micro F1: 0.7195994277539345, macro F1: 0.7268347955992759 [INFO] Relation Linking P: 0.6044349070100142, R: 0.5590128755364807, micro F1: 0.5708869814020023, macro F1: 0.5808372331119904 [INFO] Type Linking P: 0.5407725321888412, R: 0.5407725321888412, micro F1: 0.5407725321888412, macro F1: 0.5407725321888412 @@ -6536,8 +6536,8 @@ Question 466: What honours did Reigh Count receive? [INFO] Golden Answer: [http://dbpedia.org/resource/Kentucky, http://dbpedia.org/resource/Florence,_Kentucky, http://dbpedia.org/resource/National_Museum_of_Racing_and_Hall_of_Fame] [INFO] Predicted Answer: [http://dbpedia.org/resource/Kentucky, http://dbpedia.org/resource/Florence,_Kentucky, http://dbpedia.org/resource/National_Museum_of_Racing_and_Hall_of_Fame, Reigh Count Drive in Florence, Kentucky@en] [INFO] P: 0.75, R: 1.0, F: 0.8571428571428571 -[INFO] Cumulative metrics, sample: 467, P: 0.524, R: 0.576, macro F1: 0.530, macro F1*: 0.549 -[INFO] Current time: 1552 ms, average: 2758.44 ms/ques, entity detection: 428.23 ms/ques (linking tool: 337.15 ms/ques), relation detection: 900.56 ms/ques (similarity: 704.92 ms/ques), EDG generation: 53.71 ms/ques +[INFO] Cumulative metrics, sample: 467, P: 0.524, R: 0.576, macro F1: 0.549 +[INFO] Current question consumed time: 1441 ms, average: 2534.44 ms/ques, entity detection: 389.48 ms/ques (linking tool: 308.76 ms/ques), relation detection: 844.16 ms/ques (similarity: 690.42 ms/ques), EDG generation: 53.55 ms/ques [INFO] Entity Linking P: 0.7494646680942184, R: 0.7066381156316917, micro F1: 0.7201998572448255, macro F1: 0.7274215896208591 [INFO] Relation Linking P: 0.6052819414703782, R: 0.5599571734475375, micro F1: 0.5718058529621693, macro F1: 0.581738049719463 [INFO] Type Linking P: 0.5417558886509636, R: 0.5417558886509636, micro F1: 0.5417558886509636, macro F1: 0.5417558886509636 @@ -6550,8 +6550,8 @@ Question 467: What are some products of the the company whose subsidiary is Sony [INFO] Golden Answer: [http://dbpedia.org/resource/Philips_CD-i, http://dbpedia.org/resource/Sony_NW-A810, http://dbpedia.org/resource/PlayStation_(console), http://dbpedia.org/resource/PlayStation_2, http://dbpedia.org/resource/PlayStation_3, http://dbpedia.org/resource/Sony_Xperia_E, http://dbpedia.org/resource/Walkman_X_Series, http://dbpedia.org/resource/Sony_CLIÉ_PEG-NR70V, http://dbpedia.org/resource/Sony_CLIÉ_PEG-SJ22, http://dbpedia.org/resource/Sony_CLIÉ_PEG-TJ35, http://dbpedia.org/resource/Sony_CLIÉ_PEG-TJ37, http://dbpedia.org/resource/PlayStation_4, http://dbpedia.org/resource/Walkman, http://dbpedia.org/resource/PlayStation_Portable, http://dbpedia.org/resource/PlayStation_Vita, http://dbpedia.org/resource/Sony_Reader, http://dbpedia.org/resource/DualShock, http://dbpedia.org/resource/Dual_Analog_Controller, http://dbpedia.org/resource/PSX_(video_game_console), http://dbpedia.org/resource/PlayStation_Controller, http://dbpedia.org/resource/PlayStation_Mouse, http://dbpedia.org/resource/PlayStation_models, http://dbpedia.org/resource/Sixaxis, http://dbpedia.org/resource/Mylo_(Sony), http://dbpedia.org/resource/Sony_SmartWatch, http://dbpedia.org/resource/Sony_NW-A800, http://dbpedia.org/resource/Walkman_S_Series, http://dbpedia.org/resource/PocketStation, http://dbpedia.org/resource/SNES-CD, http://dbpedia.org/resource/Sony_Tablet, http://dbpedia.org/resource/PlayStation_Analog_Joystick, http://dbpedia.org/resource/Sony_Xperia_Tablet_Z, http://dbpedia.org/resource/Sony_Dash, http://dbpedia.org/resource/CLIÉ, http://dbpedia.org/resource/Magic_Link, http://dbpedia.org/resource/Yahoo!_Connected_TV, http://dbpedia.org/resource/Sony_Xperia_Tablet_S, http://dbpedia.org/resource/Sony_Walkman_Z_Series, http://dbpedia.org/resource/Sony_Xperia_Z2_Tablet, http://dbpedia.org/resource/Sony_Xperia_Z3_Tablet_Compact, http://dbpedia.org/resource/Sony_Xperia_Z4_Tablet] [INFO] Predicted Answer: [] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 468, P: 0.523, R: 0.574, macro F1: 0.529, macro F1*: 0.547 -[INFO] Current time: 4320 ms, average: 2761.78 ms/ques, entity detection: 428.00 ms/ques (linking tool: 336.98 ms/ques), relation detection: 902.74 ms/ques (similarity: 706.50 ms/ques), EDG generation: 53.70 ms/ques +[INFO] Cumulative metrics, sample: 468, P: 0.523, R: 0.574, macro F1: 0.547 +[INFO] Current question consumed time: 4027 ms, average: 2537.63 ms/ques, entity detection: 389.22 ms/ques (linking tool: 308.58 ms/ques), relation detection: 846.44 ms/ques (similarity: 692.35 ms/ques), EDG generation: 53.55 ms/ques [INFO] Entity Linking P: 0.7478632478632479, R: 0.7051282051282052, micro F1: 0.718660968660969, macro F1: 0.7258672699849171 [INFO] Relation Linking P: 0.6039886039886039, R: 0.5587606837606838, micro F1: 0.5705840455840451, macro F1: 0.580495019698695 [INFO] Type Linking P: 0.5427350427350427, R: 0.5427350427350427, micro F1: 0.5427350427350427, macro F1: 0.5427350427350427 @@ -6564,8 +6564,8 @@ Question 468: Who built the stadiums where 2014 FIFA under 17 woman's world cup [INFO] Golden Answer: [http://dbpedia.org/resource/China] [INFO] Predicted Answer: [http://dbpedia.org/resource/FIFA] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 469, P: 0.522, R: 0.573, macro F1: 0.528, macro F1*: 0.546 -[INFO] Current time: 3129 ms, average: 2762.56 ms/ques, entity detection: 428.11 ms/ques (linking tool: 337.05 ms/ques), relation detection: 902.73 ms/ques (similarity: 706.80 ms/ques), EDG generation: 53.64 ms/ques +[INFO] Cumulative metrics, sample: 469, P: 0.522, R: 0.573, macro F1: 0.546 +[INFO] Current question consumed time: 2840 ms, average: 2538.28 ms/ques, entity detection: 389.23 ms/ques (linking tool: 308.56 ms/ques), relation detection: 846.46 ms/ques (similarity: 692.66 ms/ques), EDG generation: 53.50 ms/ques [INFO] Entity Linking P: 0.746268656716418, R: 0.7036247334754797, micro F1: 0.7171286425017771, macro F1: 0.7243195785776997 [INFO] Relation Linking P: 0.6027007818052593, R: 0.5575692963752665, micro F1: 0.5693674484719256, macro F1: 0.5792572904456061 [INFO] Type Linking P: 0.5437100213219617, R: 0.5437100213219617, micro F1: 0.5437100213219617, macro F1: 0.5437100213219617 @@ -6578,8 +6578,8 @@ Question 469: Which area operataed by the South Side Elevated Railroad is also t [INFO] Golden Answer: [http://dbpedia.org/resource/Chicago] [INFO] Predicted Answer: [http://dbpedia.org/resource/Chicago] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 470, P: 0.523, R: 0.574, macro F1: 0.529, macro F1*: 0.547 -[INFO] Current time: 1699 ms, average: 2760.30 ms/ques, entity detection: 428.15 ms/ques (linking tool: 336.78 ms/ques), relation detection: 901.44 ms/ques (similarity: 705.71 ms/ques), EDG generation: 53.78 ms/ques +[INFO] Cumulative metrics, sample: 470, P: 0.523, R: 0.574, macro F1: 0.547 +[INFO] Current question consumed time: 1519 ms, average: 2536.11 ms/ques, entity detection: 389.24 ms/ques (linking tool: 308.27 ms/ques), relation detection: 845.21 ms/ques (similarity: 691.60 ms/ques), EDG generation: 53.65 ms/ques [INFO] Entity Linking P: 0.7468085106382979, R: 0.7042553191489361, micro F1: 0.7177304964539011, macro F1: 0.7249079678043302 [INFO] Relation Linking P: 0.6014184397163119, R: 0.5563829787234043, micro F1: 0.5681560283687938, macro F1: 0.5780248281255089 [INFO] Type Linking P: 0.5446808510638298, R: 0.5446808510638298, micro F1: 0.5446808510638298, macro F1: 0.5446808510638298 @@ -6592,8 +6592,8 @@ Question 470: What is the television show whose related with Doctor Who Confide [INFO] Golden Answer: [http://dbpedia.org/resource/Doctor_Who, http://dbpedia.org/resource/Torchwood_Declassified] [INFO] Predicted Answer: [http://dbpedia.org/resource/Doctor_Who, http://dbpedia.org/resource/Torchwood_Declassified] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 471, P: 0.524, R: 0.575, macro F1: 0.530, macro F1*: 0.548 -[INFO] Current time: 1215 ms, average: 2757.02 ms/ques, entity detection: 427.92 ms/ques (linking tool: 336.67 ms/ques), relation detection: 900.17 ms/ques (similarity: 704.91 ms/ques), EDG generation: 53.69 ms/ques +[INFO] Cumulative metrics, sample: 471, P: 0.524, R: 0.575, macro F1: 0.548 +[INFO] Current question consumed time: 1112 ms, average: 2533.08 ms/ques, entity detection: 389.01 ms/ques (linking tool: 308.14 ms/ques), relation detection: 844.00 ms/ques (similarity: 690.82 ms/ques), EDG generation: 53.53 ms/ques [INFO] Entity Linking P: 0.7473460721868365, R: 0.7048832271762208, micro F1: 0.7183297947629161, macro F1: 0.725493847853888 [INFO] Relation Linking P: 0.6022646850672327, R: 0.5573248407643312, micro F1: 0.5690728945506011, macro F1: 0.5789239420084782 [INFO] Type Linking P: 0.5435244161358811, R: 0.5435244161358811, micro F1: 0.5435244161358811, macro F1: 0.5435244161358811 @@ -6606,8 +6606,8 @@ Question 471: In which city wasThe Triple Door (The Courage album) performed li [INFO] Golden Answer: [http://dbpedia.org/resource/Seattle] [INFO] Predicted Answer: [http://dbpedia.org/resource/The_Courage] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 472, P: 0.523, R: 0.574, macro F1: 0.529, macro F1*: 0.547 -[INFO] Current time: 1388 ms, average: 2754.12 ms/ques, entity detection: 427.44 ms/ques (linking tool: 336.28 ms/ques), relation detection: 899.12 ms/ques (similarity: 704.25 ms/ques), EDG generation: 53.59 ms/ques +[INFO] Cumulative metrics, sample: 472, P: 0.523, R: 0.574, macro F1: 0.547 +[INFO] Current question consumed time: 1216 ms, average: 2530.29 ms/ques, entity detection: 388.53 ms/ques (linking tool: 307.76 ms/ques), relation detection: 842.97 ms/ques (similarity: 690.19 ms/ques), EDG generation: 53.43 ms/ques [INFO] Entity Linking P: 0.7478813559322034, R: 0.7055084745762712, micro F1: 0.7189265536723167, macro F1: 0.7260772347679992 [INFO] Relation Linking P: 0.6009887005649717, R: 0.5561440677966102, micro F1: 0.5678672316384176, macro F1: 0.5776974082330366 [INFO] Type Linking P: 0.5444915254237288, R: 0.5444915254237288, micro F1: 0.5444915254237288, macro F1: 0.5444915254237288 @@ -6620,8 +6620,8 @@ Question 472: Which nearest city of Jurgens Site is the death plce of Jimmie Heu [INFO] Golden Answer: [http://dbpedia.org/resource/Colorado] [INFO] Predicted Answer: [http://dbpedia.org/resource/Colorado] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 473, P: 0.524, R: 0.575, macro F1: 0.530, macro F1*: 0.548 -[INFO] Current time: 1430 ms, average: 2751.32 ms/ques, entity detection: 427.36 ms/ques (linking tool: 335.97 ms/ques), relation detection: 897.83 ms/ques (similarity: 703.14 ms/ques), EDG generation: 53.55 ms/ques +[INFO] Cumulative metrics, sample: 473, P: 0.524, R: 0.575, macro F1: 0.548 +[INFO] Current question consumed time: 1287 ms, average: 2527.67 ms/ques, entity detection: 388.41 ms/ques (linking tool: 307.47 ms/ques), relation detection: 841.76 ms/ques (similarity: 689.10 ms/ques), EDG generation: 53.39 ms/ques [INFO] Entity Linking P: 0.7484143763213531, R: 0.7061310782241015, micro F1: 0.7195207892882315, macro F1: 0.7266581444515463 [INFO] Relation Linking P: 0.6018322762508809, R: 0.5570824524312896, micro F1: 0.5687808315715287, macro F1: 0.5785933893296702 [INFO] Type Linking P: 0.5454545454545454, R: 0.5454545454545454, micro F1: 0.5454545454545454, macro F1: 0.5454545454545454 @@ -6634,8 +6634,8 @@ Question 473: List the battles fought by Ali Habib Mahmud ? [INFO] Golden Answer: [http://dbpedia.org/resource/Gulf_War, http://dbpedia.org/resource/Lebanese_Civil_War, http://dbpedia.org/resource/Yom_Kippur_War] [INFO] Predicted Answer: [http://dbpedia.org/resource/Gulf_War, http://dbpedia.org/resource/Lebanese_Civil_War, http://dbpedia.org/resource/Yom_Kippur_War] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 474, P: 0.525, R: 0.576, macro F1: 0.531, macro F1*: 0.549 -[INFO] Current time: 839 ms, average: 2747.28 ms/ques, entity detection: 426.79 ms/ques (linking tool: 335.43 ms/ques), relation detection: 896.60 ms/ques (similarity: 702.07 ms/ques), EDG generation: 53.46 ms/ques +[INFO] Cumulative metrics, sample: 474, P: 0.525, R: 0.576, macro F1: 0.549 +[INFO] Current question consumed time: 744 ms, average: 2523.90 ms/ques, entity detection: 387.88 ms/ques (linking tool: 306.97 ms/ques), relation detection: 840.59 ms/ques (similarity: 688.05 ms/ques), EDG generation: 53.30 ms/ques [INFO] Entity Linking P: 0.7489451476793249, R: 0.7067510548523207, micro F1: 0.7201125175808724, macro F1: 0.7272365926741271 [INFO] Relation Linking P: 0.60056258790436, R: 0.5559071729957806, micro F1: 0.5675808720112512, macro F1: 0.5773727281707468 [INFO] Type Linking P: 0.5464135021097046, R: 0.5464135021097046, micro F1: 0.5464135021097046, macro F1: 0.5464135021097046 @@ -6648,8 +6648,8 @@ Question 474: How many companies were started in the states? [INFO] Golden Answer: [1128] [INFO] Predicted Answer: [] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 475, P: 0.524, R: 0.574, macro F1: 0.530, macro F1*: 0.548 -[INFO] Current time: 330 ms, average: 2742.20 ms/ques, entity detection: 425.89 ms/ques (linking tool: 334.73 ms/ques), relation detection: 895.18 ms/ques (similarity: 700.91 ms/ques), EDG generation: 53.37 ms/ques +[INFO] Cumulative metrics, sample: 475, P: 0.524, R: 0.574, macro F1: 0.548 +[INFO] Current question consumed time: 312 ms, average: 2519.25 ms/ques, entity detection: 387.06 ms/ques (linking tool: 306.32 ms/ques), relation detection: 839.25 ms/ques (similarity: 686.94 ms/ques), EDG generation: 53.18 ms/ques [INFO] Entity Linking P: 0.7473684210526316, R: 0.7052631578947368, micro F1: 0.7185964912280705, macro F1: 0.7257055682684973 [INFO] Relation Linking P: 0.599298245614035, R: 0.5547368421052632, micro F1: 0.5663859649122801, macro F1: 0.5761572066377558 [INFO] Type Linking P: 0.5452631578947369, R: 0.5452631578947369, micro F1: 0.5452631578947369, macro F1: 0.5452631578947369 @@ -6662,8 +6662,8 @@ Question 475: Is Charles Drummond Ellis a doctoral student of James Chadwick? [INFO] Golden Answer: [true] [INFO] Predicted Answer: [true] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 476, P: 0.525, R: 0.575, macro F1: 0.531, macro F1*: 0.549 -[INFO] Current time: 912 ms, average: 2738.35 ms/ques, entity detection: 425.78 ms/ques (linking tool: 334.46 ms/ques), relation detection: 893.49 ms/ques (similarity: 699.57 ms/ques), EDG generation: 53.30 ms/ques +[INFO] Cumulative metrics, sample: 476, P: 0.525, R: 0.575, macro F1: 0.549 +[INFO] Current question consumed time: 781 ms, average: 2515.59 ms/ques, entity detection: 386.89 ms/ques (linking tool: 306.07 ms/ques), relation detection: 837.67 ms/ques (similarity: 685.62 ms/ques), EDG generation: 53.11 ms/ques [INFO] Entity Linking P: 0.7478991596638656, R: 0.7058823529411765, micro F1: 0.7191876750700283, macro F1: 0.7262835770146209 [INFO] Relation Linking P: 0.6001400560224088, R: 0.555672268907563, micro F1: 0.5672969187675064, macro F1: 0.5770507536549913 [INFO] Type Linking P: 0.5462184873949579, R: 0.5462184873949579, micro F1: 0.5462184873949579, macro F1: 0.5462184873949579 @@ -6676,8 +6676,8 @@ Question 476: How many movies are distributed by Cinema International Corporatio [INFO] Golden Answer: [22] [INFO] Predicted Answer: [22, 23] [INFO] P: 0.5, R: 1.0, F: 0.6666666666666666 -[INFO] Cumulative metrics, sample: 477, P: 0.525, R: 0.576, macro F1: 0.531, macro F1*: 0.549 -[INFO] Current time: 692 ms, average: 2734.06 ms/ques, entity detection: 425.25 ms/ques (linking tool: 333.96 ms/ques), relation detection: 891.85 ms/ques (similarity: 698.29 ms/ques), EDG generation: 53.21 ms/ques +[INFO] Cumulative metrics, sample: 477, P: 0.525, R: 0.576, macro F1: 0.549 +[INFO] Current question consumed time: 595 ms, average: 2511.57 ms/ques, entity detection: 386.41 ms/ques (linking tool: 305.62 ms/ques), relation detection: 836.14 ms/ques (similarity: 684.35 ms/ques), EDG generation: 53.01 ms/ques [INFO] Entity Linking P: 0.7484276729559748, R: 0.7064989517819706, micro F1: 0.719776380153739, macro F1: 0.7268591521215086 [INFO] Relation Linking P: 0.6009783368273933, R: 0.5566037735849056, micro F1: 0.5682040531097129, macro F1: 0.5779405315822744 [INFO] Type Linking P: 0.5450733752620545, R: 0.5450733752620545, micro F1: 0.5450733752620545, macro F1: 0.5450733752620545 @@ -6690,8 +6690,8 @@ Question 477: Name the local authority of Trinity School, Brentwood ? [INFO] Golden Answer: [http://dbpedia.org/resource/Essex] [INFO] Predicted Answer: [1738] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 478, P: 0.523, R: 0.575, macro F1: 0.530, macro F1*: 0.548 -[INFO] Current time: 1208 ms, average: 2730.87 ms/ques, entity detection: 424.92 ms/ques (linking tool: 333.66 ms/ques), relation detection: 890.59 ms/ques (similarity: 697.21 ms/ques), EDG generation: 53.14 ms/ques +[INFO] Cumulative metrics, sample: 478, P: 0.523, R: 0.575, macro F1: 0.548 +[INFO] Current question consumed time: 1058 ms, average: 2508.53 ms/ques, entity detection: 386.09 ms/ques (linking tool: 305.33 ms/ques), relation detection: 834.92 ms/ques (similarity: 683.28 ms/ques), EDG generation: 52.97 ms/ques [INFO] Entity Linking P: 0.7468619246861925, R: 0.7050209205020921, micro F1: 0.7182705718270574, macro F1: 0.7253385262802504 [INFO] Relation Linking P: 0.5997210599721059, R: 0.555439330543933, micro F1: 0.5670153417015337, macro F1: 0.5767314509722696 [INFO] Type Linking P: 0.5460251046025104, R: 0.5460251046025104, micro F1: 0.5460251046025104, macro F1: 0.5460251046025104 @@ -6704,8 +6704,8 @@ Question 478: Name the president of Wyche Fowler ? [INFO] Golden Answer: [http://dbpedia.org/resource/Bill_Clinton, http://dbpedia.org/resource/George_W._Bush] [INFO] Predicted Answer: [http://dbpedia.org/resource/Bill_Clinton, http://dbpedia.org/resource/George_W._Bush] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 479, P: 0.524, R: 0.576, macro F1: 0.531, macro F1*: 0.549 -[INFO] Current time: 846 ms, average: 2726.93 ms/ques, entity detection: 424.34 ms/ques (linking tool: 333.15 ms/ques), relation detection: 889.39 ms/ques (similarity: 696.17 ms/ques), EDG generation: 53.05 ms/ques +[INFO] Cumulative metrics, sample: 479, P: 0.524, R: 0.576, macro F1: 0.549 +[INFO] Current question consumed time: 682 ms, average: 2504.71 ms/ques, entity detection: 385.53 ms/ques (linking tool: 304.84 ms/ques), relation detection: 833.76 ms/ques (similarity: 682.27 ms/ques), EDG generation: 52.88 ms/ques [INFO] Entity Linking P: 0.7473903966597077, R: 0.7056367432150313, micro F1: 0.7188587334725125, macro F1: 0.7259136611235093 [INFO] Relation Linking P: 0.6005567153792623, R: 0.5563674321503131, micro F1: 0.5679192762700064, macro F1: 0.5776181581301898 [INFO] Type Linking P: 0.5469728601252609, R: 0.5469728601252609, micro F1: 0.5469728601252609, macro F1: 0.5469728601252609 @@ -6718,8 +6718,8 @@ Question 479: In which fields have people born in Lausanne worked? [INFO] Golden Answer: [http://dbpedia.org/resource/Chinese_poetry, http://dbpedia.org/resource/Buddhism] [INFO] Predicted Answer: [http://dbpedia.org/resource/Caricature, http://dbpedia.org/resource/Graphic_arts, http://dbpedia.org/resource/Illustration, http://dbpedia.org/resource/Ethnography, http://dbpedia.org/resource/Painting, http://dbpedia.org/resource/Neuroscience, http://dbpedia.org/resource/Early_Christianity, http://dbpedia.org/resource/New_Testament, http://dbpedia.org/resource/Physics, http://dbpedia.org/resource/Chinese_poetry, http://dbpedia.org/resource/Buddhism, Painting, graphic arts, illustration, caricature@en, decorative art@en] [INFO] P: 0.15384615384615385, R: 1.0, F: 0.2666666666666667 -[INFO] Cumulative metrics, sample: 480, P: 0.524, R: 0.577, macro F1: 0.530, macro F1*: 0.549 -[INFO] Current time: 7270 ms, average: 2736.40 ms/ques, entity detection: 424.27 ms/ques (linking tool: 333.06 ms/ques), relation detection: 894.72 ms/ques (similarity: 701.52 ms/ques), EDG generation: 52.96 ms/ques +[INFO] Cumulative metrics, sample: 480, P: 0.524, R: 0.577, macro F1: 0.549 +[INFO] Current question consumed time: 6335 ms, average: 2512.69 ms/ques, entity detection: 385.48 ms/ques (linking tool: 304.77 ms/ques), relation detection: 838.43 ms/ques (similarity: 687.62 ms/ques), EDG generation: 52.80 ms/ques [INFO] Entity Linking P: 0.7479166666666667, R: 0.70625, micro F1: 0.7194444444444448, macro F1: 0.7264863896848137 [INFO] Relation Linking P: 0.5993055555555554, R: 0.5552083333333333, micro F1: 0.5667361111111106, macro F1: 0.5764147869674183 [INFO] Type Linking P: 0.5479166666666667, R: 0.5479166666666667, micro F1: 0.5479166666666667, macro F1: 0.5479166666666667 @@ -6732,8 +6732,8 @@ Question 480: Which Texas based company was founded by Jim Harris ? [INFO] Golden Answer: [http://dbpedia.org/resource/Compaq] [INFO] Predicted Answer: [http://dbpedia.org/resource/Compaq] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 481, P: 0.525, R: 0.578, macro F1: 0.531, macro F1*: 0.550 -[INFO] Current time: 1237 ms, average: 2733.28 ms/ques, entity detection: 423.89 ms/ques (linking tool: 332.64 ms/ques), relation detection: 893.37 ms/ques (similarity: 700.36 ms/ques), EDG generation: 52.87 ms/ques +[INFO] Cumulative metrics, sample: 481, P: 0.525, R: 0.578, macro F1: 0.550 +[INFO] Current question consumed time: 988 ms, average: 2509.52 ms/ques, entity detection: 385.10 ms/ques (linking tool: 304.36 ms/ques), relation detection: 837.14 ms/ques (similarity: 686.48 ms/ques), EDG generation: 52.71 ms/ques [INFO] Entity Linking P: 0.7484407484407485, R: 0.7058212058212058, micro F1: 0.7193347193347197, macro F1: 0.7265064591726493 [INFO] Relation Linking P: 0.598059598059598, R: 0.5540540540540541, micro F1: 0.565557865557865, macro F1: 0.5752164194269458 [INFO] Type Linking P: 0.5467775467775468, R: 0.5467775467775468, micro F1: 0.5467775467775468, macro F1: 0.5467775467775468 @@ -6746,8 +6746,8 @@ Question 481: Name a show hosted by Brian Dunkleman and judged by Harry Connick, [INFO] Golden Answer: [http://dbpedia.org/resource/American_Idol] [INFO] Predicted Answer: [http://dbpedia.org/resource/American_Idol_(season_1)] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 482, P: 0.524, R: 0.576, macro F1: 0.530, macro F1*: 0.549 -[INFO] Current time: 2554 ms, average: 2732.91 ms/ques, entity detection: 423.75 ms/ques (linking tool: 332.45 ms/ques), relation detection: 893.18 ms/ques (similarity: 700.77 ms/ques), EDG generation: 52.83 ms/ques +[INFO] Cumulative metrics, sample: 482, P: 0.524, R: 0.576, macro F1: 0.549 +[INFO] Current question consumed time: 2273 ms, average: 2509.03 ms/ques, entity detection: 384.91 ms/ques (linking tool: 304.17 ms/ques), relation detection: 836.96 ms/ques (similarity: 686.85 ms/ques), EDG generation: 52.68 ms/ques [INFO] Entity Linking P: 0.7489626556016598, R: 0.7053941908713693, micro F1: 0.7192254495159064, macro F1: 0.7265258285436927 [INFO] Relation Linking P: 0.5968188105117564, R: 0.5529045643153527, micro F1: 0.5643845089903176, macro F1: 0.5740230243658939 [INFO] Type Linking P: 0.5456431535269709, R: 0.5456431535269709, micro F1: 0.5456431535269709, macro F1: 0.5456431535269709 @@ -6760,8 +6760,8 @@ Question 482: To which educational institute does Cornell University Press belon [INFO] Golden Answer: [http://dbpedia.org/resource/Cornell_University] [INFO] Predicted Answer: [http://dbpedia.org/resource/Cornell_University] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 483, P: 0.525, R: 0.577, macro F1: 0.531, macro F1*: 0.550 -[INFO] Current time: 1312 ms, average: 2729.97 ms/ques, entity detection: 423.70 ms/ques (linking tool: 332.48 ms/ques), relation detection: 891.74 ms/ques (similarity: 699.73 ms/ques), EDG generation: 52.75 ms/ques +[INFO] Cumulative metrics, sample: 483, P: 0.525, R: 0.577, macro F1: 0.550 +[INFO] Current question consumed time: 1283 ms, average: 2506.49 ms/ques, entity detection: 384.91 ms/ques (linking tool: 304.24 ms/ques), relation detection: 835.63 ms/ques (similarity: 685.87 ms/ques), EDG generation: 52.59 ms/ques [INFO] Entity Linking P: 0.7494824016563147, R: 0.7060041407867494, micro F1: 0.7198067632850246, macro F1: 0.7270938804119582 [INFO] Relation Linking P: 0.5976535541752932, R: 0.5538302277432712, micro F1: 0.5652864044168386, macro F1: 0.5749079738995179 [INFO] Type Linking P: 0.5445134575569358, R: 0.5445134575569358, micro F1: 0.5445134575569358, macro F1: 0.5445134575569358 @@ -6774,8 +6774,8 @@ Question 483: In which country's cuisine, ham is an important part? [INFO] Golden Answer: [http://dbpedia.org/resource/Great_Britain, http://dbpedia.org/resource/North_America, Spain and Portugal@en, http://dbpedia.org/resource/Spain, http://dbpedia.org/resource/Germany, http://dbpedia.org/resource/Bulgaria] [INFO] Predicted Answer: [http://dbpedia.org/resource/Ham_salad, http://dbpedia.org/resource/Jamón_ibérico, http://dbpedia.org/resource/Jamón_serrano, http://dbpedia.org/resource/Ham_and_egg_bun, http://dbpedia.org/resource/Toast_Hawaii, http://dbpedia.org/resource/Elenski_but, http://dbpedia.org/resource/Ham_and_cheese_sandwich] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 484, P: 0.523, R: 0.576, macro F1: 0.530, macro F1*: 0.548 -[INFO] Current time: 839 ms, average: 2726.06 ms/ques, entity detection: 423.01 ms/ques (linking tool: 331.88 ms/ques), relation detection: 890.38 ms/ques (similarity: 698.76 ms/ques), EDG generation: 52.67 ms/ques +[INFO] Cumulative metrics, sample: 484, P: 0.523, R: 0.576, macro F1: 0.548 +[INFO] Current question consumed time: 787 ms, average: 2502.94 ms/ques, entity detection: 384.29 ms/ques (linking tool: 303.69 ms/ques), relation detection: 834.38 ms/ques (similarity: 684.89 ms/ques), EDG generation: 52.51 ms/ques [INFO] Entity Linking P: 0.75, R: 0.7066115702479339, micro F1: 0.7203856749311298, macro F1: 0.727659574468085 [INFO] Relation Linking P: 0.5984848484848484, R: 0.5537190082644629, micro F1: 0.5654958677685945, macro F1: 0.5752322990816715 [INFO] Type Linking P: 0.5454545454545454, R: 0.5454545454545454, micro F1: 0.5454545454545454, macro F1: 0.5454545454545454 @@ -6788,8 +6788,8 @@ Question 484: List the softwares which runs on Microsoft Windows and has license [INFO] Golden Answer: [http://dbpedia.org/resource/Evernote, http://dbpedia.org/resource/Microsoft_Office, http://dbpedia.org/resource/Adobe_Character_Animator, http://dbpedia.org/resource/Adobe_After_Effects, http://dbpedia.org/resource/Adobe_Creative_Cloud, http://dbpedia.org/resource/Adobe_Marketing_Cloud, http://dbpedia.org/resource/Adobe_Photoshop, http://dbpedia.org/resource/Adobe_Presenter, http://dbpedia.org/resource/Inquisiq_R3] [INFO] Predicted Answer: [http://dbpedia.org/resource/Evernote, http://dbpedia.org/resource/Microsoft_Office, http://dbpedia.org/resource/Adobe_Character_Animator, http://dbpedia.org/resource/Adobe_After_Effects, http://dbpedia.org/resource/Adobe_Creative_Cloud, http://dbpedia.org/resource/Adobe_Marketing_Cloud, http://dbpedia.org/resource/Adobe_Photoshop, http://dbpedia.org/resource/Adobe_Presenter, http://dbpedia.org/resource/Inquisiq_R3, http://dbpedia.org/resource/LiveChat, http://dbpedia.org/resource/OnlyOffice] [INFO] P: 0.8181818181818182, R: 1.0, F: 0.9 -[INFO] Cumulative metrics, sample: 485, P: 0.524, R: 0.577, macro F1: 0.531, macro F1*: 0.549 -[INFO] Current time: 5505 ms, average: 2731.79 ms/ques, entity detection: 425.24 ms/ques (linking tool: 334.12 ms/ques), relation detection: 891.39 ms/ques (similarity: 699.38 ms/ques), EDG generation: 52.63 ms/ques +[INFO] Cumulative metrics, sample: 485, P: 0.524, R: 0.577, macro F1: 0.549 +[INFO] Current question consumed time: 4962 ms, average: 2508.01 ms/ques, entity detection: 386.30 ms/ques (linking tool: 305.71 ms/ques), relation detection: 835.14 ms/ques (similarity: 685.51 ms/ques), EDG generation: 52.47 ms/ques [INFO] Entity Linking P: 0.7505154639175258, R: 0.7072164948453609, micro F1: 0.7209621993127152, macro F1: 0.728222925385322 [INFO] Relation Linking P: 0.5993127147766322, R: 0.554639175257732, micro F1: 0.5663917525773191, macro F1: 0.5761112100354285 [INFO] Type Linking P: 0.5443298969072164, R: 0.5443298969072164, micro F1: 0.5443298969072164, macro F1: 0.5443298969072164 @@ -6802,8 +6802,8 @@ Question 485: Which ppolitical party of Yashpal Kapur is also the owner of Natio [INFO] Golden Answer: [http://dbpedia.org/resource/Indian_National_Congress] [INFO] Predicted Answer: [http://dbpedia.org/resource/Indian_National_Congress] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 486, P: 0.525, R: 0.578, macro F1: 0.532, macro F1*: 0.550 -[INFO] Current time: 1182 ms, average: 2728.60 ms/ques, entity detection: 425.12 ms/ques (linking tool: 333.70 ms/ques), relation detection: 889.91 ms/ques (similarity: 698.17 ms/ques), EDG generation: 52.73 ms/ques +[INFO] Cumulative metrics, sample: 486, P: 0.525, R: 0.578, macro F1: 0.550 +[INFO] Current question consumed time: 1055 ms, average: 2505.02 ms/ques, entity detection: 386.14 ms/ques (linking tool: 305.33 ms/ques), relation detection: 833.77 ms/ques (similarity: 684.30 ms/ques), EDG generation: 52.59 ms/ques [INFO] Entity Linking P: 0.7510288065843621, R: 0.7078189300411523, micro F1: 0.7215363511659811, macro F1: 0.7287839477151496 [INFO] Relation Linking P: 0.6001371742112482, R: 0.5555555555555556, micro F1: 0.5672839506172834, macro F1: 0.5769864820309925 [INFO] Type Linking P: 0.5452674897119342, R: 0.5452674897119342, micro F1: 0.5452674897119342, macro F1: 0.5452674897119342 @@ -6816,8 +6816,8 @@ Question 486: Who all have been presenters on shows by the NBC? [INFO] Golden Answer: [http://dbpedia.org/resource/Joseph_Cotten, http://dbpedia.org/resource/Jack_Clark_(television_personality), http://dbpedia.org/resource/Jay_Jackson, http://dbpedia.org/resource/Bill_Slater_(broadcaster), Dick Wilson@en, http://dbpedia.org/resource/Helen_O'Connell, http://dbpedia.org/resource/Dean_Miller_(broadcaster), http://dbpedia.org/resource/Jack_Linkletter, Joanne Jordan@en, http://dbpedia.org/resource/Barbara_Stanwyck, http://dbpedia.org/resource/Bob_Hope, http://dbpedia.org/resource/Anthony_Clark_(actor), http://dbpedia.org/resource/Anthony_Jeselnik, http://dbpedia.org/resource/Craig_Robinson_(actor), http://dbpedia.org/resource/J._B._Smoove, http://dbpedia.org/resource/Jay_Mohr, http://dbpedia.org/resource/Bill_Bellamy, http://dbpedia.org/resource/Fearne_Cotton, *@en, http://dbpedia.org/resource/Dick_Clark, George Gray@en, Anne Robinson@en, http://dbpedia.org/resource/Sebastian_Cabot_(actor), http://dbpedia.org/resource/Brooke_Burns, http://dbpedia.org/resource/Kevin_O'Connell_(U.S._TV_personality), http://dbpedia.org/resource/Nipsey_Russell, http://dbpedia.org/resource/Jack_Barry_(game_show_host), http://dbpedia.org/resource/Loretta_Young, http://dbpedia.org/resource/Geoff_Edwards, http://dbpedia.org/resource/Joey_Fatone, http://dbpedia.org/resource/Bill_Cullen, Elle Macpherson@en, Louise Roe@en, http://dbpedia.org/resource/Bill_Leyden, http://dbpedia.org/resource/Marsha_Warfield, http://dbpedia.org/resource/Chris_Schenkel, http://dbpedia.org/resource/Helen_Reddy, http://dbpedia.org/resource/NBC, http://dbpedia.org/resource/Alison_Haislip, http://dbpedia.org/resource/Christina_Milian, http://dbpedia.org/resource/Carson_Daly, http://dbpedia.org/resource/Andy_Williams, http://dbpedia.org/resource/Tom_Kennedy_(television_presenter), http://dbpedia.org/resource/Gisele_MacKenzie, http://dbpedia.org/resource/Alex_Trebek, http://dbpedia.org/resource/Roy_Black_(attorney), Duane Ellett@en, http://dbpedia.org/resource/Wendy_Barrie, http://dbpedia.org/resource/Bill_Stern, http://dbpedia.org/resource/Alistair_Cooke, Bear Grylls@en, http://dbpedia.org/resource/Bruce_Gordon_(actor), http://dbpedia.org/resource/Gary_Owens, http://dbpedia.org/resource/Lou_Crosby, Dean Cain@en, Ed McMahon@en, http://dbpedia.org/resource/Karyn_Bryant, http://dbpedia.org/resource/Mike_Wallace, http://dbpedia.org/resource/John_Conte_(actor), http://dbpedia.org/resource/Lindsay_Wagner, Everett McGill@en, http://dbpedia.org/resource/Jimmy_Fallon, http://dbpedia.org/resource/Gene_Raymond, http://dbpedia.org/resource/Jane_Wyman, http://dbpedia.org/resource/Frank_Wisbar, http://dbpedia.org/resource/Nick_Lachey, http://dbpedia.org/resource/Otto_Kruger, http://dbpedia.org/resource/Gordon_MacRae, http://dbpedia.org/resource/James_Mason, http://dbpedia.org/resource/Ken_Carpenter_(announcer), http://dbpedia.org/resource/Boris_Karloff, http://dbpedia.org/resource/James_Beard, http://dbpedia.org/resource/Clinton_McLean, http://dbpedia.org/resource/Kamaya_Jones, http://dbpedia.org/resource/Michelle_Armitage, http://dbpedia.org/resource/Rob_Czar, http://dbpedia.org/resource/Merv_Griffin, http://dbpedia.org/resource/Red_Skelton, http://dbpedia.org/resource/Michael_Nesmith, http://dbpedia.org/resource/Bill_Rafferty, http://dbpedia.org/resource/Howie_Mandel, http://dbpedia.org/resource/Leslie_Uggams, http://dbpedia.org/resource/Peter_Marshall_(entertainer), http://dbpedia.org/resource/Don_Knotts, Richard Boone@en, http://dbpedia.org/resource/Nate_Berkus, http://dbpedia.org/resource/Ron_Cochran, http://dbpedia.org/resource/Douglas_Edwards, http://dbpedia.org/resource/Sandy_Becker, http://dbpedia.org/resource/John_Cameron_Swayze, http://dbpedia.org/resource/Henry_Hamilton_(TV_presenter), Nelson Case@en, Bob Sherry@en, Joe Ripley@en, http://dbpedia.org/resource/Kathy_Hilton, Jonathan Winters@en, http://dbpedia.org/resource/J.D._Roth, http://dbpedia.org/resource/Arthur_Murray, http://dbpedia.org/resource/Dean_Martin, http://dbpedia.org/resource/Dinah_Shore, http://dbpedia.org/resource/Ahmad_Rashād, http://dbpedia.org/resource/Arlene_Francis, http://dbpedia.org/resource/John_O'Hurley, David Frei@en, http://dbpedia.org/resource/Chuck_Woolery, http://dbpedia.org/resource/Ryan_Seacrest, http://dbpedia.org/resource/Matt_Iseman, http://dbpedia.org/resource/Akbar_Gbaja-Biamila, Alex Curry@en, Kristine Leahy@en, http://dbpedia.org/resource/Bert_Parks, http://dbpedia.org/resource/Bud_Collyer, http://dbpedia.org/resource/Faye_Emerson, http://dbpedia.org/resource/John_McCaffery, Virgilia Peterson@en, http://dbpedia.org/resource/Burgess_Meredith, http://dbpedia.org/resource/Norman_Rose, http://dbpedia.org/resource/Ben_Grauer, http://dbpedia.org/resource/Des_O'Connor, http://dbpedia.org/resource/Deems_Taylor, http://dbpedia.org/resource/John_Mason_Brown, with Vivian Ferrer@en, http://dbpedia.org/resource/Nick_Cannon, http://dbpedia.org/resource/Conrad_Nagel, http://dbpedia.org/resource/Paul_Gallico, Ralph McNair@en, http://dbpedia.org/resource/Debbie_Allen, Zetta Wells@en, http://dbpedia.org/resource/Helen_Parrish, http://dbpedia.org/resource/Eddie_Mayehoff, http://dbpedia.org/resource/Marlon_Wayans, http://dbpedia.org/resource/Monty_Hall, http://dbpedia.org/resource/Fred_Allen, http://dbpedia.org/resource/Arte_Johnson, D'Avalos@en, http://dbpedia.org/resource/Tatyana_Ali, http://dbpedia.org/resource/Jordan_Brady, http://dbpedia.org/resource/Mario_Lopez, Kelly Vrooman@en, http://dbpedia.org/resource/Jon_Gnagy, http://dbpedia.org/resource/Warren_Hull, http://dbpedia.org/resource/Arthur_Shields, http://dbpedia.org/resource/Jack_Lescoulie, http://dbpedia.org/resource/Danielle_Harris, http://dbpedia.org/resource/Mark-Paul_Gosselaar, http://dbpedia.org/resource/Fred_Davis_(broadcaster), http://dbpedia.org/resource/Ben_Bailey, James McClain@en, Jay Owen@en, Lew Valentine@en, http://dbpedia.org/resource/Connie_Chung, http://dbpedia.org/resource/Roger_Mudd, with Lee Menning@en, http://dbpedia.org/resource/Mike_Adamle, http://dbpedia.org/resource/Bryant_Gumbel, http://dbpedia.org/resource/Barbara_Feldon, http://dbpedia.org/resource/Jackie_Cooper, http://dbpedia.org/resource/Robert_Montgomery_(actor), http://dbpedia.org/resource/Vernon_Kay, http://dbpedia.org/resource/Grant_Hill, Kristen Ledlow@en, http://dbpedia.org/resource/James_Melton, http://dbpedia.org/resource/Marco_Pierre_White, http://dbpedia.org/resource/IJustine, http://dbpedia.org/resource/Rossi_Morreale, http://dbpedia.org/resource/Allen_Ludden, http://dbpedia.org/resource/Robert_Earle, http://dbpedia.org/resource/Shirley_Temple, http://dbpedia.org/resource/Lloyd_Dobyns, http://dbpedia.org/resource/Linda_Ellerbee, Leo Durocher; Gale Storm; various@en] [INFO] Predicted Answer: [http://dbpedia.org/resource/Peter_Pan_Live!, http://dbpedia.org/resource/The_Sound_of_Music_Live!, http://dbpedia.org/resource/American_Comedy_Awards, http://dbpedia.org/resource/Nero_Wolfe_(1981_TV_series), http://dbpedia.org/resource/The_Singing_Bee_(U.S._game_show)] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 487, P: 0.524, R: 0.577, macro F1: 0.531, macro F1*: 0.549 -[INFO] Current time: 2829 ms, average: 2728.81 ms/ques, entity detection: 425.86 ms/ques (linking tool: 334.46 ms/ques), relation detection: 889.97 ms/ques (similarity: 697.50 ms/ques), EDG generation: 52.66 ms/ques +[INFO] Cumulative metrics, sample: 487, P: 0.524, R: 0.577, macro F1: 0.549 +[INFO] Current question consumed time: 2467 ms, average: 2504.94 ms/ques, entity detection: 386.76 ms/ques (linking tool: 305.98 ms/ques), relation detection: 833.74 ms/ques (similarity: 683.68 ms/ques), EDG generation: 52.52 ms/ques [INFO] Entity Linking P: 0.7515400410677618, R: 0.7084188911704312, micro F1: 0.7221081451060921, macro F1: 0.7293426558885452 [INFO] Relation Linking P: 0.6009582477754961, R: 0.555441478439425, micro F1: 0.5674880219028059, macro F1: 0.5773040758447057 [INFO] Type Linking P: 0.5441478439425051, R: 0.5441478439425051, micro F1: 0.5441478439425051, macro F1: 0.5441478439425051 @@ -6830,8 +6830,8 @@ Question 487: Give me the list of cars which are the predecessor of Full Sized C [INFO] Golden Answer: [http://dbpedia.org/resource/Honda_1300, http://dbpedia.org/resource/Holden_Brougham, http://dbpedia.org/resource/Willys_Jeep_Station_Wagon, http://dbpedia.org/resource/Dodge_M37, http://dbpedia.org/resource/Toyota_Verossa, http://dbpedia.org/resource/Toyota_Mark_II, http://dbpedia.org/resource/1941_Ford, http://dbpedia.org/resource/Ford_Fairlane_Crown_Victoria_Skyliner, http://dbpedia.org/resource/Ford_Taurus_(fourth_generation), http://dbpedia.org/resource/Ford_LTD_(Americas), http://dbpedia.org/resource/Ford_Pilot, http://dbpedia.org/resource/Pontiac_Star_Chief, http://dbpedia.org/resource/Pontiac_Torpedo, http://dbpedia.org/resource/Pontiac_Bonneville, http://dbpedia.org/resource/Pontiac_Catalina, http://dbpedia.org/resource/Pontiac_Streamliner, http://dbpedia.org/resource/Chevrolet_Master, http://dbpedia.org/resource/Austin_Twenty, http://dbpedia.org/resource/Acura_Legend, http://dbpedia.org/resource/Acura_RL, http://dbpedia.org/resource/Mercury_Monterey, http://dbpedia.org/resource/Ford_Fairlane_(Americas), http://dbpedia.org/resource/Mercedes_24/100/140_PS, http://dbpedia.org/resource/Peugeot_604, http://dbpedia.org/resource/Kia_Potentia, http://dbpedia.org/resource/Mazda_Luce, http://dbpedia.org/resource/DeSoto_Firedome, http://dbpedia.org/resource/GM_G_platform_(1969), http://dbpedia.org/resource/GM_B_platform, http://dbpedia.org/resource/AMC_Rebel, http://dbpedia.org/resource/Ford_Zephyr, http://dbpedia.org/resource/Nissan_Bluebird, http://dbpedia.org/resource/Nissan_Primera, http://dbpedia.org/resource/Nissan_Violet, http://dbpedia.org/resource/Mercury_Montego, http://dbpedia.org/resource/Mercury_Marquis, http://dbpedia.org/resource/Daimler_DK400, http://dbpedia.org/resource/Daimler_Regency, http://dbpedia.org/resource/GM_C_platform_(1985), http://dbpedia.org/resource/Chrysler_300M, http://dbpedia.org/resource/Dodge_Intrepid, http://dbpedia.org/resource/Chrysler_Concorde, http://dbpedia.org/resource/Chrysler_Royal_(Australia), http://dbpedia.org/resource/Dodge_Aspen, http://dbpedia.org/resource/Dodge_Daytona, http://dbpedia.org/resource/Dodge_Lancer, http://dbpedia.org/resource/Holden_Kingswood, http://dbpedia.org/resource/Mercedes-Benz_W116, http://dbpedia.org/resource/Mercedes-Benz_R107_and_C107, http://dbpedia.org/resource/Nissan_Cedric, http://dbpedia.org/resource/Mercury_Park_Lane, http://dbpedia.org/resource/Mercury_Montclair, http://dbpedia.org/resource/Honda_Vigor, http://dbpedia.org/resource/Lincoln_Continental, http://dbpedia.org/resource/ZIS-101, http://dbpedia.org/resource/Chevrolet_Brookwood, http://dbpedia.org/resource/GM_A_platform_(1982), http://dbpedia.org/resource/Chrysler_Six, http://dbpedia.org/resource/Chrysler_VIP, http://dbpedia.org/resource/Chevrolet_C/K, http://dbpedia.org/resource/Cadillac_Ciel, http://dbpedia.org/resource/Renault_Reinastella, http://dbpedia.org/resource/Renault_40CV, http://dbpedia.org/resource/BMW_326, http://dbpedia.org/resource/Bentley, http://dbpedia.org/resource/Bentley_3.5_Litre, http://dbpedia.org/resource/Bentley_Mark_V, http://dbpedia.org/resource/Bentley_T-series, http://dbpedia.org/resource/Bentley_R_Type, http://dbpedia.org/resource/Bentley_S2, http://dbpedia.org/resource/Bentley_S3, http://dbpedia.org/resource/Plymouth_Cranbrook, http://dbpedia.org/resource/GAZ-M1, http://dbpedia.org/resource/ZIS-110, http://dbpedia.org/resource/Studebaker_Big_Six, http://dbpedia.org/resource/Cadillac_Fleetwood, http://dbpedia.org/resource/Rolls-Royce_Phantom_VI, http://dbpedia.org/resource/Riley_RM, http://dbpedia.org/resource/Wolseley_6/90, http://dbpedia.org/resource/GMT400, http://dbpedia.org/resource/GMT800, http://dbpedia.org/resource/BMW_501, http://dbpedia.org/resource/Chevrolet_150, http://dbpedia.org/resource/Ford_F-Series_fourth_generation, http://dbpedia.org/resource/Ford_Falcon_(EL), http://dbpedia.org/resource/Ford_F-Series_second_generation, http://dbpedia.org/resource/BMW_New_Class, http://dbpedia.org/resource/BMW_7_Series_(F01), http://dbpedia.org/resource/Oldsmobile_Series_70, http://dbpedia.org/resource/Holden_Caprice, http://dbpedia.org/resource/Buick_Electra, http://dbpedia.org/resource/Chevrolet_Lumina, http://dbpedia.org/resource/Mexico, http://dbpedia.org/resource/North_America, http://dbpedia.org/resource/Pontiac_G8, http://dbpedia.org/resource/Chevrolet_Impala, http://dbpedia.org/resource/Chevrolet_K5_Blazer, http://dbpedia.org/resource/GM_V_platform_(1966), http://dbpedia.org/resource/GM_F_platform, http://dbpedia.org/resource/Statesman_(automobile), http://dbpedia.org/resource/Holden_Commodore_(VZ), http://dbpedia.org/resource/Holden_Commodore_(VE), http://dbpedia.org/resource/Holden_Commodore_(VL), http://dbpedia.org/resource/Holden_Commodore_(VP), http://dbpedia.org/resource/Holden_Commodore_(VS), http://dbpedia.org/resource/Holden_Commodore_(VY), http://dbpedia.org/resource/Holden_HJ, http://dbpedia.org/resource/Holden_HX, http://dbpedia.org/resource/Holden_Special, http://dbpedia.org/resource/Holden_HZ, http://dbpedia.org/resource/Mitsubishi_Galant, http://dbpedia.org/resource/Opel_Olympia, http://dbpedia.org/resource/Opel_Diplomat, http://dbpedia.org/resource/Pontiac_Grand_Prix, http://dbpedia.org/resource/Pontiac_GTO, http://dbpedia.org/resource/Ford_Falcon_(XE), http://dbpedia.org/resource/Nissan_Caravan, http://dbpedia.org/resource/Cadillac_Sixty_Special, http://dbpedia.org/resource/Ford_Bronco, http://dbpedia.org/resource/Ford_D186_platform, http://dbpedia.org/resource/Ford_Panther_platform, http://dbpedia.org/resource/Full-size_Ford, http://dbpedia.org/resource/Lincoln_Cosmopolitan, http://dbpedia.org/resource/Lincoln_Continental_Mark_V, http://dbpedia.org/resource/Lincoln_EL-series, http://dbpedia.org/resource/Lincoln_H-series, http://dbpedia.org/resource/Lincoln_Town_Car, http://dbpedia.org/resource/Lincoln_Blackwood, http://dbpedia.org/resource/Lincoln_Capri, http://dbpedia.org/resource/Volvo_P80, http://dbpedia.org/resource/Jaguar_Mark_IX, http://dbpedia.org/resource/Eagle_Vision, http://dbpedia.org/resource/GMC_Envoy, http://dbpedia.org/resource/Chevrolet_Astro, http://dbpedia.org/resource/Pontiac_Montana, http://dbpedia.org/resource/Renault_Type_JT, http://dbpedia.org/resource/Renault_Type_KR, http://dbpedia.org/resource/Saturn_Relay, http://dbpedia.org/resource/Chrysler_300_letter_series, http://dbpedia.org/resource/Chrysler_C_platform, http://dbpedia.org/resource/Nissan_Gloria, http://dbpedia.org/resource/Nissan_President, http://dbpedia.org/resource/Nissan_Cima, http://dbpedia.org/resource/Nissan_Leopard, http://dbpedia.org/resource/Holden_Commodore_(VN), http://dbpedia.org/resource/Holden_Commodore_(VR), http://dbpedia.org/resource/Holden_Commodore_(VT), http://dbpedia.org/resource/Kia_Opirus, http://dbpedia.org/resource/Lincoln_MKS, http://dbpedia.org/resource/Toyota_Cressida, http://dbpedia.org/resource/Toyota_Camry, http://dbpedia.org/resource/Buick_Regal, http://dbpedia.org/resource/Buick_Century, http://dbpedia.org/resource/Buick_Lucerne, http://dbpedia.org/resource/GM_G_platform_(1995), http://dbpedia.org/resource/GM_W_platform, http://dbpedia.org/resource/GM_N_platform, http://dbpedia.org/resource/GM2900_platform, http://dbpedia.org/resource/Buick_Series_40, http://dbpedia.org/resource/Lincoln-Zephyr, http://dbpedia.org/resource/Mercedes-Benz_600, http://dbpedia.org/resource/Mercedes-Benz_W126, http://dbpedia.org/resource/Mercedes-Benz_W111, http://dbpedia.org/resource/Buick_Series_60, http://dbpedia.org/resource/Buick_Roadmaster, http://dbpedia.org/resource/Buick_Super, http://dbpedia.org/resource/Buick_Terraza, http://dbpedia.org/resource/Buick_Rainier, http://dbpedia.org/resource/Buick_Rendezvous, http://dbpedia.org/resource/Buick_Special, http://dbpedia.org/resource/Buick_Invicta, http://dbpedia.org/resource/Cadillac_Series_62, http://dbpedia.org/resource/Dodge_B_Series, http://dbpedia.org/resource/Dodge_Model_DU, http://dbpedia.org/resource/Dodge_Dynasty, http://dbpedia.org/resource/Dodge_Monaco, http://dbpedia.org/resource/Dodge_Coronet, http://dbpedia.org/resource/Dodge_Diplomat, http://dbpedia.org/resource/Dodge_Polara, http://dbpedia.org/resource/Dodge_St._Regis, http://dbpedia.org/resource/Ford_F-Series_fifth_generation, http://dbpedia.org/resource/Hyundai_Dynasty, http://dbpedia.org/resource/Ford_Granada_(Europe), http://dbpedia.org/resource/Hyundai_Stellar, http://dbpedia.org/resource/Chrysler_Airflow, http://dbpedia.org/resource/Rolls-Royce_Silver_Shadow, http://dbpedia.org/resource/Mercedes-Benz_770, http://dbpedia.org/resource/Mercury_Eight, http://dbpedia.org/resource/GMT900, http://dbpedia.org/resource/Ford_Falcon_(AU), http://dbpedia.org/resource/Ford_Falcon_(BA), http://dbpedia.org/resource/GAZ-12_ZIM, http://dbpedia.org/resource/Ford_Falcon_(EF), http://dbpedia.org/resource/Range_Rover_(L322), http://dbpedia.org/resource/Dodge_Custom_Royal, http://dbpedia.org/resource/Chevrolet_210, http://dbpedia.org/resource/Chevrolet_Task_Force, http://dbpedia.org/resource/Cadillac_de_Ville_series, http://dbpedia.org/resource/Cadillac_Series_65, http://dbpedia.org/resource/Cadillac_Series_70, http://dbpedia.org/resource/Cadillac_STS, http://dbpedia.org/resource/Cadillac_DTS, http://dbpedia.org/resource/Ford_Falcon_(BF), http://dbpedia.org/resource/Austin_Kimberley, http://dbpedia.org/resource/Rover_P4, http://dbpedia.org/resource/Mercedes-Benz_Type_300, http://dbpedia.org/resource/DeSoto_Airflow, http://dbpedia.org/resource/DeSoto_Airstream, http://dbpedia.org/resource/DeSoto_Custom, http://dbpedia.org/resource/GM_Zeta_platform, http://dbpedia.org/resource/Chevrolet_Deluxe, http://dbpedia.org/resource/Plymouth_Belvedere, http://dbpedia.org/resource/Plymouth_Plaza, http://dbpedia.org/resource/Plymouth_Savoy, http://dbpedia.org/resource/Pontiac_Chieftain, http://dbpedia.org/resource/Cadillac_Series_355, http://dbpedia.org/resource/Cadillac_Fleetwood_Brougham, http://dbpedia.org/resource/Oldsmobile_88, http://dbpedia.org/resource/Nash_Ambassador, http://dbpedia.org/resource/Chrysler_New_Yorker, http://dbpedia.org/resource/Chrysler_LHS, http://dbpedia.org/resource/Eagle_Premier, http://dbpedia.org/resource/Lincoln_Aviator, http://dbpedia.org/resource/Tatra_603, http://dbpedia.org/resource/Tatra_613, http://dbpedia.org/resource/Chrysler_LeBaron, http://dbpedia.org/resource/Plymouth_Deluxe, http://dbpedia.org/resource/Plymouth_Fury, http://dbpedia.org/resource/Ford_Crown_Victoria_Police_Interceptor, http://dbpedia.org/resource/Ford_Taurus_(fifth_generation), http://dbpedia.org/resource/Plymouth_Model_PJ, http://dbpedia.org/resource/Ford_Five_Hundred, http://dbpedia.org/resource/Chevrolet_Series_C_Classic_Six, http://dbpedia.org/resource/Chrysler_Sebring] [INFO] Predicted Answer: [http://dbpedia.org/resource/Plymouth_Gran_Fury__First_generation__1, http://dbpedia.org/resource/Plymouth_Gran_Fury] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 488, P: 0.523, R: 0.575, macro F1: 0.530, macro F1*: 0.548 -[INFO] Current time: 933 ms, average: 2725.13 ms/ques, entity detection: 425.34 ms/ques (linking tool: 333.97 ms/ques), relation detection: 888.51 ms/ques (similarity: 696.35 ms/ques), EDG generation: 52.59 ms/ques +[INFO] Cumulative metrics, sample: 488, P: 0.523, R: 0.575, macro F1: 0.548 +[INFO] Current question consumed time: 824 ms, average: 2501.50 ms/ques, entity detection: 386.25 ms/ques (linking tool: 305.52 ms/ques), relation detection: 832.36 ms/ques (similarity: 682.54 ms/ques), EDG generation: 52.44 ms/ques [INFO] Entity Linking P: 0.7520491803278688, R: 0.7090163934426229, micro F1: 0.7226775956284157, macro F1: 0.7298990642172304 [INFO] Relation Linking P: 0.6017759562841529, R: 0.555327868852459, micro F1: 0.5676912568306007, macro F1: 0.5776196605183193 [INFO] Type Linking P: 0.5430327868852459, R: 0.5430327868852459, micro F1: 0.5430327868852459, macro F1: 0.5430327868852459 @@ -6844,8 +6844,8 @@ Question 488: List the people with final resting place as Plymouth, Vermont and [INFO] Golden Answer: [http://dbpedia.org/resource/Calvin_Coolidge] [INFO] Predicted Answer: [http://dbpedia.org/resource/Calvin_Coolidge, http://dbpedia.org/resource/David_I._Walsh] [INFO] P: 0.5, R: 1.0, F: 0.6666666666666666 -[INFO] Cumulative metrics, sample: 489, P: 0.523, R: 0.576, macro F1: 0.530, macro F1*: 0.548 -[INFO] Current time: 7897 ms, average: 2735.70 ms/ques, entity detection: 426.47 ms/ques (linking tool: 334.82 ms/ques), relation detection: 892.17 ms/ques (similarity: 701.16 ms/ques), EDG generation: 52.58 ms/ques +[INFO] Cumulative metrics, sample: 489, P: 0.523, R: 0.576, macro F1: 0.548 +[INFO] Current question consumed time: 7056 ms, average: 2510.81 ms/ques, entity detection: 387.12 ms/ques (linking tool: 306.18 ms/ques), relation detection: 835.70 ms/ques (similarity: 687.38 ms/ques), EDG generation: 52.43 ms/ques [INFO] Entity Linking P: 0.7525562372188139, R: 0.7085889570552147, micro F1: 0.7225630538513979, macro F1: 0.7299110880232863 [INFO] Relation Linking P: 0.6025903203817313, R: 0.5552147239263804, micro F1: 0.567893660531697, macro F1: 0.5779332539898974 [INFO] Type Linking P: 0.5419222903885481, R: 0.5419222903885481, micro F1: 0.5419222903885481, macro F1: 0.5419222903885481 @@ -6858,8 +6858,8 @@ Question 489: What disease led to the demise of Letitia MacTavish Hargrave and M [INFO] Golden Answer: [http://dbpedia.org/resource/Cholera] [INFO] Predicted Answer: [http://dbpedia.org/resource/Gabriel_Richard, http://dbpedia.org/resource/George_Bradshaw, http://dbpedia.org/resource/Martino_Martini, http://dbpedia.org/resource/Patrick_Kennedy_(1823–1858), http://dbpedia.org/resource/John_Thomson_Mason_(1787–1850), http://dbpedia.org/resource/Letitia_MacTavish_Hargrave, http://dbpedia.org/resource/Abdallah_Somekh, http://dbpedia.org/resource/Sextus_Barbour, http://dbpedia.org/resource/Farid-ud-Din_Qadri, http://dbpedia.org/resource/Elias_Higbee, http://dbpedia.org/resource/Paul-Henri_Marron, http://dbpedia.org/resource/John_Brant_(Mohawk_leader)] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 490, P: 0.522, R: 0.575, macro F1: 0.529, macro F1*: 0.547 -[INFO] Current time: 1778 ms, average: 2733.75 ms/ques, entity detection: 425.91 ms/ques (linking tool: 334.30 ms/ques), relation detection: 891.48 ms/ques (similarity: 700.57 ms/ques), EDG generation: 52.54 ms/ques +[INFO] Cumulative metrics, sample: 490, P: 0.522, R: 0.575, macro F1: 0.547 +[INFO] Current question consumed time: 1520 ms, average: 2508.79 ms/ques, entity detection: 386.58 ms/ques (linking tool: 305.69 ms/ques), relation detection: 835.03 ms/ques (similarity: 686.81 ms/ques), EDG generation: 52.42 ms/ques [INFO] Entity Linking P: 0.753061224489796, R: 0.7081632653061225, micro F1: 0.7224489795918372, macro F1: 0.729922471782009 [INFO] Relation Linking P: 0.6034013605442176, R: 0.5561224489795918, micro F1: 0.5687755102040812, macro F1: 0.5787980196478709 [INFO] Type Linking P: 0.5408163265306123, R: 0.5408163265306123, micro F1: 0.5408163265306123, macro F1: 0.5408163265306123 @@ -6872,8 +6872,8 @@ Question 490: What are some software which are written in languages that are run [INFO] Golden Answer: [http://dbpedia.org/resource/Farstream, http://dbpedia.org/resource/Archon_(software), http://dbpedia.org/resource/Omeka, http://dbpedia.org/resource/Adminer, http://dbpedia.org/resource/Doctrine_(PHP), http://dbpedia.org/resource/Joomla, http://dbpedia.org/resource/PhpBB, http://dbpedia.org/resource/PhpMyAdmin, http://dbpedia.org/resource/SQLBuddy, http://dbpedia.org/resource/TYPO3, http://dbpedia.org/resource/Rocks_(PHP), http://dbpedia.org/resource/XHP, http://dbpedia.org/resource/Yasca, http://dbpedia.org/resource/Bitrix24, http://dbpedia.org/resource/Zinf, http://dbpedia.org/resource/VBulletin, http://dbpedia.org/resource/Xplico, http://dbpedia.org/resource/Magento, http://dbpedia.org/resource/OsCommerce, http://dbpedia.org/resource/Roundcube, http://dbpedia.org/resource/The_Next_Generation_of_Genealogy_Sitebuilding, http://dbpedia.org/resource/Dolphin_(social_networking_software), http://dbpedia.org/resource/Elgg_(software), http://dbpedia.org/resource/OCS_Inventory, http://dbpedia.org/resource/Zabbix, http://dbpedia.org/resource/Phoronix_Test_Suite, http://dbpedia.org/resource/PhpDocumentor, http://dbpedia.org/resource/Horde_(software), http://dbpedia.org/resource/TxtSQL, http://dbpedia.org/resource/Open_Power_Template, http://dbpedia.org/resource/TinyButStrong, http://dbpedia.org/resource/Piwigo, http://dbpedia.org/resource/PhpLDAPadmin, http://dbpedia.org/resource/Silex_(web_framework), http://dbpedia.org/resource/XOS_Webtop, http://dbpedia.org/resource/Sajax, http://dbpedia.org/resource/Cloudike, http://dbpedia.org/resource/TestLink, http://dbpedia.org/resource/Croogo, http://dbpedia.org/resource/Ilibrarian, http://dbpedia.org/resource/Inoreader, http://dbpedia.org/resource/Drupal, http://dbpedia.org/resource/Moodle, http://dbpedia.org/resource/MindTouch, http://dbpedia.org/resource/OPUS_(software), http://dbpedia.org/resource/PrestaShop, http://dbpedia.org/resource/Microfin360, http://dbpedia.org/resource/PHPIDS, http://dbpedia.org/resource/Phabricator, http://dbpedia.org/resource/WikiTrust, http://dbpedia.org/resource/WikkaWiki, http://dbpedia.org/resource/Tine_2.0, http://dbpedia.org/resource/DAViCal, http://dbpedia.org/resource/SabreDAV, http://dbpedia.org/resource/MultiOTP, http://dbpedia.org/resource/Behat_(computer_science), http://dbpedia.org/resource/Avactis, http://dbpedia.org/resource/Drupal_Commerce, http://dbpedia.org/resource/IzzoNet, http://dbpedia.org/resource/Jigoshop, http://dbpedia.org/resource/LemonStand, http://dbpedia.org/resource/OsCMax, http://dbpedia.org/resource/Aegir_Hosting_System, http://dbpedia.org/resource/Network_Security_Toolkit, http://dbpedia.org/resource/HRHIS, http://dbpedia.org/resource/Simple_Cloud_API, http://dbpedia.org/resource/Observium, http://dbpedia.org/resource/RIPS, http://dbpedia.org/resource/PHP-Qt, http://dbpedia.org/resource/WxPHP, http://dbpedia.org/resource/Robotfindskitten, http://dbpedia.org/resource/WooCommerce, http://dbpedia.org/resource/GLPI, http://dbpedia.org/resource/Tuleap_(project_management), http://dbpedia.org/resource/OpenSearchServer, http://dbpedia.org/resource/PhpLiteAdmin, http://dbpedia.org/resource/Genealone, http://dbpedia.org/resource/Geneotree, http://dbpedia.org/resource/HuMo-gen, http://dbpedia.org/resource/BEdita, http://dbpedia.org/resource/Cgiapp, http://dbpedia.org/resource/AppScale, http://dbpedia.org/resource/GIMP, http://dbpedia.org/resource/Tencent_QQ, http://dbpedia.org/resource/PHPUnit, http://dbpedia.org/resource/Leafpad, http://dbpedia.org/resource/Savane_(software), http://dbpedia.org/resource/OpenBroadcaster, http://dbpedia.org/resource/Pandora_FMS, http://dbpedia.org/resource/2-plan_project_management_software, http://dbpedia.org/resource/F-Droid, http://dbpedia.org/resource/XenForo, http://dbpedia.org/resource/TAO_(e-Testing_platform), http://dbpedia.org/resource/Ampache, http://dbpedia.org/resource/Batavi_(software), http://dbpedia.org/resource/AfterLogic_WebMail_Lite, http://dbpedia.org/resource/SuiteCRM, http://dbpedia.org/resource/X2Engine, http://dbpedia.org/resource/PHP_Unit_Testing_Framework, http://dbpedia.org/resource/PMB_(software), http://dbpedia.org/resource/SimpleTest, http://dbpedia.org/resource/Ajax_Animator, http://dbpedia.org/resource/Gnash, http://dbpedia.org/resource/CodeIgniter, http://dbpedia.org/resource/Coppermine_Photo_Gallery, http://dbpedia.org/resource/HipHop_for_PHP, http://dbpedia.org/resource/PRADO_(framework), http://dbpedia.org/resource/Phalcon_(framework), http://dbpedia.org/resource/PhpWebSite, http://dbpedia.org/resource/XOOPS, http://dbpedia.org/resource/Zikula, http://dbpedia.org/resource/ATutor, http://dbpedia.org/resource/Active_Agenda, http://dbpedia.org/resource/Chamilo, http://dbpedia.org/resource/Claroline, http://dbpedia.org/resource/DoceboLMS, http://dbpedia.org/resource/Dolibarr, http://dbpedia.org/resource/EFront_(eLearning_software), http://dbpedia.org/resource/FlightPath_(software), http://dbpedia.org/resource/Frontaccounting, http://dbpedia.org/resource/HomeBank, http://dbpedia.org/resource/ILIAS, http://dbpedia.org/resource/Ino_erp, http://dbpedia.org/resource/OpenBiblio, http://dbpedia.org/resource/OpenDAM, http://dbpedia.org/resource/OpenSIS, http://dbpedia.org/resource/OwnCloud, http://dbpedia.org/resource/Oxwall, http://dbpedia.org/resource/Qcodo, http://dbpedia.org/resource/Vtiger_CRM, http://dbpedia.org/resource/WebERP, http://dbpedia.org/resource/Lime_(software), http://dbpedia.org/resource/Vanilla_Forums, http://dbpedia.org/resource/GeSHi, http://dbpedia.org/resource/MPack_(software), http://dbpedia.org/resource/GKrellM, http://dbpedia.org/resource/Internet_Messaging_Program, http://dbpedia.org/resource/Yahoo_Sherpa, http://dbpedia.org/resource/ADOdb, http://dbpedia.org/resource/Friendica, http://dbpedia.org/resource/Yii, http://dbpedia.org/resource/OntoWiki, http://dbpedia.org/resource/J-Pilot, http://dbpedia.org/resource/SiteBar, http://dbpedia.org/resource/Agavi, http://dbpedia.org/resource/Hazaar_MVC, http://dbpedia.org/resource/TYPO3_Flow, http://dbpedia.org/resource/IS4C, http://dbpedia.org/resource/Humhub, http://dbpedia.org/resource/Tiny_Tiny_RSS, http://dbpedia.org/resource/Revive_Adserver, http://dbpedia.org/resource/PHPMailer, http://dbpedia.org/resource/Max_Editor, http://dbpedia.org/resource/GenoCAD, http://dbpedia.org/resource/SPIP, http://dbpedia.org/resource/ROX_Desktop, http://dbpedia.org/resource/MediaWiki, http://dbpedia.org/resource/PHP-Fusion, http://dbpedia.org/resource/VisualEditor, http://dbpedia.org/resource/BlueSpice_for_MediaWiki, http://dbpedia.org/resource/PHP-Crawler, http://dbpedia.org/resource/Semantic_MediaWiki, http://dbpedia.org/resource/Tiki_Wiki_CMS_Groupware, http://dbpedia.org/resource/Chemicalize.org, http://dbpedia.org/resource/Packetsquare, http://dbpedia.org/resource/Cotonti, http://dbpedia.org/resource/ImpressCMS, http://dbpedia.org/resource/Jamroom, http://dbpedia.org/resource/MODX, http://dbpedia.org/resource/Mambo_(software), http://dbpedia.org/resource/ProcessWire, http://dbpedia.org/resource/Xaraya, http://dbpedia.org/resource/SMW+, http://dbpedia.org/resource/BioPHP, http://dbpedia.org/resource/TCPDF, http://dbpedia.org/resource/UBB.threads, http://dbpedia.org/resource/TorrentFlux, http://dbpedia.org/resource/ISPConfig, http://dbpedia.org/resource/Discuz!, http://dbpedia.org/resource/Iko_(software), http://dbpedia.org/resource/Phorum, http://dbpedia.org/resource/PunBB, http://dbpedia.org/resource/Lithium_(software), http://dbpedia.org/resource/Propel_(PHP), http://dbpedia.org/resource/Redbean_(software), http://dbpedia.org/resource/Scuttle_(software), http://dbpedia.org/resource/OpenConf, http://dbpedia.org/resource/La-La, http://dbpedia.org/resource/Roockbuilder, http://dbpedia.org/resource/EsoTalk, http://dbpedia.org/resource/Google_App_Engine, http://dbpedia.org/resource/SOBI2, http://dbpedia.org/resource/PHP-Nuke, http://dbpedia.org/resource/Appserver.io, http://dbpedia.org/resource/Twig_(template_engine), http://dbpedia.org/resource/Cacti_(software), http://dbpedia.org/resource/Ganglia_(software), http://dbpedia.org/resource/WebAnywhere, http://dbpedia.org/resource/Webtrees, http://dbpedia.org/resource/Gollum_browser, http://dbpedia.org/resource/Fat-Free_Framework, http://dbpedia.org/resource/Collabtive, http://dbpedia.org/resource/ShiftSpace, http://dbpedia.org/resource/PHProjekt, http://dbpedia.org/resource/The_Bug_Genie, http://dbpedia.org/resource/ClearHealth, http://dbpedia.org/resource/FreeMED, http://dbpedia.org/resource/Bluemix, http://dbpedia.org/resource/Ultrose, http://dbpedia.org/resource/Kohana_(framework), http://dbpedia.org/resource/RavenNuke, http://dbpedia.org/resource/IDSlot, http://dbpedia.org/resource/WeeVis, http://dbpedia.org/resource/Hoa_project, http://dbpedia.org/resource/Fusebox_(programming), http://dbpedia.org/resource/Open_Semantic_Framework, http://dbpedia.org/resource/Concrete5, http://dbpedia.org/resource/EGroupWare, http://dbpedia.org/resource/EyeOS, http://dbpedia.org/resource/FUDforum, http://dbpedia.org/resource/FusionForge, http://dbpedia.org/resource/GNU_social, http://dbpedia.org/resource/Mantis_Bug_Tracker, http://dbpedia.org/resource/OpenQRM, http://dbpedia.org/resource/PhpGedView, http://dbpedia.org/resource/PhpList, http://dbpedia.org/resource/SquirrelMail, http://dbpedia.org/resource/Airtime, http://dbpedia.org/resource/B2evolution, http://dbpedia.org/resource/CMS_Made_Simple, http://dbpedia.org/resource/Contao, http://dbpedia.org/resource/Dot.orange, http://dbpedia.org/resource/Dotclear, http://dbpedia.org/resource/Frontis_Archive_Publishing_System, http://dbpedia.org/resource/GetSimple_CMS, http://dbpedia.org/resource/ImpressPages, http://dbpedia.org/resource/Kajona, http://dbpedia.org/resource/MiaCMS, http://dbpedia.org/resource/Novius_OS, http://dbpedia.org/resource/October_(CMS), http://dbpedia.org/resource/PageFramer, http://dbpedia.org/resource/Ph7cms, http://dbpedia.org/resource/PhpWebLog, http://dbpedia.org/resource/PhpWiki, http://dbpedia.org/resource/Pixie_(CMS), http://dbpedia.org/resource/Pluck_(software), http://dbpedia.org/resource/Pulse_CMS, http://dbpedia.org/resource/Smarty, http://dbpedia.org/resource/Wolf_CMS, http://dbpedia.org/resource/VIPS_(software), http://dbpedia.org/resource/Sugar_(software), http://dbpedia.org/resource/Mapbender, http://dbpedia.org/resource/LDAP_Account_Manager, http://dbpedia.org/resource/Hierarchical_Cluster_Engine_Project, http://dbpedia.org/resource/Taskfreak, http://dbpedia.org/resource/Scrumedge, http://dbpedia.org/resource/Phreedom, http://dbpedia.org/resource/Flash_Gallery, http://dbpedia.org/resource/TinyWebGallery, http://dbpedia.org/resource/Heurist, http://dbpedia.org/resource/PeerTracker, http://dbpedia.org/resource/TalentLMS, http://dbpedia.org/resource/Far-Play, http://dbpedia.org/resource/Open-Realty, http://dbpedia.org/resource/ADOdb_Lite, http://dbpedia.org/resource/Online_Blood_Bowl_League_Manager, http://dbpedia.org/resource/Glossword, http://dbpedia.org/resource/DaDaBIK, http://dbpedia.org/resource/Composer_(software), http://dbpedia.org/resource/Flash_MP3_Player, http://dbpedia.org/resource/ParadisoLMS, http://dbpedia.org/resource/Totara_LMS, http://dbpedia.org/resource/OpenAutonomy, http://dbpedia.org/resource/OrfeoGPL, http://dbpedia.org/resource/VTECRM, http://dbpedia.org/resource/Ultamatix] [INFO] Predicted Answer: [http://dbpedia.org/resource/Stephen_R._Bourne, http://dbpedia.org/resource/Bram_Moolenaar, http://dbpedia.org/resource/David_Korn_(computer_scientist), http://dbpedia.org/resource/Bill_Joy, Innovative Software@en] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 491, P: 0.521, R: 0.574, macro F1: 0.528, macro F1*: 0.546 -[INFO] Current time: 2553 ms, average: 2733.38 ms/ques, entity detection: 425.53 ms/ques (linking tool: 334.03 ms/ques), relation detection: 891.64 ms/ques (similarity: 700.79 ms/ques), EDG generation: 52.52 ms/ques +[INFO] Cumulative metrics, sample: 491, P: 0.521, R: 0.574, macro F1: 0.546 +[INFO] Current question consumed time: 2244 ms, average: 2508.25 ms/ques, entity detection: 386.22 ms/ques (linking tool: 305.43 ms/ques), relation detection: 835.05 ms/ques (similarity: 687.04 ms/ques), EDG generation: 52.42 ms/ques [INFO] Entity Linking P: 0.7515274949083504, R: 0.7067209775967414, micro F1: 0.7209775967413447, macro F1: 0.7284358679698256 [INFO] Relation Linking P: 0.6031907671418872, R: 0.5560081466395111, micro F1: 0.5686354378818733, macro F1: 0.5786392249361121 [INFO] Type Linking P: 0.539714867617108, R: 0.539714867617108, micro F1: 0.539714867617108, macro F1: 0.539714867617108 @@ -6886,8 +6886,8 @@ Question 491: List all the movies whose directors were born in Kenosha, Wisconsi [INFO] Golden Answer: [http://dbpedia.org/resource/The_Christmas_Candle, http://dbpedia.org/resource/One_Man_Band_(unfinished_film), http://dbpedia.org/resource/The_Heroine, http://dbpedia.org/resource/Citizen_Kane, http://dbpedia.org/resource/The_Magnificent_Ambersons_(film), http://dbpedia.org/resource/The_Lady_from_Shanghai, http://dbpedia.org/resource/The_Stranger_(1946_film), http://dbpedia.org/resource/David_and_Goliath_(1960_film), http://dbpedia.org/resource/F_for_Fake, http://dbpedia.org/resource/The_Trial_(1962_film), http://dbpedia.org/resource/Chimes_at_Midnight, http://dbpedia.org/resource/Citizen_Kane_trailer, http://dbpedia.org/resource/Mr._Arkadin, http://dbpedia.org/resource/Touch_of_Evil, http://dbpedia.org/resource/Journey_into_Fear_(1943_film), http://dbpedia.org/resource/Too_Much_Johnson, http://dbpedia.org/resource/The_Deep_(unfinished_film), http://dbpedia.org/resource/Moby_Dick_(unfinished_film), http://dbpedia.org/resource/It's_All_True_(film), http://dbpedia.org/resource/In_the_Land_of_Don_Quixote, http://dbpedia.org/resource/The_Merchant_of_Venice_(1969_film), http://dbpedia.org/resource/The_Miracle_of_St._Anne, http://dbpedia.org/resource/Vienna_(film), http://dbpedia.org/resource/Othello_(1952_film), http://dbpedia.org/resource/Around_the_World_with_Orson_Welles, http://dbpedia.org/resource/Black_Magic_(1949_film), http://dbpedia.org/resource/Filming_Othello, http://dbpedia.org/resource/Filming_The_Trial, http://dbpedia.org/resource/Orson_Welles'_Magic_Show, http://dbpedia.org/resource/The_Fountain_of_Youth_(film), http://dbpedia.org/resource/The_Immortal_Story, http://dbpedia.org/resource/The_Other_Side_of_the_Wind, http://dbpedia.org/resource/Three_Cases_of_Murder, http://dbpedia.org/resource/Macbeth_(1948_film), http://dbpedia.org/resource/Don_Quixote_(unfinished_film), http://dbpedia.org/resource/Orson_Welles_and_People, http://dbpedia.org/resource/Julia_(TV_series), http://dbpedia.org/resource/Sympathy_for_Delicious, http://dbpedia.org/resource/Village_of_the_Giants, http://dbpedia.org/resource/Burned_at_the_Stake, http://dbpedia.org/resource/Tormented_(1960_film), http://dbpedia.org/resource/The_Cyclops_(film), http://dbpedia.org/resource/The_Magic_Sword_(1962_film), http://dbpedia.org/resource/Empire_of_the_Ants_(film), http://dbpedia.org/resource/The_Food_of_the_Gods_(film), http://dbpedia.org/resource/King_Dinosaur, http://dbpedia.org/resource/Beginning_of_the_End_(film), http://dbpedia.org/resource/Satan's_Princess, http://dbpedia.org/resource/The_Boy_and_the_Pirates, http://dbpedia.org/resource/The_Mad_Bomber_(1972_film), http://dbpedia.org/resource/The_Amazing_Colossal_Man, http://dbpedia.org/resource/Attack_of_the_Puppet_People, http://dbpedia.org/resource/Picture_Mommy_Dead, http://dbpedia.org/resource/Earth_vs._the_Spider, http://dbpedia.org/resource/War_of_the_Colossal_Beast, http://dbpedia.org/resource/Necromancy_(film), http://dbpedia.org/resource/Bunyan_and_Babe, http://dbpedia.org/resource/Cracked_Nuts, http://dbpedia.org/resource/Never_Give_a_Sucker_an_Even_Break, http://dbpedia.org/resource/You_Can't_Cheat_an_Honest_Man, http://dbpedia.org/resource/Bringing_Up_Father_(1946_film), http://dbpedia.org/resource/Hard_Luck_(1921_film), http://dbpedia.org/resource/Hat_Check_Honey, http://dbpedia.org/resource/Hook,_Line_and_Sinker_(1930_film), http://dbpedia.org/resource/Jiggs_and_Maggie_in_Society, http://dbpedia.org/resource/Let_It_Rain_(film), http://dbpedia.org/resource/Soft_Cushions, http://dbpedia.org/resource/The_Dude_Ranger, http://dbpedia.org/resource/Meet_the_Chump, http://dbpedia.org/resource/Circus_Days, http://dbpedia.org/resource/In_the_Next_Room, http://dbpedia.org/resource/Sweet_Mama_(film), http://dbpedia.org/resource/The_Forward_Pass, http://dbpedia.org/resource/The_Naughty_Flirt, http://dbpedia.org/resource/The_Widow_from_Chicago, http://dbpedia.org/resource/Neighbors_(1920_film), http://dbpedia.org/resource/Leathernecking, http://dbpedia.org/resource/Day_Dreams_(1922_film), http://dbpedia.org/resource/Ghost_Catchers, http://dbpedia.org/resource/F-Man, http://dbpedia.org/resource/Private_Snuffy_Smith, http://dbpedia.org/resource/Hawaii_Calls_(film), http://dbpedia.org/resource/Hellzapoppin'_(film), http://dbpedia.org/resource/Crazy_House_(1943_film), http://dbpedia.org/resource/Give_Out,_Sisters, http://dbpedia.org/resource/Along_Came_Ruth, http://dbpedia.org/resource/Captain_January_(1924_film), http://dbpedia.org/resource/Convict_13, http://dbpedia.org/resource/Little_Robinson_Crusoe, http://dbpedia.org/resource/My_Wife's_Relations, http://dbpedia.org/resource/Old_Clothes, http://dbpedia.org/resource/The_Haunted_House_(1921_film), http://dbpedia.org/resource/The_Love_Nest_(1923_film), http://dbpedia.org/resource/The_Paleface_(1922_film), http://dbpedia.org/resource/The_Rag_Man, http://dbpedia.org/resource/The_Scarecrow_(1920_film), http://dbpedia.org/resource/Parole_Girl, http://dbpedia.org/resource/High_Flyers, http://dbpedia.org/resource/The_Villain_Still_Pursued_Her, http://dbpedia.org/resource/When_a_Man's_a_Man, http://dbpedia.org/resource/My_Little_Chickadee, http://dbpedia.org/resource/Million_Dollar_Legs_(1932_film), http://dbpedia.org/resource/Private_Buckaroo, http://dbpedia.org/resource/What's_Cookin'%3F, http://dbpedia.org/resource/Ladies'_Night_in_a_Turkish_Bath, http://dbpedia.org/resource/Cops_(film), http://dbpedia.org/resource/The_Balloonatic, http://dbpedia.org/resource/The_High_Sign, http://dbpedia.org/resource/One_Week_(1920_film), http://dbpedia.org/resource/The_Boat_(film), http://dbpedia.org/resource/The_Electric_House, http://dbpedia.org/resource/The_Frozen_North, http://dbpedia.org/resource/The_Playhouse_(film), http://dbpedia.org/resource/Three_Ages, http://dbpedia.org/resource/Peck's_Bad_Boy_with_the_Circus, http://dbpedia.org/resource/Peck's_Bad_Boy_(1934_film), http://dbpedia.org/resource/The_Cowboy_Millionaire_(film), http://dbpedia.org/resource/The_Girl_Habit, http://dbpedia.org/resource/Go_Chase_Yourself, http://dbpedia.org/resource/On_Again-Off_Again, http://dbpedia.org/resource/Moonlight_and_Cactus_(1944_film), http://dbpedia.org/resource/The_Bank_Dick, http://dbpedia.org/resource/Breaking_the_Ice_(1938_film), http://dbpedia.org/resource/Forty_Naughty_Girls, http://dbpedia.org/resource/Fighting_to_Live, http://dbpedia.org/resource/Medea_Culpa] [INFO] Predicted Answer: [http://dbpedia.org/resource/Paul_Sorensen, http://dbpedia.org/resource/Tony_Russel] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 492, P: 0.520, R: 0.573, macro F1: 0.527, macro F1*: 0.545 -[INFO] Current time: 3545 ms, average: 2735.03 ms/ques, entity detection: 425.87 ms/ques (linking tool: 334.41 ms/ques), relation detection: 891.84 ms/ques (similarity: 701.55 ms/ques), EDG generation: 52.47 ms/ques +[INFO] Cumulative metrics, sample: 492, P: 0.520, R: 0.573, macro F1: 0.545 +[INFO] Current question consumed time: 3088 ms, average: 2509.43 ms/ques, entity detection: 386.46 ms/ques (linking tool: 305.75 ms/ques), relation detection: 835.13 ms/ques (similarity: 687.81 ms/ques), EDG generation: 52.36 ms/ques [INFO] Entity Linking P: 0.7520325203252033, R: 0.7073170731707317, micro F1: 0.7215447154471549, macro F1: 0.7289897411508933 [INFO] Relation Linking P: 0.6039972899728997, R: 0.5558943089430894, micro F1: 0.5688346883468831, macro F1: 0.5789483369424815 [INFO] Type Linking P: 0.5386178861788617, R: 0.5386178861788617, micro F1: 0.5386178861788617, macro F1: 0.5386178861788617 @@ -6900,8 +6900,8 @@ Question 492: Does the St lawrence river start in Kingston, Ontario? [INFO] Golden Answer: [true] [INFO] Predicted Answer: [true] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 493, P: 0.521, R: 0.574, macro F1: 0.528, macro F1*: 0.546 -[INFO] Current time: 6286 ms, average: 2742.23 ms/ques, entity detection: 427.37 ms/ques (linking tool: 335.76 ms/ques), relation detection: 895.52 ms/ques (similarity: 702.61 ms/ques), EDG generation: 52.39 ms/ques +[INFO] Cumulative metrics, sample: 493, P: 0.521, R: 0.574, macro F1: 0.546 +[INFO] Current question consumed time: 5197 ms, average: 2514.88 ms/ques, entity detection: 387.69 ms/ques (linking tool: 306.87 ms/ques), relation detection: 838.24 ms/ques (similarity: 688.90 ms/ques), EDG generation: 52.28 ms/ques [INFO] Entity Linking P: 0.7505070993914807, R: 0.7058823529411765, micro F1: 0.7200811359026373, macro F1: 0.7275110601343602 [INFO] Relation Linking P: 0.6027721433400945, R: 0.5547667342799188, micro F1: 0.5676808654496278, macro F1: 0.5777739995450324 [INFO] Type Linking P: 0.539553752535497, R: 0.539553752535497, micro F1: 0.539553752535497, macro F1: 0.539553752535497 @@ -6914,8 +6914,8 @@ Question 493: What stadium's operator is Stan Kroenke [INFO] Golden Answer: [http://dbpedia.org/resource/Dick's_Sporting_Goods_Park, http://dbpedia.org/resource/Pepsi_Center] [INFO] Predicted Answer: [http://dbpedia.org/resource/Dick's_Sporting_Goods_Park, http://dbpedia.org/resource/Pepsi_Center] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 494, P: 0.522, R: 0.574, macro F1: 0.529, macro F1*: 0.547 -[INFO] Current time: 1043 ms, average: 2738.79 ms/ques, entity detection: 426.87 ms/ques (linking tool: 335.29 ms/ques), relation detection: 894.38 ms/ques (similarity: 701.88 ms/ques), EDG generation: 52.30 ms/ques +[INFO] Cumulative metrics, sample: 494, P: 0.522, R: 0.574, macro F1: 0.547 +[INFO] Current question consumed time: 929 ms, average: 2511.67 ms/ques, entity detection: 387.19 ms/ques (linking tool: 306.42 ms/ques), relation detection: 837.18 ms/ques (similarity: 688.20 ms/ques), EDG generation: 52.19 ms/ques [INFO] Entity Linking P: 0.7510121457489879, R: 0.7064777327935222, micro F1: 0.7206477732793527, macro F1: 0.7280645524066576 [INFO] Relation Linking P: 0.6035762483130903, R: 0.555668016194332, micro F1: 0.5685560053981104, macro F1: 0.578632177515518 [INFO] Type Linking P: 0.5384615384615384, R: 0.5384615384615384, micro F1: 0.5384615384615384, macro F1: 0.5384615384615384 @@ -6928,8 +6928,8 @@ Question 494: Is Mike Babcock the head coach of Toronto Maple Leafs? [INFO] Golden Answer: [true] [INFO] Predicted Answer: [true] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 495, P: 0.523, R: 0.575, macro F1: 0.530, macro F1*: 0.548 -[INFO] Current time: 2008 ms, average: 2737.32 ms/ques, entity detection: 427.25 ms/ques (linking tool: 335.51 ms/ques), relation detection: 894.14 ms/ques (similarity: 701.33 ms/ques), EDG generation: 52.22 ms/ques +[INFO] Cumulative metrics, sample: 495, P: 0.523, R: 0.575, macro F1: 0.548 +[INFO] Current question consumed time: 1755 ms, average: 2510.14 ms/ques, entity detection: 387.46 ms/ques (linking tool: 306.59 ms/ques), relation detection: 836.93 ms/ques (similarity: 687.69 ms/ques), EDG generation: 52.12 ms/ques [INFO] Entity Linking P: 0.7515151515151515, R: 0.7070707070707071, micro F1: 0.7212121212121216, macro F1: 0.7286157978678753 [INFO] Relation Linking P: 0.6043771043771043, R: 0.5565656565656566, micro F1: 0.5694276094276091, macro F1: 0.579486863999625 [INFO] Type Linking P: 0.5393939393939394, R: 0.5393939393939394, micro F1: 0.5393939393939394, macro F1: 0.5393939393939394 @@ -6942,8 +6942,8 @@ Question 495: Whose commanders are John Mc Ewen and Park Chunghee? [INFO] Golden Answer: [http://dbpedia.org/resource/Vietnam_War] [INFO] Predicted Answer: [Coast of Scotland, HMS Tiger, HMS Phaeton, HMS Glorious] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 496, P: 0.522, R: 0.574, macro F1: 0.529, macro F1*: 0.547 -[INFO] Current time: 4075 ms, average: 2740.01 ms/ques, entity detection: 427.19 ms/ques (linking tool: 335.37 ms/ques), relation detection: 894.93 ms/ques (similarity: 702.51 ms/ques), EDG generation: 52.16 ms/ques +[INFO] Cumulative metrics, sample: 496, P: 0.522, R: 0.574, macro F1: 0.547 +[INFO] Current question consumed time: 3570 ms, average: 2512.28 ms/ques, entity detection: 387.28 ms/ques (linking tool: 306.38 ms/ques), relation detection: 837.61 ms/ques (similarity: 688.90 ms/ques), EDG generation: 52.05 ms/ques [INFO] Entity Linking P: 0.75, R: 0.7056451612903226, micro F1: 0.7197580645161294, macro F1: 0.7271468144044323 [INFO] Relation Linking P: 0.6031586021505375, R: 0.5554435483870968, micro F1: 0.5682795698924727, macro F1: 0.5783185437093031 [INFO] Type Linking P: 0.5403225806451613, R: 0.5403225806451613, micro F1: 0.5403225806451613, macro F1: 0.5403225806451613 @@ -6956,8 +6956,8 @@ Question 496: Who gave guest appearances on the show which is the next episode a [INFO] Golden Answer: [http://dbpedia.org/resource/Thorsten_Kaye, http://dbpedia.org/resource/Michael_Cristofer, http://dbpedia.org/resource/Jennifer_Hudson, http://dbpedia.org/resource/Brian_d'Arcy_James, as Jerry Rand@en, as Frank Houston@en, as Nick@en, as Veronica Moore@en] [INFO] Predicted Answer: [] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 497, P: 0.520, R: 0.573, macro F1: 0.528, macro F1*: 0.545 -[INFO] Current time: 1441 ms, average: 2737.40 ms/ques, entity detection: 426.61 ms/ques (linking tool: 334.86 ms/ques), relation detection: 893.90 ms/ques (similarity: 701.94 ms/ques), EDG generation: 52.15 ms/ques +[INFO] Cumulative metrics, sample: 497, P: 0.520, R: 0.573, macro F1: 0.545 +[INFO] Current question consumed time: 1306 ms, average: 2509.85 ms/ques, entity detection: 386.72 ms/ques (linking tool: 305.89 ms/ques), relation detection: 836.66 ms/ques (similarity: 688.36 ms/ques), EDG generation: 52.04 ms/ques [INFO] Entity Linking P: 0.7484909456740443, R: 0.704225352112676, micro F1: 0.7183098591549301, macro F1: 0.7256837423432562 [INFO] Relation Linking P: 0.601945003353454, R: 0.5543259557344065, micro F1: 0.5671361502347415, macro F1: 0.5771549249090833 [INFO] Type Linking P: 0.5392354124748491, R: 0.5392354124748491, micro F1: 0.5392354124748491, macro F1: 0.5392354124748491 @@ -6970,8 +6970,8 @@ Question 497: What is manufactured by a company headed by Kazuo Hirai? [INFO] Golden Answer: [http://dbpedia.org/resource/Philips_CD-i, http://dbpedia.org/resource/Sony_NW-A810, http://dbpedia.org/resource/PlayStation_(console), http://dbpedia.org/resource/PlayStation_2, http://dbpedia.org/resource/PlayStation_3, http://dbpedia.org/resource/Sony_Xperia_E, http://dbpedia.org/resource/Walkman_X_Series, http://dbpedia.org/resource/Sony_CLIÉ_PEG-NR70V, http://dbpedia.org/resource/Sony_CLIÉ_PEG-SJ22, http://dbpedia.org/resource/Sony_CLIÉ_PEG-TJ35, http://dbpedia.org/resource/Sony_CLIÉ_PEG-TJ37, http://dbpedia.org/resource/PlayStation_4, http://dbpedia.org/resource/Walkman, http://dbpedia.org/resource/PlayStation_Portable, http://dbpedia.org/resource/PlayStation_Vita, http://dbpedia.org/resource/Sony_Reader, http://dbpedia.org/resource/DualShock, http://dbpedia.org/resource/Dual_Analog_Controller, http://dbpedia.org/resource/PSX_(video_game_console), http://dbpedia.org/resource/PlayStation_Controller, http://dbpedia.org/resource/PlayStation_Mouse, http://dbpedia.org/resource/PlayStation_models, http://dbpedia.org/resource/Sixaxis, http://dbpedia.org/resource/Mylo_(Sony), http://dbpedia.org/resource/Sony_SmartWatch, http://dbpedia.org/resource/Sony_NW-A800, http://dbpedia.org/resource/Walkman_S_Series, http://dbpedia.org/resource/PocketStation, http://dbpedia.org/resource/SNES-CD, http://dbpedia.org/resource/Sony_Tablet, http://dbpedia.org/resource/PlayStation_Analog_Joystick, http://dbpedia.org/resource/Sony_Xperia_Tablet_Z, http://dbpedia.org/resource/Sony_Dash, http://dbpedia.org/resource/CLIÉ, http://dbpedia.org/resource/Magic_Link, http://dbpedia.org/resource/Yahoo!_Connected_TV, http://dbpedia.org/resource/Sony_Xperia_Tablet_S, http://dbpedia.org/resource/Sony_Walkman_Z_Series, http://dbpedia.org/resource/Sony_Xperia_Z2_Tablet, http://dbpedia.org/resource/Sony_Xperia_Z3_Tablet_Compact, http://dbpedia.org/resource/Sony_Xperia_Z4_Tablet] [INFO] Predicted Answer: [http://dbpedia.org/resource/Philips_CD-i, http://dbpedia.org/resource/Sony_NW-A810, http://dbpedia.org/resource/PlayStation_(console), http://dbpedia.org/resource/PlayStation_2, http://dbpedia.org/resource/PlayStation_3, http://dbpedia.org/resource/Sony_Xperia_E, http://dbpedia.org/resource/Walkman_X_Series, http://dbpedia.org/resource/Sony_CLIÉ_PEG-NR70V, http://dbpedia.org/resource/Sony_CLIÉ_PEG-SJ22, http://dbpedia.org/resource/Sony_CLIÉ_PEG-TJ35, http://dbpedia.org/resource/Sony_CLIÉ_PEG-TJ37, http://dbpedia.org/resource/PlayStation_4, http://dbpedia.org/resource/Walkman, http://dbpedia.org/resource/PlayStation_Portable, http://dbpedia.org/resource/PlayStation_Vita, http://dbpedia.org/resource/Sony_Reader, http://dbpedia.org/resource/DualShock, http://dbpedia.org/resource/Dual_Analog_Controller, http://dbpedia.org/resource/PSX_(video_game_console), http://dbpedia.org/resource/PlayStation_Controller, http://dbpedia.org/resource/PlayStation_Mouse, http://dbpedia.org/resource/PlayStation_models, http://dbpedia.org/resource/Sixaxis, http://dbpedia.org/resource/Mylo_(Sony), http://dbpedia.org/resource/Sony_SmartWatch, http://dbpedia.org/resource/Sony_NW-A800, http://dbpedia.org/resource/Walkman_S_Series, http://dbpedia.org/resource/PocketStation, http://dbpedia.org/resource/SNES-CD, http://dbpedia.org/resource/Sony_Tablet, http://dbpedia.org/resource/PlayStation_Analog_Joystick, http://dbpedia.org/resource/Sony_Xperia_Tablet_Z, http://dbpedia.org/resource/Sony_Dash, http://dbpedia.org/resource/CLIÉ, http://dbpedia.org/resource/Magic_Link, http://dbpedia.org/resource/Yahoo!_Connected_TV, http://dbpedia.org/resource/Sony_Xperia_Tablet_S, http://dbpedia.org/resource/Sony_Walkman_Z_Series, http://dbpedia.org/resource/Sony_Xperia_Z2_Tablet, http://dbpedia.org/resource/Sony_Xperia_Z3_Tablet_Compact, http://dbpedia.org/resource/Sony_Xperia_Z4_Tablet, http://dbpedia.org/resource/AIBO, http://dbpedia.org/resource/QRIO, http://dbpedia.org/resource/Sony_Ericsson_Xperia_neo_V] [INFO] P: 0.9318181818181818, R: 1.0, F: 0.9647058823529412 -[INFO] Cumulative metrics, sample: 498, P: 0.521, R: 0.574, macro F1: 0.528, macro F1*: 0.546 -[INFO] Current time: 1767 ms, average: 2735.45 ms/ques, entity detection: 426.04 ms/ques (linking tool: 334.36 ms/ques), relation detection: 893.70 ms/ques (similarity: 701.62 ms/ques), EDG generation: 52.08 ms/ques +[INFO] Cumulative metrics, sample: 498, P: 0.521, R: 0.574, macro F1: 0.546 +[INFO] Current question consumed time: 1669 ms, average: 2508.17 ms/ques, entity detection: 386.18 ms/ques (linking tool: 305.42 ms/ques), relation detection: 836.55 ms/ques (similarity: 688.10 ms/ques), EDG generation: 51.98 ms/ques [INFO] Entity Linking P: 0.748995983935743, R: 0.7048192771084337, micro F1: 0.7188755020080326, macro F1: 0.7262364374625574 [INFO] Relation Linking P: 0.6017402945113788, R: 0.5542168674698795, micro F1: 0.5670013386880853, macro F1: 0.5770016952581606 [INFO] Type Linking P: 0.5381526104417671, R: 0.5381526104417671, micro F1: 0.5381526104417671, macro F1: 0.5381526104417671 @@ -6984,8 +6984,8 @@ Question 498: Under which presidents have politicians from the Continental army [INFO] Golden Answer: [http://dbpedia.org/resource/James_Madison, http://dbpedia.org/resource/Thomas_Jefferson, http://dbpedia.org/resource/George_Washington, http://dbpedia.org/resource/John_Adams, http://dbpedia.org/resource/James_Monroe, http://dbpedia.org/resource/John_Quincy_Adams] [INFO] Predicted Answer: [http://dbpedia.org/resource/George_Clinton_(vice_president), http://dbpedia.org/resource/George_Washington, http://dbpedia.org/resource/George_Washington_in_the_American_Revolution, http://dbpedia.org/resource/James_Monroe, http://dbpedia.org/resource/Aaron_Burr, http://dbpedia.org/resource/Commander-in-Chief's_Guard, http://dbpedia.org/resource/Elihu_Adams] [INFO] P: 0.2857142857142857, R: 0.3333333333333333, F: 0.30769230769230765 -[INFO] Cumulative metrics, sample: 499, P: 0.521, R: 0.573, macro F1: 0.528, macro F1*: 0.546 -[INFO] Current time: 1663 ms, average: 2733.30 ms/ques, entity detection: 425.77 ms/ques (linking tool: 334.09 ms/ques), relation detection: 893.07 ms/ques (similarity: 701.29 ms/ques), EDG generation: 52.00 ms/ques +[INFO] Cumulative metrics, sample: 499, P: 0.521, R: 0.573, macro F1: 0.546 +[INFO] Current question consumed time: 1523 ms, average: 2506.19 ms/ques, entity detection: 385.93 ms/ques (linking tool: 305.16 ms/ques), relation detection: 835.96 ms/ques (similarity: 687.81 ms/ques), EDG generation: 51.91 ms/ques [INFO] Entity Linking P: 0.749498997995992, R: 0.7054108216432866, micro F1: 0.7194388777555115, macro F1: 0.7267869071476287 [INFO] Relation Linking P: 0.6005344021376084, R: 0.5531062124248497, micro F1: 0.5658650634602536, macro F1: 0.5758453792355991 [INFO] Type Linking P: 0.5370741482965932, R: 0.5370741482965932, micro F1: 0.5370741482965932, macro F1: 0.5370741482965932 @@ -6998,8 +6998,8 @@ Question 499: Philadelphia City Council is the governing body of which city? [INFO] Golden Answer: [http://dbpedia.org/resource/Philadelphia] [INFO] Predicted Answer: [http://dbpedia.org/resource/Philadelphia] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 500, P: 0.522, R: 0.574, macro F1: 0.529, macro F1*: 0.547 -[INFO] Current time: 1179 ms, average: 2730.20 ms/ques, entity detection: 425.39 ms/ques (linking tool: 333.77 ms/ques), relation detection: 891.88 ms/ques (similarity: 700.62 ms/ques), EDG generation: 51.93 ms/ques +[INFO] Cumulative metrics, sample: 500, P: 0.522, R: 0.574, macro F1: 0.547 +[INFO] Current question consumed time: 1064 ms, average: 2503.31 ms/ques, entity detection: 385.55 ms/ques (linking tool: 304.82 ms/ques), relation detection: 834.87 ms/ques (similarity: 687.16 ms/ques), EDG generation: 51.83 ms/ques [INFO] Entity Linking P: 0.75, R: 0.706, micro F1: 0.7200000000000004, macro F1: 0.7273351648351648 [INFO] Relation Linking P: 0.6013333333333333, R: 0.554, micro F1: 0.566733333333333, macro F1: 0.5766970571263704 [INFO] Type Linking P: 0.536, R: 0.536, micro F1: 0.536, macro F1: 0.536 @@ -7012,8 +7012,8 @@ Question 500: What are the binomial authority of Menetries's warbler? [INFO] Golden Answer: [http://dbpedia.org/resource/Édouard_Ménétries, http://dbpedia.org/resource/Azerbaijan, http://dbpedia.org/resource/Kura_River, http://dbpedia.org/resource/Transcaucasia, http://dbpedia.org/resource/Sal'yana] [INFO] Predicted Answer: [http://dbpedia.org/resource/Carl_Jakob_Sundevall, Sundevall, 1860@en] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 501, P: 0.521, R: 0.573, macro F1: 0.528, macro F1*: 0.546 -[INFO] Current time: 835 ms, average: 2726.41 ms/ques, entity detection: 424.99 ms/ques (linking tool: 333.34 ms/ques), relation detection: 890.47 ms/ques (similarity: 699.45 ms/ques), EDG generation: 51.86 ms/ques +[INFO] Cumulative metrics, sample: 501, P: 0.521, R: 0.573, macro F1: 0.546 +[INFO] Current question consumed time: 749 ms, average: 2499.81 ms/ques, entity detection: 385.16 ms/ques (linking tool: 304.41 ms/ques), relation detection: 833.57 ms/ques (similarity: 686.01 ms/ques), EDG generation: 51.76 ms/ques [INFO] Entity Linking P: 0.7485029940119761, R: 0.7045908183632734, micro F1: 0.7185628742514975, macro F1: 0.7258833980390867 [INFO] Relation Linking P: 0.6021290751829673, R: 0.5548902195608783, micro F1: 0.5675981370592146, macro F1: 0.5775453119063769 [INFO] Type Linking P: 0.5369261477045908, R: 0.5369261477045908, micro F1: 0.5369261477045908, macro F1: 0.5369261477045908 @@ -7026,8 +7026,8 @@ Question 501: Name the movie which has music composed by Elliot Goldenthal and s [INFO] Golden Answer: [http://dbpedia.org/resource/Batman_Forever, http://dbpedia.org/resource/A_Time_to_Kill_(1996_film)] [INFO] Predicted Answer: [http://dbpedia.org/resource/Batman_Forever, http://dbpedia.org/resource/A_Time_to_Kill_(1996_film)] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 502, P: 0.522, R: 0.574, macro F1: 0.529, macro F1*: 0.547 -[INFO] Current time: 2025 ms, average: 2725.02 ms/ques, entity detection: 424.90 ms/ques (linking tool: 333.12 ms/ques), relation detection: 889.73 ms/ques (similarity: 699.10 ms/ques), EDG generation: 51.86 ms/ques +[INFO] Cumulative metrics, sample: 502, P: 0.522, R: 0.574, macro F1: 0.547 +[INFO] Current question consumed time: 1843 ms, average: 2498.50 ms/ques, entity detection: 385.02 ms/ques (linking tool: 304.19 ms/ques), relation detection: 832.87 ms/ques (similarity: 685.69 ms/ques), EDG generation: 51.78 ms/ques [INFO] Entity Linking P: 0.749003984063745, R: 0.7051792828685259, micro F1: 0.7191235059760961, macro F1: 0.7264312612563445 [INFO] Relation Linking P: 0.6029216467463479, R: 0.5557768924302788, micro F1: 0.5684594953519253, macro F1: 0.5783901642712815 [INFO] Type Linking P: 0.5358565737051793, R: 0.5358565737051793, micro F1: 0.5358565737051793, macro F1: 0.5358565737051793 @@ -7040,8 +7040,8 @@ Question 502: Name the fictional character whose relative are Duck family and Cl [INFO] Golden Answer: [http://dbpedia.org/resource/Donald_Duck] [INFO] Predicted Answer: [http://dbpedia.org/resource/Donald_Duck] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 503, P: 0.523, R: 0.575, macro F1: 0.530, macro F1*: 0.547 -[INFO] Current time: 1755 ms, average: 2723.09 ms/ques, entity detection: 424.84 ms/ques (linking tool: 332.98 ms/ques), relation detection: 888.67 ms/ques (similarity: 698.50 ms/ques), EDG generation: 51.82 ms/ques +[INFO] Cumulative metrics, sample: 503, P: 0.523, R: 0.575, macro F1: 0.547 +[INFO] Current question consumed time: 1596 ms, average: 2496.70 ms/ques, entity detection: 385.01 ms/ques (linking tool: 304.09 ms/ques), relation detection: 831.91 ms/ques (similarity: 685.14 ms/ques), EDG generation: 51.75 ms/ques [INFO] Entity Linking P: 0.7495029821073559, R: 0.705765407554672, micro F1: 0.7196819085487082, macro F1: 0.7269769361970255 [INFO] Relation Linking P: 0.6037110669317428, R: 0.5556660039761432, micro F1: 0.5686547382372429, macro F1: 0.5786930318631226 [INFO] Type Linking P: 0.5347912524850894, R: 0.5347912524850894, micro F1: 0.5347912524850894, macro F1: 0.5347912524850894 @@ -7054,8 +7054,8 @@ Question 503: Which university with athletics department in NCAA Division I Foot [INFO] Golden Answer: [] [INFO] Predicted Answer: [http://dbpedia.org/resource/Tulane_University] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 504, P: 0.522, R: 0.574, macro F1: 0.529, macro F1*: 0.546 -[INFO] Current time: 4323 ms, average: 2726.26 ms/ques, entity detection: 424.88 ms/ques (linking tool: 332.93 ms/ques), relation detection: 890.64 ms/ques (similarity: 699.27 ms/ques), EDG generation: 51.80 ms/ques +[INFO] Cumulative metrics, sample: 504, P: 0.522, R: 0.574, macro F1: 0.546 +[INFO] Current question consumed time: 3878 ms, average: 2499.44 ms/ques, entity detection: 384.93 ms/ques (linking tool: 303.95 ms/ques), relation detection: 833.82 ms/ques (similarity: 685.92 ms/ques), EDG generation: 51.74 ms/ques [INFO] Entity Linking P: 0.748015873015873, R: 0.7043650793650794, micro F1: 0.7182539682539687, macro F1: 0.725534521641079 [INFO] Relation Linking P: 0.6025132275132274, R: 0.5545634920634921, micro F1: 0.5675264550264547, macro F1: 0.5775448314030767 [INFO] Type Linking P: 0.5357142857142857, R: 0.5357142857142857, micro F1: 0.5357142857142857, macro F1: 0.5357142857142857 @@ -7068,8 +7068,8 @@ Question 504: Which state does Jerome B. Chaffee belong ? [INFO] Golden Answer: [http://dbpedia.org/resource/Colorado, Colorado@en, Colorado Territory@en] [INFO] Predicted Answer: [http://dbpedia.org/resource/Colorado, Colorado@en, Colorado Territory@en] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 505, P: 0.523, R: 0.575, macro F1: 0.530, macro F1*: 0.547 -[INFO] Current time: 1079 ms, average: 2723.00 ms/ques, entity detection: 424.31 ms/ques (linking tool: 332.43 ms/ques), relation detection: 889.43 ms/ques (similarity: 698.50 ms/ques), EDG generation: 51.71 ms/ques +[INFO] Cumulative metrics, sample: 505, P: 0.523, R: 0.575, macro F1: 0.547 +[INFO] Current question consumed time: 985 ms, average: 2496.45 ms/ques, entity detection: 384.41 ms/ques (linking tool: 303.51 ms/ques), relation detection: 832.67 ms/ques (similarity: 685.13 ms/ques), EDG generation: 51.64 ms/ques [INFO] Entity Linking P: 0.7485148514851485, R: 0.7049504950495049, micro F1: 0.7188118811881192, macro F1: 0.7260798014406344 [INFO] Relation Linking P: 0.6033003300330032, R: 0.5554455445544555, micro F1: 0.5683828382838281, macro F1: 0.5783847652780089 [INFO] Type Linking P: 0.5366336633663367, R: 0.5366336633663367, micro F1: 0.5366336633663367, macro F1: 0.5366336633663367 @@ -7082,8 +7082,8 @@ Question 505: In which of the parties that have governed over Rishikesh, does Go [INFO] Golden Answer: [http://dbpedia.org/resource/Indian_National_Congress] [INFO] Predicted Answer: [http://dbpedia.org/resource/Indian_National_Congress, Deep Sharma@en] [INFO] P: 0.5, R: 1.0, F: 0.6666666666666666 -[INFO] Cumulative metrics, sample: 506, P: 0.523, R: 0.575, macro F1: 0.530, macro F1*: 0.548 -[INFO] Current time: 2967 ms, average: 2723.48 ms/ques, entity detection: 424.01 ms/ques (linking tool: 332.14 ms/ques), relation detection: 889.91 ms/ques (similarity: 699.72 ms/ques), EDG generation: 51.67 ms/ques +[INFO] Cumulative metrics, sample: 506, P: 0.523, R: 0.575, macro F1: 0.548 +[INFO] Current question consumed time: 2694 ms, average: 2496.84 ms/ques, entity detection: 384.12 ms/ques (linking tool: 303.22 ms/ques), relation detection: 833.13 ms/ques (similarity: 686.43 ms/ques), EDG generation: 51.60 ms/ques [INFO] Entity Linking P: 0.7490118577075099, R: 0.7045454545454546, micro F1: 0.71870882740448, macro F1: 0.7260985105988506 [INFO] Relation Linking P: 0.6040843214756257, R: 0.5553359683794467, micro F1: 0.5685770750988139, macro F1: 0.5786853215954006 [INFO] Type Linking P: 0.5375494071146245, R: 0.5375494071146245, micro F1: 0.5375494071146245, macro F1: 0.5375494071146245 @@ -7096,8 +7096,8 @@ Question 506: What awards are won by the screenwriter of Lucky You? [INFO] Golden Answer: [http://dbpedia.org/resource/Academy_Awards, http://dbpedia.org/resource/Laurel_Award_for_Screenwriting_Achievement] [INFO] Predicted Answer: [http://dbpedia.org/resource/Academy_Awards, http://dbpedia.org/resource/Laurel_Award_for_Screenwriting_Achievement] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 507, P: 0.523, R: 0.576, macro F1: 0.531, macro F1*: 0.549 -[INFO] Current time: 1774 ms, average: 2721.61 ms/ques, entity detection: 423.51 ms/ques (linking tool: 331.72 ms/ques), relation detection: 889.57 ms/ques (similarity: 699.46 ms/ques), EDG generation: 51.74 ms/ques +[INFO] Cumulative metrics, sample: 507, P: 0.523, R: 0.576, macro F1: 0.549 +[INFO] Current question consumed time: 1556 ms, average: 2494.98 ms/ques, entity detection: 383.63 ms/ques (linking tool: 302.81 ms/ques), relation detection: 832.77 ms/ques (similarity: 686.19 ms/ques), EDG generation: 51.68 ms/ques [INFO] Entity Linking P: 0.7495069033530573, R: 0.7051282051282052, micro F1: 0.719263642340566, macro F1: 0.7266405910473709 [INFO] Relation Linking P: 0.60387902695595, R: 0.5552268244575936, micro F1: 0.5684418145956605, macro F1: 0.5785318641682358 [INFO] Type Linking P: 0.5364891518737672, R: 0.5364891518737672, micro F1: 0.5364891518737672, macro F1: 0.5364891518737672 @@ -7110,8 +7110,8 @@ Question 507: Name the Pole driver of 1994 Spanish Grand Prix ? [INFO] Golden Answer: [http://dbpedia.org/resource/Michael_Schumacher] [INFO] Predicted Answer: [http://dbpedia.org/resource/Michael_Schumacher] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 508, P: 0.524, R: 0.577, macro F1: 0.532, macro F1*: 0.549 -[INFO] Current time: 1068 ms, average: 2718.35 ms/ques, entity detection: 422.91 ms/ques (linking tool: 331.18 ms/ques), relation detection: 888.68 ms/ques (similarity: 698.61 ms/ques), EDG generation: 51.65 ms/ques +[INFO] Cumulative metrics, sample: 508, P: 0.524, R: 0.577, macro F1: 0.549 +[INFO] Current question consumed time: 896 ms, average: 2491.83 ms/ques, entity detection: 383.04 ms/ques (linking tool: 302.29 ms/ques), relation detection: 831.91 ms/ques (similarity: 685.35 ms/ques), EDG generation: 51.60 ms/ques [INFO] Entity Linking P: 0.75, R: 0.7057086614173228, micro F1: 0.7198162729658798, macro F1: 0.7271805273833672 [INFO] Relation Linking P: 0.6046587926509186, R: 0.5561023622047244, micro F1: 0.5692913385826769, macro F1: 0.5793649822178107 [INFO] Type Linking P: 0.5374015748031497, R: 0.5374015748031497, micro F1: 0.5374015748031497, macro F1: 0.5374015748031497 @@ -7124,8 +7124,8 @@ Question 508: Where did Georges Wolinski and Judson Huss die/ [INFO] Golden Answer: [http://dbpedia.org/resource/France, http://dbpedia.org/resource/Paris] [INFO] Predicted Answer: [http://dbpedia.org/resource/France, http://dbpedia.org/resource/Paris] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 509, P: 0.525, R: 0.578, macro F1: 0.533, macro F1*: 0.550 -[INFO] Current time: 1121 ms, average: 2715.22 ms/ques, entity detection: 422.77 ms/ques (linking tool: 330.89 ms/ques), relation detection: 887.23 ms/ques (similarity: 697.40 ms/ques), EDG generation: 51.60 ms/ques +[INFO] Cumulative metrics, sample: 509, P: 0.525, R: 0.578, macro F1: 0.550 +[INFO] Current question consumed time: 870 ms, average: 2488.65 ms/ques, entity detection: 382.77 ms/ques (linking tool: 301.94 ms/ques), relation detection: 830.54 ms/ques (similarity: 684.15 ms/ques), EDG generation: 51.55 ms/ques [INFO] Entity Linking P: 0.75049115913556, R: 0.7053045186640472, micro F1: 0.7197118533071387, macro F1: 0.7271965617534845 [INFO] Relation Linking P: 0.6054354944335297, R: 0.555992141453831, micro F1: 0.5694826457105433, macro F1: 0.5796613868328918 [INFO] Type Linking P: 0.5383104125736738, R: 0.5383104125736738, micro F1: 0.5383104125736738, macro F1: 0.5383104125736738 @@ -7138,8 +7138,8 @@ Question 509: What is the parent company of the airline whose hub is in Detroit? [INFO] Golden Answer: [http://dbpedia.org/resource/Ford_Motor_Company] [INFO] Predicted Answer: [http://dbpedia.org/resource/Ford_Motor_Company] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 510, P: 0.526, R: 0.579, macro F1: 0.534, macro F1*: 0.551 -[INFO] Current time: 5967 ms, average: 2721.59 ms/ques, entity detection: 423.65 ms/ques (linking tool: 331.73 ms/ques), relation detection: 889.79 ms/ques (similarity: 700.43 ms/ques), EDG generation: 51.55 ms/ques +[INFO] Cumulative metrics, sample: 510, P: 0.526, R: 0.579, macro F1: 0.551 +[INFO] Current question consumed time: 5140 ms, average: 2493.85 ms/ques, entity detection: 383.34 ms/ques (linking tool: 302.52 ms/ques), relation detection: 832.74 ms/ques (similarity: 687.18 ms/ques), EDG generation: 51.51 ms/ques [INFO] Entity Linking P: 0.7509803921568627, R: 0.7058823529411765, micro F1: 0.7202614379084973, macro F1: 0.727733354445412 [INFO] Relation Linking P: 0.6062091503267973, R: 0.5568627450980392, micro F1: 0.5703267973856208, macro F1: 0.5804891217515386 [INFO] Type Linking P: 0.5372549019607843, R: 0.5372549019607843, micro F1: 0.5372549019607843, macro F1: 0.5372549019607843 @@ -7152,8 +7152,8 @@ Question 510: What is the appliance which uses the central processing unit manuf [INFO] Golden Answer: [http://dbpedia.org/resource/Amazon_Kindle, http://dbpedia.org/resource/Toshiba_g500] [INFO] Predicted Answer: [http://dbpedia.org/resource/Ubuntu_(operating_system), http://dbpedia.org/resource/Debian] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 511, P: 0.525, R: 0.578, macro F1: 0.533, macro F1*: 0.550 -[INFO] Current time: 1988 ms, average: 2720.16 ms/ques, entity detection: 423.54 ms/ques (linking tool: 331.58 ms/ques), relation detection: 889.15 ms/ques (similarity: 699.91 ms/ques), EDG generation: 51.50 ms/ques +[INFO] Cumulative metrics, sample: 511, P: 0.525, R: 0.578, macro F1: 0.550 +[INFO] Current question consumed time: 1828 ms, average: 2492.54 ms/ques, entity detection: 383.26 ms/ques (linking tool: 302.39 ms/ques), relation detection: 832.18 ms/ques (similarity: 686.61 ms/ques), EDG generation: 51.43 ms/ques [INFO] Entity Linking P: 0.7514677103718199, R: 0.7064579256360078, micro F1: 0.7208088714938036, macro F1: 0.7282680360918846 [INFO] Relation Linking P: 0.6050228310502282, R: 0.5557729941291585, micro F1: 0.5692106979778211, macro F1: 0.5793531352119071 [INFO] Type Linking P: 0.5362035225048923, R: 0.5362035225048923, micro F1: 0.5362035225048923, macro F1: 0.5362035225048923 @@ -7166,8 +7166,8 @@ Question 511: Give me all magazines whose founder died in Berkeley, California? [INFO] Golden Answer: [http://dbpedia.org/resource/Rolling_Stone] [INFO] Predicted Answer: [http://dbpedia.org/resource/Adolphe_Menjou, http://dbpedia.org/resource/Andy_Clyde, http://dbpedia.org/resource/Barbara_Jo_Allen, http://dbpedia.org/resource/Bernard_Brodie_(military_strategist), http://dbpedia.org/resource/Bill_Bixby, http://dbpedia.org/resource/Billy_Gilbert, http://dbpedia.org/resource/David_Groh, http://dbpedia.org/resource/E._Hoffmann_Price, http://dbpedia.org/resource/Eddie_Quillan, http://dbpedia.org/resource/Ernie_Anderson, http://dbpedia.org/resource/Ernie_Kovacs, http://dbpedia.org/resource/Estelle_Getty, http://dbpedia.org/resource/Frank_Tashlin, http://dbpedia.org/resource/Franklin_Pangborn, http://dbpedia.org/resource/George_Emil_Palade, http://dbpedia.org/resource/Gilda_Radner, http://dbpedia.org/resource/Harold_Ritz, http://dbpedia.org/resource/Harry_Langdon, http://dbpedia.org/resource/Henry_Brant, http://dbpedia.org/resource/Jerry_Colonna_(entertainer), http://dbpedia.org/resource/Jerzy_Neyman, http://dbpedia.org/resource/Jimmy_Durante, http://dbpedia.org/resource/Jimmy_Ritz_(comedian), http://dbpedia.org/resource/Joaquin_Miller, http://dbpedia.org/resource/Joe_E._Brown, http://dbpedia.org/resource/John_Carter_(jazz_musician), http://dbpedia.org/resource/John_Lounsbery, http://dbpedia.org/resource/Joseph_Barbera, http://dbpedia.org/resource/Juan_Bautista_Sacasa, http://dbpedia.org/resource/Judi_Bari, http://dbpedia.org/resource/Julia_Faye, http://dbpedia.org/resource/Larry_Semon, http://dbpedia.org/resource/Leo_Gorcey, http://dbpedia.org/resource/Leon_Errol, http://dbpedia.org/resource/Mako_Iwamatsu, http://dbpedia.org/resource/Mel_Blanc, http://dbpedia.org/resource/Moe_Howard, http://dbpedia.org/resource/Morey_Amsterdam, http://dbpedia.org/resource/Phil_Hartman, http://dbpedia.org/resource/Redd_Foxx, http://dbpedia.org/resource/Roger_J._Traynor, http://dbpedia.org/resource/Sam_Simon, http://dbpedia.org/resource/Ted_Healy, http://dbpedia.org/resource/Ted_Knight, http://dbpedia.org/resource/Terence_MacManus, http://dbpedia.org/resource/Theodore_Roberts, http://dbpedia.org/resource/Vance_Colvig, http://dbpedia.org/resource/William_Hanna, http://dbpedia.org/resource/Woody_Herman, http://dbpedia.org/resource/Alan_Campbell_(screenwriter), http://dbpedia.org/resource/Antony_Alda, http://dbpedia.org/resource/Ben_Weisman, http://dbpedia.org/resource/Bill_Belew, http://dbpedia.org/resource/Bob_Carroll,_Jr., http://dbpedia.org/resource/Brittany_Murphy, http://dbpedia.org/resource/Cliff_Osmond, http://dbpedia.org/resource/Dan_Blocker, http://dbpedia.org/resource/Dan_Hicks_(singer), http://dbpedia.org/resource/Dennis_Morgan, http://dbpedia.org/resource/Edward_LeSaint, http://dbpedia.org/resource/Elias_Disney, http://dbpedia.org/resource/Gene_Barry, http://dbpedia.org/resource/Hans_Suess, http://dbpedia.org/resource/Helen_Palmer_Geisel, http://dbpedia.org/resource/James_Moody_(saxophonist), http://dbpedia.org/resource/Jean_Parker, http://dbpedia.org/resource/Jeanne_Crain, http://dbpedia.org/resource/Jeffrey_Hunter, http://dbpedia.org/resource/Jimmie_F._Skaggs, http://dbpedia.org/resource/Jimmy_Blanton, http://dbpedia.org/resource/Joe_Mantell, http://dbpedia.org/resource/John_Braheny, http://dbpedia.org/resource/John_Robert_Beyster, http://dbpedia.org/resource/John_Spence_(musician), http://dbpedia.org/resource/Karen_Morley, http://dbpedia.org/resource/Karl_Malden, http://dbpedia.org/resource/Katherine_MacDonald, http://dbpedia.org/resource/Keenan_Wynn, http://dbpedia.org/resource/Kenneth_Patchen, http://dbpedia.org/resource/Leslie_Orgel, http://dbpedia.org/resource/Lewis_Arquette, http://dbpedia.org/resource/Mala_Powers, http://dbpedia.org/resource/Mark_Peattie, http://dbpedia.org/resource/Mary_Jackson_(actress), http://dbpedia.org/resource/May_Robson, http://dbpedia.org/resource/Michael_Clarke_Duncan, http://dbpedia.org/resource/Nick_Adams_(actor), http://dbpedia.org/resource/Norman_D._Wilson_(actor), http://dbpedia.org/resource/Paul_Winchell, http://dbpedia.org/resource/Ralph_Rainger, http://dbpedia.org/resource/Raoul_Bott, http://dbpedia.org/resource/Red_Rhodes, http://dbpedia.org/resource/Richard_D._Zanuck, http://dbpedia.org/resource/Rory_Markas, http://dbpedia.org/resource/Sam_Jaffe, http://dbpedia.org/resource/Sean_McClory, http://dbpedia.org/resource/Sidney_Miller_(actor), http://dbpedia.org/resource/Tom_Bradley_(American_politician), http://dbpedia.org/resource/Tommy_Rettig, http://dbpedia.org/resource/Torin_Thatcher, http://dbpedia.org/resource/William_A._Wellman, http://dbpedia.org/resource/Arthur_Marx, http://dbpedia.org/resource/Dan_White, http://dbpedia.org/resource/Dean_Grennell, http://dbpedia.org/resource/Felicia_Tang, http://dbpedia.org/resource/Herbert_Marshall, http://dbpedia.org/resource/Joe_Farrell, http://dbpedia.org/resource/John_Darrow, http://dbpedia.org/resource/Lowell_Sherman, http://dbpedia.org/resource/Maria_Rasputin, http://dbpedia.org/resource/Minnie_Riperton, http://dbpedia.org/resource/Philip_Merivale, http://dbpedia.org/resource/Richard_Cromwell_(actor), http://dbpedia.org/resource/Ron_%22Pigpen%22_McKernan, http://dbpedia.org/resource/Sam_Marx, http://dbpedia.org/resource/Sam_Wood, http://dbpedia.org/resource/William_Goetz, http://dbpedia.org/resource/William_J._Seymour, http://dbpedia.org/resource/William_Powell, http://dbpedia.org/resource/Glen_A._Larson, http://dbpedia.org/resource/Herbert_Brenon, http://dbpedia.org/resource/John_Joseph_Ford, http://dbpedia.org/resource/Alan_O'Day, http://dbpedia.org/resource/Andrew_Martinez, http://dbpedia.org/resource/Bill_Pickering_(rocket_scientist), http://dbpedia.org/resource/Bob_Nolan, http://dbpedia.org/resource/Bob_Thompson_(musician), http://dbpedia.org/resource/Charles_Lewis,_Jr., http://dbpedia.org/resource/Christian_Brando, http://dbpedia.org/resource/Dolla_(rapper), http://dbpedia.org/resource/Eb_Gaines, http://dbpedia.org/resource/Eduardo_Palomo, http://dbpedia.org/resource/Eric_Blore, http://dbpedia.org/resource/Eric_Krenz, http://dbpedia.org/resource/Frank_Marocco, http://dbpedia.org/resource/Frank_de_Kova, http://dbpedia.org/resource/Freddie_Perren, http://dbpedia.org/resource/George_Aratani, http://dbpedia.org/resource/Hal_Needham, http://dbpedia.org/resource/Harold_J._Stone, http://dbpedia.org/resource/Henry_Corden, http://dbpedia.org/resource/Herb_Peterson, http://dbpedia.org/resource/Jocelyn_Brando, http://dbpedia.org/resource/John_Marley, http://dbpedia.org/resource/José_Manuel_Rodriguez_Delgado, http://dbpedia.org/resource/Lewis_Collins, http://dbpedia.org/resource/Lincoln_J._Beachey, http://dbpedia.org/resource/Linden_Chiles, http://dbpedia.org/resource/Lorenzo_Sawyer, http://dbpedia.org/resource/Michael_Rye, http://dbpedia.org/resource/Patrick_Suppes, http://dbpedia.org/resource/Peter_Whitney, http://dbpedia.org/resource/Raymond_Telles, http://dbpedia.org/resource/Robert_Gist, http://dbpedia.org/resource/Robert_Sterling, http://dbpedia.org/resource/Romaine_Fielding, http://dbpedia.org/resource/Ross_Hunter, http://dbpedia.org/resource/Scotty_Beckett, http://dbpedia.org/resource/Tim_Asch, http://dbpedia.org/resource/Al_Silvani, http://dbpedia.org/resource/Bing_Russell, http://dbpedia.org/resource/Clarence_M._Pendleton,_Jr., http://dbpedia.org/resource/Dave_Barbour, http://dbpedia.org/resource/Donald_DeFreeze, http://dbpedia.org/resource/George_Chandler, http://dbpedia.org/resource/George_Christopher, http://dbpedia.org/resource/Herschel_Burke_Gilbert, http://dbpedia.org/resource/James_Philbrook, http://dbpedia.org/resource/Jimmie_Mattern, http://dbpedia.org/resource/John_Smith_(actor), http://dbpedia.org/resource/L._Wolfe_Gilbert, http://dbpedia.org/resource/Laurence_J._Rittenband, http://dbpedia.org/resource/Louise_Carver, http://dbpedia.org/resource/Marthe_Vogt, http://dbpedia.org/resource/Michael_Wenning, http://dbpedia.org/resource/Nat_Pendleton, http://dbpedia.org/resource/Oliver_Payne_Pearson, http://dbpedia.org/resource/Patrick_J._Hillings, http://dbpedia.org/resource/Peter_Shaw_(producer), http://dbpedia.org/resource/Simeon_Gannett_Reed, http://dbpedia.org/resource/Albert_Dekker, http://dbpedia.org/resource/Bryan_Gregory, http://dbpedia.org/resource/Charles_Hallahan, http://dbpedia.org/resource/Ian_Wolfe, http://dbpedia.org/resource/John_B._Parkin, http://dbpedia.org/resource/John_Pearce_(actor), http://dbpedia.org/resource/Orlando_Anderson, http://dbpedia.org/resource/Walter_Coy, http://dbpedia.org/resource/Walter_Hampden, http://dbpedia.org/resource/John_Reynolds_(physicist), http://dbpedia.org/resource/Lee_Reherman, http://dbpedia.org/resource/Paul_Jabara, http://dbpedia.org/resource/Schlitzie, http://dbpedia.org/resource/Stephen_Lee_(actor), http://dbpedia.org/resource/Tom_Tully, http://dbpedia.org/resource/Hugh_Daniel, http://dbpedia.org/resource/James_Smoot_Coleman, http://dbpedia.org/resource/Lorenda_Starfelt, http://dbpedia.org/resource/Nao_Takasugi, http://dbpedia.org/resource/Roscoe_G._Dickinson, http://dbpedia.org/resource/Arthur_Bernard_Lewis, http://dbpedia.org/resource/Charles_H._Veeder, http://dbpedia.org/resource/Dark_Cloud_(actor), http://dbpedia.org/resource/Horace_F._Page, http://dbpedia.org/resource/Ilya_Zhitomirskiy, http://dbpedia.org/resource/J._E._Freeman, http://dbpedia.org/resource/Ross_Russell, http://dbpedia.org/resource/George_Cates, http://dbpedia.org/resource/Hans_Hermann_Behr, http://dbpedia.org/resource/Isaac_Roop, http://dbpedia.org/resource/Jack_Hively, http://dbpedia.org/resource/Vicente_Francisco_de_Sarría, http://dbpedia.org/resource/Aurania_Rouverol, http://dbpedia.org/resource/Benedicte_Wrensted, http://dbpedia.org/resource/Bernard_Wolf, http://dbpedia.org/resource/Beverly_Long, http://dbpedia.org/resource/Burton_E._Green, http://dbpedia.org/resource/Charles_Van_Enger, http://dbpedia.org/resource/Chief_Yowlachie, http://dbpedia.org/resource/Clara_Beranger, http://dbpedia.org/resource/Dave_Barry_(actor), http://dbpedia.org/resource/Douglas_Spencer, http://dbpedia.org/resource/Edward_G._Robinson_Jr., http://dbpedia.org/resource/Frank_Fenton_(writer), http://dbpedia.org/resource/Frank_McGrath_(actor), http://dbpedia.org/resource/George_Bassman, http://dbpedia.org/resource/George_Loane_Tucker, http://dbpedia.org/resource/John_Sheehan_(actor), http://dbpedia.org/resource/Lynn_Reynolds, http://dbpedia.org/resource/Maxine_Elliott_Hicks, http://dbpedia.org/resource/Pal_(dog), http://dbpedia.org/resource/Patrick_Joseph_Sullivan, http://dbpedia.org/resource/Richard_Webb_(actor), http://dbpedia.org/resource/Robert_DoQui, http://dbpedia.org/resource/Scotty_Mattraw, http://dbpedia.org/resource/Tim_Whelan, http://dbpedia.org/resource/Virginia_Huston, http://dbpedia.org/resource/William_Gillespie_(actor), http://dbpedia.org/resource/Edith_Massey, http://dbpedia.org/resource/Hazel_Court, http://dbpedia.org/resource/Al_Bertino, http://dbpedia.org/resource/Doug_Weston, http://dbpedia.org/resource/Edward_H._Schafer, http://dbpedia.org/resource/Gordon_Novel, http://dbpedia.org/resource/Irving_J._Moore, http://dbpedia.org/resource/Muriel_Landers, http://dbpedia.org/resource/Paul_Sorensen, http://dbpedia.org/resource/Tom_Fuccello, http://dbpedia.org/resource/Vernard_Eller, http://dbpedia.org/resource/Edwin_Blum, http://dbpedia.org/resource/Gordon_Jones_(actor), http://dbpedia.org/resource/Hugh_Sanders, http://dbpedia.org/resource/Leonard_B._Stern, http://dbpedia.org/resource/Paul_Gilbert_(actor), http://dbpedia.org/resource/Caroline_Decker, http://dbpedia.org/resource/Clark_Glasson, http://dbpedia.org/resource/Franz_Schurmann, http://dbpedia.org/resource/Harry_Lauter, http://dbpedia.org/resource/Jim_Boggio, http://dbpedia.org/resource/Myron_Healey, http://dbpedia.org/resource/Rodney_Walker_(architect), http://dbpedia.org/resource/Ab_Saunders, http://dbpedia.org/resource/Archie_San_Romani, http://dbpedia.org/resource/Belle_Benchley, http://dbpedia.org/resource/Bernie_West, http://dbpedia.org/resource/George_Pepperdine, http://dbpedia.org/resource/Hal_Gibney, http://dbpedia.org/resource/Joan_Gardner, http://dbpedia.org/resource/John_Mills_Houston, http://dbpedia.org/resource/Joseph_R._Burton, http://dbpedia.org/resource/Lamon_V._Harkness, http://dbpedia.org/resource/Leo_Thomas_Maher, http://dbpedia.org/resource/Lowell_Gilmore, http://dbpedia.org/resource/Mildred_Lillie, http://dbpedia.org/resource/Ralph_Leroy_Nafziger, http://dbpedia.org/resource/Stacy_Harris, http://dbpedia.org/resource/Steve_Darrell, http://dbpedia.org/resource/William_P._Burnham, http://dbpedia.org/resource/William_Ward_Johnson, http://dbpedia.org/resource/Charles_Fredericks_(actor), http://dbpedia.org/resource/Lois_Hall, http://dbpedia.org/resource/Lucille_Bogan, http://dbpedia.org/resource/Buzz_Kulik, http://dbpedia.org/resource/Edmund_Kara, http://dbpedia.org/resource/Elwood_L._Haines, http://dbpedia.org/resource/John_A._Ferraro, http://dbpedia.org/resource/Ted_Laux, http://dbpedia.org/resource/Aylett_R._Cotton, http://dbpedia.org/resource/Badja_Djola, http://dbpedia.org/resource/Coit_Albertson, http://dbpedia.org/resource/Gloria_Foy, http://dbpedia.org/resource/Harriet_Taylor_Upton, http://dbpedia.org/resource/John_Schlee, http://dbpedia.org/resource/Joseph_Fleck, http://dbpedia.org/resource/Kirtland_I._Perky, http://dbpedia.org/resource/Ruggero_J._Aldisert, http://dbpedia.org/resource/Judson_Pratt, http://dbpedia.org/resource/Muriel_Evans, http://dbpedia.org/resource/Richard_Bullock, http://dbpedia.org/resource/Darrell_Sandeen, http://dbpedia.org/resource/Ge_Ge_Pearson, http://dbpedia.org/resource/Jean-Yves_Escoffier, http://dbpedia.org/resource/John_Burnside_(inventor), http://dbpedia.org/resource/Peter_Bromilow, http://dbpedia.org/resource/Billy_Booth_(actor), http://dbpedia.org/resource/Constance_Moore, http://dbpedia.org/resource/Phillip_Burton, http://dbpedia.org/resource/Sarah_A._Bowman, http://dbpedia.org/resource/Sigrid_Valdis, http://dbpedia.org/resource/Catya_Sassoon, http://dbpedia.org/resource/Christian_Theodore_Pedersen, http://dbpedia.org/resource/Douglas_Wood_(actor), http://dbpedia.org/resource/Ella_Hill_Hutch, http://dbpedia.org/resource/Fred_Santley, http://dbpedia.org/resource/John_Lourie_Beveridge, http://dbpedia.org/resource/Marion_Jackson, http://dbpedia.org/resource/Matthew_Ansara, http://dbpedia.org/resource/Peter_Godfrey_(director), http://dbpedia.org/resource/Tudor_Owen_(actor), http://dbpedia.org/resource/West_Arkeen, http://dbpedia.org/resource/Del_Porter, http://dbpedia.org/resource/Jah_Paul_Jo, http://dbpedia.org/resource/Jay_Sommers, http://dbpedia.org/resource/Pat_Close, http://dbpedia.org/resource/Robert_Livingston_(scientist), http://dbpedia.org/resource/Russell_Thorson, http://dbpedia.org/resource/Scott_Smith_(activist), http://dbpedia.org/resource/Charles_Garry, http://dbpedia.org/resource/J._Warren_Madden, http://dbpedia.org/resource/Gordon_Roberts_(ice_hockey), http://dbpedia.org/resource/Michael_A._Healy, http://dbpedia.org/resource/Yale_Gracey, http://dbpedia.org/resource/Christopher_Dark, http://dbpedia.org/resource/Jeremiah_James_Harty, http://dbpedia.org/resource/Joyce_Blair, http://dbpedia.org/resource/Kim_Spalding, http://dbpedia.org/resource/Rosemary_Theby, http://dbpedia.org/resource/Vivion_Brewer, http://dbpedia.org/resource/Willie_Hunter_(golfer), http://dbpedia.org/resource/Adrian_Spies, http://dbpedia.org/resource/Benjamin_Mayfield, http://dbpedia.org/resource/Jacob_Matijevic, http://dbpedia.org/resource/Louis_Lesser, http://dbpedia.org/resource/Kenny_Friedkin, http://dbpedia.org/resource/Ronald_Long, http://dbpedia.org/resource/William_Irvine_(lawyer), http://dbpedia.org/resource/Claudio_Guzmán, http://dbpedia.org/resource/Lafayette_Guild, http://dbpedia.org/resource/Buster_Millerick, http://dbpedia.org/resource/Al_Aarons, http://dbpedia.org/resource/Andrew_MacNaughtan, http://dbpedia.org/resource/Horace_R._Byers, http://dbpedia.org/resource/Jeri_Southern, http://dbpedia.org/resource/Luther_Tucker, http://dbpedia.org/resource/Thomas_B._Klakring, http://dbpedia.org/resource/Edwin_S._Shneidman, http://dbpedia.org/resource/Michael_Peters, http://dbpedia.org/resource/Philip_Saffman, http://dbpedia.org/resource/Sarah_Cunningham_(actress), http://dbpedia.org/resource/Bob_Finkel, http://dbpedia.org/resource/Howard_Kinsey, http://dbpedia.org/resource/Les_Damon, http://dbpedia.org/resource/Richard_Gordon_(photographer), http://dbpedia.org/resource/Albert_DeMond, http://dbpedia.org/resource/Don_Sleet, http://dbpedia.org/resource/Howard_Gilkey, http://dbpedia.org/resource/Robert_B._Williams_(actor), http://dbpedia.org/resource/Washington_Lafayette_Elliott, http://dbpedia.org/resource/Ron_Maudsley, http://dbpedia.org/resource/Sloan_Nibley, http://dbpedia.org/resource/George_E._White_(missionary), http://dbpedia.org/resource/Virginia_Sale, http://dbpedia.org/resource/Charles_F._Miller, http://dbpedia.org/resource/Joanna_Lee_(writer), http://dbpedia.org/resource/Lewis_D._Thill, http://dbpedia.org/resource/Louis_Stevens_(writer), http://dbpedia.org/resource/Seth_Roberts, http://dbpedia.org/resource/Adda_Gleason, http://dbpedia.org/resource/Clarence_Shaw, http://dbpedia.org/resource/Martin_Fierro_(saxophonist), http://dbpedia.org/resource/Marshall_Rohner, http://dbpedia.org/resource/Jerry_Warren, http://dbpedia.org/resource/Reginald_Golledge, http://dbpedia.org/resource/Thomas_Morris_(musician), http://dbpedia.org/resource/Bob_Wilson_(politician), http://dbpedia.org/resource/George_Snook, http://dbpedia.org/resource/Marilyn_Jorgenson_Reece, http://dbpedia.org/resource/Albert_Cavens, http://dbpedia.org/resource/Carlos_Moorhead, http://dbpedia.org/resource/André_Cheron_(actor), http://dbpedia.org/resource/Houseley_Stevenson, http://dbpedia.org/resource/Lloyd_Monserratt, http://dbpedia.org/resource/Samuel_Bradford_Caswell, http://dbpedia.org/resource/Hasan_Dosti, http://dbpedia.org/resource/Mai_Kitazawa_Arbegast, http://dbpedia.org/resource/Mickey_Cafagna, http://dbpedia.org/resource/Donald_Briggs, http://dbpedia.org/resource/John_W._Boyle, http://dbpedia.org/resource/Philip_Cahn, http://dbpedia.org/resource/Robert_M._Leeds, http://dbpedia.org/resource/Sigi_Ziering, http://dbpedia.org/resource/Don_Ross_(bodybuilder), http://dbpedia.org/resource/Ethel_Hill, http://dbpedia.org/resource/Kevin_Gaines_(police_officer), http://dbpedia.org/resource/Walter_Abraham_Jacobs, http://dbpedia.org/resource/Otis_Gibson, http://dbpedia.org/resource/Charles_Herbert_Locke, http://dbpedia.org/resource/Claire_Carleton, http://dbpedia.org/resource/Gina_Kaus, http://dbpedia.org/resource/Rube_Brandow, http://dbpedia.org/resource/Matt_Hazeltine, http://dbpedia.org/resource/Walter_Kuhlman, http://dbpedia.org/resource/Joseph_Robbins_Kinney, http://dbpedia.org/resource/Joseph_N._Ermolieff, http://dbpedia.org/resource/Robert_Guenette, http://dbpedia.org/resource/Wilbert_Hiller, http://dbpedia.org/resource/William_W._Fraser, http://dbpedia.org/resource/Louisa_Benson_Craig, http://dbpedia.org/resource/Evelyn_Knight_(singer), http://dbpedia.org/resource/Marta_Golden, http://dbpedia.org/resource/Ike_Frankian, http://dbpedia.org/resource/Vincent_Tubbs, http://dbpedia.org/resource/Menachem_Zilberman, http://dbpedia.org/resource/Felix_Hathaway, http://dbpedia.org/resource/Benny_Goldberg, http://dbpedia.org/resource/Ronald_McKinnon_(economist), http://dbpedia.org/resource/Skinny_Dennis, http://dbpedia.org/resource/Gordon_V._Smith, http://dbpedia.org/resource/Josephine_Wayne, http://dbpedia.org/resource/James_Corrigan, http://dbpedia.org/resource/Philip_Klein, http://dbpedia.org/resource/Tom_Hernández, http://dbpedia.org/resource/Jimmie_Dolan, http://dbpedia.org/resource/Isabel_Branson_Cartwright, http://dbpedia.org/resource/Edward_R._Hanford, http://dbpedia.org/resource/Parke_Swartzel, http://dbpedia.org/resource/Divie_Bethune_McCartee, http://dbpedia.org/resource/Sibby_Nichols, http://dbpedia.org/resource/David_E._Sorensen, http://dbpedia.org/resource/George_Washington_Stephens,_Jr., http://dbpedia.org/resource/H._Dean_Brown, http://dbpedia.org/resource/Barney_Sarecky, http://dbpedia.org/resource/Denis_Kendall, http://dbpedia.org/resource/Hooky_Rothman, http://dbpedia.org/resource/Jack_Cheatham, http://dbpedia.org/resource/Murder_of_Marion_Parker, http://dbpedia.org/resource/De_Peyster_Brown, http://dbpedia.org/resource/Sacha_Grunpeter, http://dbpedia.org/resource/Rollin_S._Sturgeon, http://dbpedia.org/resource/Edith_Borella, http://dbpedia.org/resource/Frank_MacQuarrie, http://dbpedia.org/resource/Hayward_Mack, http://dbpedia.org/resource/Ogden_Crane, http://dbpedia.org/resource/Harry_Dunkinson, http://dbpedia.org/resource/William_Stack, http://dbpedia.org/resource/Barry_Conners, http://dbpedia.org/resource/John_War_Eagle, http://dbpedia.org/resource/Selma_Rubin, http://dbpedia.org/resource/Ivan_Bonar, http://dbpedia.org/resource/Lester_Vail, http://dbpedia.org/resource/William_Wagner_(actor), http://dbpedia.org/resource/Charles_Clague, http://dbpedia.org/resource/Paul_Borofsky, http://dbpedia.org/resource/John_Spacely, http://dbpedia.org/resource/Timothy_Lin, http://dbpedia.org/resource/August_Rinaldi, http://dbpedia.org/resource/George_Garner, http://dbpedia.org/resource/Tom_Ebersold, http://dbpedia.org/resource/Abbie_Newell, http://dbpedia.org/resource/Alva_Sibbetts, http://dbpedia.org/resource/Walter_Clarence_Thornton, http://dbpedia.org/resource/Alan_Watts, http://dbpedia.org/resource/Albert_Ghiorso, http://dbpedia.org/resource/Anton_Furst, http://dbpedia.org/resource/Armen_Alchian, http://dbpedia.org/resource/Barbara_Luddy, http://dbpedia.org/resource/Beatrice_Straight, http://dbpedia.org/resource/Bernard_Herrmann, http://dbpedia.org/resource/Bernie_Brillstein, http://dbpedia.org/resource/Bruce_Gary, http://dbpedia.org/resource/Cecil_B._DeMille, http://dbpedia.org/resource/Charles_Sumner_Tainter, http://dbpedia.org/resource/Christopher_Hewett, http://dbpedia.org/resource/Clark_Ashton_Smith, http://dbpedia.org/resource/Corinne_Calvet, http://dbpedia.org/resource/D._W._Griffith, http://dbpedia.org/resource/Dan_Walker_(politician), http://dbpedia.org/resource/Daniel_B._Cathcart, http://dbpedia.org/resource/Daniel_Petrie, http://dbpedia.org/resource/David_O._Selznick, http://dbpedia.org/resource/David_Opatoshu, http://dbpedia.org/resource/Donald_Wills_Douglas,_Sr., http://dbpedia.org/resource/Edwin_B._Willis, http://dbpedia.org/resource/Elizabeth_Montgomery, http://dbpedia.org/resource/Elizabeth_Taylor, http://dbpedia.org/resource/Elliott_Smith, http://dbpedia.org/resource/Ernst_G._Straus, http://dbpedia.org/resource/Eve_Arden, http://dbpedia.org/resource/Frank_Capra, http://dbpedia.org/resource/Frank_Martínez_(artist), http://dbpedia.org/resource/Fritz_Leiber, http://dbpedia.org/resource/Gene_Raymond, http://dbpedia.org/resource/George_Cukor, http://dbpedia.org/resource/George_Milo, http://dbpedia.org/resource/Gertrude_Baines, http://dbpedia.org/resource/Gia_Scala, http://dbpedia.org/resource/Gordon_Jump, http://dbpedia.org/resource/Gore_Vidal, http://dbpedia.org/resource/Gregory_Hines, http://dbpedia.org/resource/Groucho_Marx, http://dbpedia.org/resource/H._R._Haldeman, http://dbpedia.org/resource/Halston, http://dbpedia.org/resource/Henry_Fonda, http://dbpedia.org/resource/Hubert_Selby,_Jr., http://dbpedia.org/resource/Ida_Lupino, http://dbpedia.org/resource/Imogen_Cunningham, http://dbpedia.org/resource/Inger_Stevens, http://dbpedia.org/resource/Ira_S._Webb, http://dbpedia.org/resource/Jack_Kramer, http://dbpedia.org/resource/Jack_London, http://dbpedia.org/resource/James_Jamerson, http://dbpedia.org/resource/James_Komack, http://dbpedia.org/resource/Janet_Leigh, http://dbpedia.org/resource/Janis_Joplin, http://dbpedia.org/resource/Jason_Rhoades, http://dbpedia.org/resource/Jim_Davis_(actor), http://dbpedia.org/resource/Joan_Gerber, http://dbpedia.org/resource/Joan_Kroc, http://dbpedia.org/resource/Joe_Harnell, http://dbpedia.org/resource/Joel_McCrea, http://dbpedia.org/resource/John_Frankenheimer, http://dbpedia.org/resource/John_Phillips_(musician), http://dbpedia.org/resource/John_Williams_(actor), http://dbpedia.org/resource/Johnny_Green, http://dbpedia.org/resource/Jon_Dough, http://dbpedia.org/resource/Jonas_Salk, http://dbpedia.org/resource/Karen_Black, http://dbpedia.org/resource/Kenneth_Ross_MacKenzie, http://dbpedia.org/resource/Lawrence_Brown_(jazz_trombonist), http://dbpedia.org/resource/Leo_Penn, http://dbpedia.org/resource/Lita_Grey, http://dbpedia.org/resource/Lon_Chaney, http://dbpedia.org/resource/Margaret_Herrick, http://dbpedia.org/resource/Marlon_Brando, http://dbpedia.org/resource/Marvin_Gaye, http://dbpedia.org/resource/Max_Fleischer, http://dbpedia.org/resource/Melville_Shavelson, http://dbpedia.org/resource/Mike_Gray, http://dbpedia.org/resource/Mike_Levey, http://dbpedia.org/resource/Miklós_Rózsa, http://dbpedia.org/resource/Milton_Friedman, http://dbpedia.org/resource/Ormer_Locklear, http://dbpedia.org/resource/Orson_Welles, http://dbpedia.org/resource/Patrick_Swayze, http://dbpedia.org/resource/Phil_Abramson, http://dbpedia.org/resource/Poul_Anderson, http://dbpedia.org/resource/Ray_Bradbury, http://dbpedia.org/resource/Ray_Kroc, http://dbpedia.org/resource/Regis_Toomey, http://dbpedia.org/resource/Richard_Conte, http://dbpedia.org/resource/Rick_James, http://dbpedia.org/resource/Robert_Altman, http://dbpedia.org/resource/Robert_Bloch, http://dbpedia.org/resource/Robert_Priestley, http://dbpedia.org/resource/Rock_Hudson, http://dbpedia.org/resource/Rod_Taylor, http://dbpedia.org/resource/Roger_Voudouris, http://dbpedia.org/resource/Ronald_Reagan, http://dbpedia.org/resource/Sally_Ride, http://dbpedia.org/resource/Samuel_Edward_Konkin_III, http://dbpedia.org/resource/Samuel_Goldwyn, http://dbpedia.org/resource/Scott_Safran, http://dbpedia.org/resource/Sergei_Rachmaninoff, http://dbpedia.org/resource/Shari_Lewis, http://dbpedia.org/resource/Shig_Murao, http://dbpedia.org/resource/Shirley_Thomas_(USC_professor), http://dbpedia.org/resource/Sidney_Lumet, http://dbpedia.org/resource/Soledad_Mexia, http://dbpedia.org/resource/Stafford_Repp, http://dbpedia.org/resource/Steven_Parent, http://dbpedia.org/resource/Strother_Martin, http://dbpedia.org/resource/Suzanne_Pleshette, http://dbpedia.org/resource/Ted_Smith_(art_director), http://dbpedia.org/resource/Timothy_Leary, http://dbpedia.org/resource/Travis_Bean, http://dbpedia.org/resource/Wally_Hedrick, http://dbpedia.org/resource/Wes_Craven, http://dbpedia.org/resource/William_Alfred_Fowler, http://dbpedia.org/resource/William_Bendix, http://dbpedia.org/resource/William_Rosecrans, http://dbpedia.org/resource/Allen_Drury, http://dbpedia.org/resource/Anna_Elizabeth_Klumpke, http://dbpedia.org/resource/Beatriz_Michelena, http://dbpedia.org/resource/Bhante_Dharmawara, http://dbpedia.org/resource/Bill_Tilden, http://dbpedia.org/resource/Bronislava_Nijinska, http://dbpedia.org/resource/Cornel_Wilde, http://dbpedia.org/resource/Dan_Duryea, http://dbpedia.org/resource/Darren_McGavin, http://dbpedia.org/resource/David_Ackles, http://dbpedia.org/resource/Don_%22Sugarcane%22_Harris, http://dbpedia.org/resource/Doug_Fieger, http://dbpedia.org/resource/Doug_Henning, http://dbpedia.org/resource/Doug_McClure, http://dbpedia.org/resource/Dub_Taylor, http://dbpedia.org/resource/Durham_Stevens, http://dbpedia.org/resource/Forrest_J_Ackerman, http://dbpedia.org/resource/Forrest_Tucker, http://dbpedia.org/resource/Gale_Gordon, http://dbpedia.org/resource/Gerry_Day, http://dbpedia.org/resource/Haing_S._Ngor, http://dbpedia.org/resource/Harvey_Lembeck, http://dbpedia.org/resource/Henry_S._Reuss, http://dbpedia.org/resource/Herbert_Stothart, http://dbpedia.org/resource/Jim_Backus, http://dbpedia.org/resource/Jim_Harmon, http://dbpedia.org/resource/John_Lautner, http://dbpedia.org/resource/John_Vernon, http://dbpedia.org/resource/Kenne_Duncan, http://dbpedia.org/resource/Larry_Jennings, http://dbpedia.org/resource/Leo_T._McCarthy, http://dbpedia.org/resource/McLean_Stevenson, http://dbpedia.org/resource/Mike_Stepovich, http://dbpedia.org/resource/Nico_Minardos, http://dbpedia.org/resource/Nils_Olaf_Chrisander, http://dbpedia.org/resource/Oskar_Fischinger, http://dbpedia.org/resource/Paul_Hartman, http://dbpedia.org/resource/Peg_Entwistle, http://dbpedia.org/resource/Peggy_Gilbert, http://dbpedia.org/resource/Philip_Abbott, http://dbpedia.org/resource/Philip_Taylor_Kramer, http://dbpedia.org/resource/Raymond_St._Jacques, http://dbpedia.org/resource/Renato_Dulbecco, http://dbpedia.org/resource/Renée_Adorée, http://dbpedia.org/resource/Richard_Matheson, http://dbpedia.org/resource/Richard_Ramirez, http://dbpedia.org/resource/Robert_Kardashian, http://dbpedia.org/resource/Robert_Rockwell, http://dbpedia.org/resource/Ron_Ziegler, http://dbpedia.org/resource/Rudolph_Maté, http://dbpedia.org/resource/Stanley_Kramer, http://dbpedia.org/resource/Stewart_%22Dirk%22_Fischer, http://dbpedia.org/resource/Tim_Buckley, http://dbpedia.org/resource/Timothy_Carey, http://dbpedia.org/resource/Tyrone_Mitchell, http://dbpedia.org/resource/Vic_Morrow, http://dbpedia.org/resource/Walker_Edmiston, http://dbpedia.org/resource/Wayne_Allwine, http://dbpedia.org/resource/Weldon_Kees, http://dbpedia.org/resource/Whit_Bissell, http://dbpedia.org/resource/William_Desmond_Taylor, http://dbpedia.org/resource/Altovise_Davis, http://dbpedia.org/resource/Art_Babbitt, http://dbpedia.org/resource/Art_Gilmore, http://dbpedia.org/resource/Betty_Schade, http://dbpedia.org/resource/De'Angelo_Wilson, http://dbpedia.org/resource/Don_Grady, http://dbpedia.org/resource/Ed_Peck, http://dbpedia.org/resource/Grant_Withers, http://dbpedia.org/resource/Hank_Rieger, http://dbpedia.org/resource/Jim_Walsh_(basketball), http://dbpedia.org/resource/Merlin_Santana, http://dbpedia.org/resource/Michelle_Nicastro, http://dbpedia.org/resource/Paul_Brinegar, http://dbpedia.org/resource/Paul_László, http://dbpedia.org/resource/Richard_Haydn, http://dbpedia.org/resource/Rico_Alaniz, http://dbpedia.org/resource/Robert_Samuel_Maclay, http://dbpedia.org/resource/Robert_William_Wood, http://dbpedia.org/resource/Roland_Arnall, http://dbpedia.org/resource/Ted_Adams_(actor), http://dbpedia.org/resource/Tige_Andrews, http://dbpedia.org/resource/Virginia_Christine, http://dbpedia.org/resource/Virginia_Grey, http://dbpedia.org/resource/Warner_Anderson, http://dbpedia.org/resource/Warren_Stevens, http://dbpedia.org/resource/William_Beaudine, http://dbpedia.org/resource/A._S._Ruth, http://dbpedia.org/resource/Chuck_Ruff_(musician), http://dbpedia.org/resource/Curtis_Bernhardt, http://dbpedia.org/resource/Elmer_T._Cunningham, http://dbpedia.org/resource/Grant_Williams, http://dbpedia.org/resource/Jody_Reynolds, http://dbpedia.org/resource/John_Calley, http://dbpedia.org/resource/Mike_Ragan, http://dbpedia.org/resource/Paul_Gayten, http://dbpedia.org/resource/Rhea_Mitchell, http://dbpedia.org/resource/Robert_Bray, http://dbpedia.org/resource/Robert_V._Barron, http://dbpedia.org/resource/William_Trautmann, http://dbpedia.org/resource/Addison_Richards, http://dbpedia.org/resource/Adelle_Davis, http://dbpedia.org/resource/Al_Norgard, http://dbpedia.org/resource/Aleta_Freel, http://dbpedia.org/resource/Allan_V._Cox, http://dbpedia.org/resource/Almira_Sessions, http://dbpedia.org/resource/Arthur_L._White, http://dbpedia.org/resource/Berry_L._Cannon, http://dbpedia.org/resource/Betty_Bronson, http://dbpedia.org/resource/Betty_Compson, http://dbpedia.org/resource/Bob_Steele_(actor), http://dbpedia.org/resource/Brandon_Hurst, http://dbpedia.org/resource/Brown_Holmes, http://dbpedia.org/resource/Calvin_Boze, http://dbpedia.org/resource/Cecilia_Parker, http://dbpedia.org/resource/Charles_Butterworth_(actor), http://dbpedia.org/resource/Chubby_Johnson, http://dbpedia.org/resource/Chuck_Higgins, http://dbpedia.org/resource/Clara_Williams, http://dbpedia.org/resource/Conrad_Hilton,_Jr., http://dbpedia.org/resource/Corbin_Harney, http://dbpedia.org/resource/Craig_Reynolds_(actor), http://dbpedia.org/resource/Damian_O'Flynn, http://dbpedia.org/resource/David_Gorcey, http://dbpedia.org/resource/Donald_Meek, http://dbpedia.org/resource/Dorothy_Harrell, http://dbpedia.org/resource/Douglass_Dumbrille, http://dbpedia.org/resource/Ed_Wolff_(actor), http://dbpedia.org/resource/Eddie_Polo, http://dbpedia.org/resource/Elsie_Allen, http://dbpedia.org/resource/Emile_Ardolino, http://dbpedia.org/resource/Florence_Owens_Thompson, http://dbpedia.org/resource/Frank_Parker, http://dbpedia.org/resource/George_Cisar_(actor), http://dbpedia.org/resource/George_F._Houston, http://dbpedia.org/resource/Gerson_Goldhaber, http://dbpedia.org/resource/Gordon_Bau, http://dbpedia.org/resource/Hanns_Scharff, http://dbpedia.org/resource/Harold_Mirisch, http://dbpedia.org/resource/Harry_Davenport_(actor), http://dbpedia.org/resource/Helen_Ware, http://dbpedia.org/resource/Herb_Edelman, http://dbpedia.org/resource/Hope_Summers, http://dbpedia.org/resource/Howard_Pease, http://dbpedia.org/resource/Irene_Tedrow, http://dbpedia.org/resource/Jack_Lambert_(actor), http://dbpedia.org/resource/James_C._Morton, http://dbpedia.org/resource/Jan_Savitt, http://dbpedia.org/resource/Janet_Margolin, http://dbpedia.org/resource/Jerome_Cowan, http://dbpedia.org/resource/Jewett_W._Adams, http://dbpedia.org/resource/John_Howard_(American_actor), http://dbpedia.org/resource/John_Ince_(actor), http://dbpedia.org/resource/John_S._Robertson, http://dbpedia.org/resource/Joseph_LeConte, http://dbpedia.org/resource/Lane_Chandler, http://dbpedia.org/resource/Leo_Kennedy, http://dbpedia.org/resource/Leon_Shamroy, http://dbpedia.org/resource/Leonard_Firestone, http://dbpedia.org/resource/Leonard_Penn, http://dbpedia.org/resource/Lloyd_Gough, http://dbpedia.org/resource/Martha_Wentworth, http://dbpedia.org/resource/Mary_Nolan, http://dbpedia.org/resource/Michael_Blankfort, http://dbpedia.org/resource/Mildred_Davis, http://dbpedia.org/resource/Ned_Glass, http://dbpedia.org/resource/Newton_E._Mason, http://dbpedia.org/resource/Nicolay_Paskevich, http://dbpedia.org/resource/Nigel_De_Brulier, http://dbpedia.org/resource/Norton_Simon, http://dbpedia.org/resource/Olive_Carey, http://dbpedia.org/resource/Otto_Kruger, http://dbpedia.org/resource/Paul_Richards_(actor), http://dbpedia.org/resource/Pee_Wee_Crayton, http://dbpedia.org/resource/Philippe_De_Lacy, http://dbpedia.org/resource/Ralph_Randles_Stewart, http://dbpedia.org/resource/Ray_Danton, http://dbpedia.org/resource/Reed_Hadley, http://dbpedia.org/resource/Rex_Lease, http://dbpedia.org/resource/Rhys_Williams_(Welsh-American_actor), http://dbpedia.org/resource/Rick_Vallin, http://dbpedia.org/resource/Robert_Cornthwaite_(actor), http://dbpedia.org/resource/Robert_Edeson, http://dbpedia.org/resource/Robert_Karnes, http://dbpedia.org/resource/Robert_Warwick, http://dbpedia.org/resource/Roy_Barcroft, http://dbpedia.org/resource/Roy_Milton, http://dbpedia.org/resource/Roy_Raymond_(businessman), http://dbpedia.org/resource/Rusty_Wescoatt, http://dbpedia.org/resource/Shelton_Brooks, http://dbpedia.org/resource/Sir_Lancelot_(singer), http://dbpedia.org/resource/Walter_R._Nickel, http://dbpedia.org/resource/Walter_Strenge, http://dbpedia.org/resource/William_Eythe, http://dbpedia.org/resource/William_French_Smith, http://dbpedia.org/resource/Willis_Bouchey, http://dbpedia.org/resource/Wonderful_Smith, http://dbpedia.org/resource/Aloysius_Masnata, http://dbpedia.org/resource/Amasa_Cobb, http://dbpedia.org/resource/Bob_Thomas_(reporter), http://dbpedia.org/resource/Charles_Emmett_Mack, http://dbpedia.org/resource/Dorothy_Kamenshek, http://dbpedia.org/resource/Edward_Sedgwick, http://dbpedia.org/resource/Ephraim_Morse, http://dbpedia.org/resource/Estelle_Taylor, http://dbpedia.org/resource/Harry_Kellar, http://dbpedia.org/resource/Herbert_Standing, http://dbpedia.org/resource/Jean_Carson, http://dbpedia.org/resource/Jean_Havez, http://dbpedia.org/resource/Jetta_Goudal, http://dbpedia.org/resource/Joanna_Moore, http://dbpedia.org/resource/Karl_Dane, http://dbpedia.org/resource/Lionel_Belmore, http://dbpedia.org/resource/Lou_Tellegen, http://dbpedia.org/resource/Mabel_Paige, http://dbpedia.org/resource/Reginald_Barker, http://dbpedia.org/resource/Richard_%22Skeets%22_Gallagher, http://dbpedia.org/resource/Sandy_Kenyon, http://dbpedia.org/resource/Sydney_Ayres, http://dbpedia.org/resource/Thomas_Whaley, http://dbpedia.org/resource/Walter_Hiers, http://dbpedia.org/resource/William_A._Glassford, http://dbpedia.org/resource/William_A._Richardson, http://dbpedia.org/resource/Al_Pearce, http://dbpedia.org/resource/Alfred_J._Eggers, http://dbpedia.org/resource/Charles_C._Wilson_(actor), http://dbpedia.org/resource/George_Ullman, http://dbpedia.org/resource/Jack_Dragna, http://dbpedia.org/resource/Lincoln_Kilpatrick, http://dbpedia.org/resource/Olaf_Pooley, http://dbpedia.org/resource/Samuel_Bronston, http://dbpedia.org/resource/Fathia_Ghali, http://dbpedia.org/resource/Frank_Waldman, http://dbpedia.org/resource/George_G._Anderson, http://dbpedia.org/resource/Gerry_Johnson, http://dbpedia.org/resource/Ian_Fraser_(composer), http://dbpedia.org/resource/Jack_Elliott_(composer), http://dbpedia.org/resource/Nazli_Sabri, http://dbpedia.org/resource/René_Hall, http://dbpedia.org/resource/Roderick_Cook, http://dbpedia.org/resource/Sherburne_F._Cook, http://dbpedia.org/resource/Aut_Erickson, http://dbpedia.org/resource/Donald_H._Clausen, http://dbpedia.org/resource/Glenn_M._Anderson, http://dbpedia.org/resource/Jock_Gaynor, http://dbpedia.org/resource/Ron_Gans, http://dbpedia.org/resource/Samuel_J._Briskin, http://dbpedia.org/resource/Al_Bernardin, http://dbpedia.org/resource/Paul_Carr_(actor), http://dbpedia.org/resource/Winston_and_Weston_Doty, http://dbpedia.org/resource/Alan_Bible, http://dbpedia.org/resource/Alfred_Ellet_Hitchner, http://dbpedia.org/resource/Carol_Thurston, http://dbpedia.org/resource/Charles_Henry_King, http://dbpedia.org/resource/Edwin_Corle, http://dbpedia.org/resource/George_Ainslie_(delegate), http://dbpedia.org/resource/Robert_Simpson_(athlete), http://dbpedia.org/resource/Tasker_Oddie, http://dbpedia.org/resource/William_Hansen_(actor), http://dbpedia.org/resource/William_Hauber, http://dbpedia.org/resource/William_R._Lyman, http://dbpedia.org/resource/Frederick_Bee, http://dbpedia.org/resource/Gregory_Walcott, http://dbpedia.org/resource/John_Roche_(actor), http://dbpedia.org/resource/Leslie_E._Gehres, http://dbpedia.org/resource/Gil_Stratton, http://dbpedia.org/resource/Jack_Pleis, http://dbpedia.org/resource/Richard_Rust, http://dbpedia.org/resource/Albert_Popwell, http://dbpedia.org/resource/Andy_Iona, http://dbpedia.org/resource/Bob_Denton, http://dbpedia.org/resource/Bruce_Lester, http://dbpedia.org/resource/Buddy_Feyne, http://dbpedia.org/resource/Charitie_Lees_Smith, http://dbpedia.org/resource/Gertrude_Niesen, http://dbpedia.org/resource/Jeanne_Bates, http://dbpedia.org/resource/John_Garrick, http://dbpedia.org/resource/Ken_Steadman, http://dbpedia.org/resource/Mariska_Aldrich, http://dbpedia.org/resource/Michael_Asher_(artist), http://dbpedia.org/resource/Michael_Stroka, http://dbpedia.org/resource/Montrose_Hagins, http://dbpedia.org/resource/Sammy_Lerner, http://dbpedia.org/resource/Tom_Clay, http://dbpedia.org/resource/Tony_Duquette, http://dbpedia.org/resource/Bill_Anderson_(producer), http://dbpedia.org/resource/Chief_Buffalo_Child_Long_Lance, http://dbpedia.org/resource/Edward_L._Baker,_Jr., http://dbpedia.org/resource/Herschel_Rosenthal, http://dbpedia.org/resource/John_Bloom_(actor), http://dbpedia.org/resource/John_J._Lloyd, http://dbpedia.org/resource/John_Nicolella, http://dbpedia.org/resource/Madlyn_Rhue, http://dbpedia.org/resource/Micole_Mercurio, http://dbpedia.org/resource/Adrienne_D'Ambricourt, http://dbpedia.org/resource/Albert_W._Hawkes, http://dbpedia.org/resource/Alex_Lovy, http://dbpedia.org/resource/Alice_Pike_Barney, http://dbpedia.org/resource/Andrew_Arbuckle_(actor), http://dbpedia.org/resource/Bull_Montana, http://dbpedia.org/resource/Carl_Greenberg, http://dbpedia.org/resource/Charles_Eyton, http://dbpedia.org/resource/Cornelius_Cole, http://dbpedia.org/resource/Danny_Dark, http://dbpedia.org/resource/David_Worth_Clark, http://dbpedia.org/resource/Dehl_Berti, http://dbpedia.org/resource/Don_Oreck, http://dbpedia.org/resource/Doris_May, http://dbpedia.org/resource/Dorothea_Holt_Redmond, http://dbpedia.org/resource/Emory_Johnson, http://dbpedia.org/resource/Estelle_Etterre, http://dbpedia.org/resource/Ethel_Shannon, http://dbpedia.org/resource/Etienne_Girardot, http://dbpedia.org/resource/Eugene_Casserly, http://dbpedia.org/resource/Frank_Lackteen, http://dbpedia.org/resource/Frank_Sullivan_(film_editor), http://dbpedia.org/resource/Fred_Abbott, http://dbpedia.org/resource/Gypsy_Abbott, http://dbpedia.org/resource/Henry_Cronjager, http://dbpedia.org/resource/Henry_Cuesta, http://dbpedia.org/resource/Howard_Smit, http://dbpedia.org/resource/Hugh_Reilly, http://dbpedia.org/resource/Ivor_Francis, http://dbpedia.org/resource/James_Helmick_Beatty, http://dbpedia.org/resource/James_Luisi, http://dbpedia.org/resource/Jay_Belasco, http://dbpedia.org/resource/Johnny_Carpenter, http://dbpedia.org/resource/Joseph_De_Stefani, http://dbpedia.org/resource/Kenneth_Hahn, http://dbpedia.org/resource/Larry_McCormick_(TV), http://dbpedia.org/resource/Lillian_Kemble-Cooper, http://dbpedia.org/resource/Louise_Emmons, http://dbpedia.org/resource/Margarita_Sierra, http://dbpedia.org/resource/Mark_Landon, http://dbpedia.org/resource/Maurice_Cass, http://dbpedia.org/resource/Michael_Blodgett, http://dbpedia.org/resource/Michael_Garrison_(producer), http://dbpedia.org/resource/Miriam_Snitzer, http://dbpedia.org/resource/Mozelle_Britton, http://dbpedia.org/resource/Nat_Carr, http://dbpedia.org/resource/Nell_Franzen, http://dbpedia.org/resource/Nick_Licata_(mobster), http://dbpedia.org/resource/Otto_Fries, http://dbpedia.org/resource/Pat_Hogan, http://dbpedia.org/resource/Renie_Riano, http://dbpedia.org/resource/Roger_Gimbel, http://dbpedia.org/resource/Rose_A._George, http://dbpedia.org/resource/Steve_James_(actor), http://dbpedia.org/resource/Tara_Correa-McMullen, http://dbpedia.org/resource/Wanda_McKay, http://dbpedia.org/resource/Carl_Stockdale, http://dbpedia.org/resource/Dick_Hammer, http://dbpedia.org/resource/Jorge_de_Sena, http://dbpedia.org/resource/Karl_Brown_(cinematographer), http://dbpedia.org/resource/Paul_Napier, http://dbpedia.org/resource/William_Attaway, http://dbpedia.org/resource/Alphonzo_Bell, http://dbpedia.org/resource/Fernando_Alegría, http://dbpedia.org/resource/Dorothy_Van, http://dbpedia.org/resource/Stephen_Wayne_Anderson, http://dbpedia.org/resource/Emile_Norman, http://dbpedia.org/resource/Ethan_Crosby, http://dbpedia.org/resource/William_Keith_(artist), http://dbpedia.org/resource/Bob_Gordon_(saxophonist), http://dbpedia.org/resource/Byron_Randall, http://dbpedia.org/resource/Jack_Molinas, http://dbpedia.org/resource/John_Entenza, http://dbpedia.org/resource/Rod_O'Connor_(announcer), http://dbpedia.org/resource/Eilley_Bowers, http://dbpedia.org/resource/Mike_Gold, http://dbpedia.org/resource/Catherine_Murphy_Urner, http://dbpedia.org/resource/Edgar_Gott, http://dbpedia.org/resource/George_Schneiderman, http://dbpedia.org/resource/Herbie_Steward, http://dbpedia.org/resource/Radie_Britain, http://dbpedia.org/resource/Rosalina_Abejo, http://dbpedia.org/resource/Dean_Elliott, http://dbpedia.org/resource/Arthur_Lawrence_Alarcon, http://dbpedia.org/resource/Brian_Norton, http://dbpedia.org/resource/H._W._Harkness, http://dbpedia.org/resource/Arthur_Gould-Porter, http://dbpedia.org/resource/Grigori_Mints, http://dbpedia.org/resource/Marguerite_P._Justice, http://dbpedia.org/resource/Prudent_Beaudry, http://dbpedia.org/resource/Coles_Trapnell, http://dbpedia.org/resource/John_Cossette, http://dbpedia.org/resource/Lucius_E._Pinkham, http://dbpedia.org/resource/Charles_Craft, http://dbpedia.org/resource/David_Charles_Collier, http://dbpedia.org/resource/Morton_Everel_Post, http://dbpedia.org/resource/Viktor_Zhivov, http://dbpedia.org/resource/Earl_Zindars, http://dbpedia.org/resource/Frank_Shuter, http://dbpedia.org/resource/Henry_Liu, http://dbpedia.org/resource/Richard_Russell_(Dow_Theory), http://dbpedia.org/resource/Franklin_Pierce_Burnham, http://dbpedia.org/resource/Robert_P._Sharp, http://dbpedia.org/resource/Helen_Freeman_Corle, http://dbpedia.org/resource/Tody_Smith, http://dbpedia.org/resource/Hazen_Aldrich, http://dbpedia.org/resource/Beniamino_Bufano, http://dbpedia.org/resource/Ned_Miller_(composer), http://dbpedia.org/resource/John_E._Jacobs, http://dbpedia.org/resource/Paul_Harris_(actor), http://dbpedia.org/resource/Gary_Jasgur, http://dbpedia.org/resource/Janet_Burston, http://dbpedia.org/resource/Margaret_Mann, http://dbpedia.org/resource/Fred_Fleck, http://dbpedia.org/resource/Hans_Joby, http://dbpedia.org/resource/Lorimer_Johnston]... [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 512, P: 0.524, R: 0.576, macro F1: 0.531, macro F1*: 0.549 -[INFO] Current time: 13586 ms, average: 2741.38 ms/ques, entity detection: 425.98 ms/ques (linking tool: 334.00 ms/ques), relation detection: 895.81 ms/ques (similarity: 705.47 ms/ques), EDG generation: 51.44 ms/ques +[INFO] Cumulative metrics, sample: 512, P: 0.524, R: 0.576, macro F1: 0.549 +[INFO] Current question consumed time: 12473 ms, average: 2512.04 ms/ques, entity detection: 385.53 ms/ques (linking tool: 304.63 ms/ques), relation detection: 838.25 ms/ques (similarity: 692.09 ms/ques), EDG generation: 51.40 ms/ques [INFO] Entity Linking P: 0.75, R: 0.705078125, micro F1: 0.7194010416666672, macro F1: 0.7268456375838926 [INFO] Relation Linking P: 0.6057942708333333, R: 0.5556640625, micro F1: 0.5694010416666665, macro F1: 0.5796473208029709 [INFO] Type Linking P: 0.53515625, R: 0.53515625, micro F1: 0.53515625, macro F1: 0.53515625 @@ -7180,8 +7180,8 @@ Question 512: Which technological products were manufactured by Foxconn? [INFO] Golden Answer: [http://dbpedia.org/resource/Amazon_Kindle, http://dbpedia.org/resource/IPad, http://dbpedia.org/resource/IPad_(1st_generation), http://dbpedia.org/resource/IPad_Air, http://dbpedia.org/resource/IPad_Air_2, http://dbpedia.org/resource/IPad_Mini, http://dbpedia.org/resource/IPad_Mini_3, http://dbpedia.org/resource/IPhone, http://dbpedia.org/resource/IPod_Nano, http://dbpedia.org/resource/IPod_Touch, http://dbpedia.org/resource/IPod_Touch_(6th_generation), http://dbpedia.org/resource/Nintendo_64, http://dbpedia.org/resource/Nintendo_DS, http://dbpedia.org/resource/PlayStation_3, http://dbpedia.org/resource/Wii, http://dbpedia.org/resource/Xbox_360, http://dbpedia.org/resource/PlayStation_4, http://dbpedia.org/resource/Wii_U, http://dbpedia.org/resource/Xbox_One, http://dbpedia.org/resource/Nintendo_3DS, http://dbpedia.org/resource/Nokia_N1, http://dbpedia.org/resource/GameCube, http://dbpedia.org/resource/New_Nintendo_3DS, http://dbpedia.org/resource/Google_Glass, http://dbpedia.org/resource/IPad_Mini_2, http://dbpedia.org/resource/IPad_Mini_4, http://dbpedia.org/resource/IPad_Pro, http://dbpedia.org/resource/IPad_(4th_generation), http://dbpedia.org/resource/Barnes_&_Noble_Nook_1st_Edition, http://dbpedia.org/resource/IPad_(3rd_generation), http://dbpedia.org/resource/IPad_Mini_(1st_generation), http://dbpedia.org/resource/IPad_(2nd_generation), http://dbpedia.org/resource/Nook_Simple_Touch, http://dbpedia.org/resource/Sony_Tablet_S, http://dbpedia.org/resource/Xiaomi_Mi_Band] [INFO] Predicted Answer: [http://dbpedia.org/resource/Amazon_Kindle, http://dbpedia.org/resource/IPad, http://dbpedia.org/resource/IPad_(1st_generation), http://dbpedia.org/resource/IPad_Air, http://dbpedia.org/resource/IPad_Air_2, http://dbpedia.org/resource/IPad_Mini, http://dbpedia.org/resource/IPad_Mini_3, http://dbpedia.org/resource/IPhone, http://dbpedia.org/resource/IPhone_3G, http://dbpedia.org/resource/IPhone_3GS, http://dbpedia.org/resource/IPhone_4, http://dbpedia.org/resource/IPhone_4S, http://dbpedia.org/resource/IPhone_5, http://dbpedia.org/resource/IPhone_5C, http://dbpedia.org/resource/IPhone_5S, http://dbpedia.org/resource/IPhone_6, http://dbpedia.org/resource/IPod_Nano, http://dbpedia.org/resource/IPod_Touch, http://dbpedia.org/resource/IPod_Touch_(6th_generation), http://dbpedia.org/resource/Nintendo_64, http://dbpedia.org/resource/Nintendo_DS, http://dbpedia.org/resource/PlayStation_3, http://dbpedia.org/resource/Wii, http://dbpedia.org/resource/Xbox_360, http://dbpedia.org/resource/PlayStation_4, http://dbpedia.org/resource/Wii_U, http://dbpedia.org/resource/Xbox_One, http://dbpedia.org/resource/IPhone_(1st_generation), http://dbpedia.org/resource/Nintendo_3DS, http://dbpedia.org/resource/Nokia_N1, http://dbpedia.org/resource/GameCube, http://dbpedia.org/resource/New_Nintendo_3DS, http://dbpedia.org/resource/Google_Glass, http://dbpedia.org/resource/IPad_Mini_2, http://dbpedia.org/resource/IPad_Mini_4, http://dbpedia.org/resource/IPad_Pro, http://dbpedia.org/resource/Fire_Phone, http://dbpedia.org/resource/IPad_(4th_generation), http://dbpedia.org/resource/Barnes_&_Noble_Nook_1st_Edition, http://dbpedia.org/resource/Vibo_A688, http://dbpedia.org/resource/IPad_(3rd_generation), http://dbpedia.org/resource/IPad_Mini_(1st_generation), http://dbpedia.org/resource/IPad_(2nd_generation), http://dbpedia.org/resource/Nook_Simple_Touch, http://dbpedia.org/resource/Sony_Tablet_S, http://dbpedia.org/resource/Xiaomi_Mi_Band, http://dbpedia.org/resource/Apple_Wireless_Keyboard, http://dbpedia.org/resource/PocketBook_International] [INFO] P: 0.7291666666666666, R: 1.0, F: 0.8433734939759037 -[INFO] Cumulative metrics, sample: 513, P: 0.525, R: 0.577, macro F1: 0.532, macro F1*: 0.550 -[INFO] Current time: 785 ms, average: 2737.57 ms/ques, entity detection: 425.30 ms/ques (linking tool: 333.46 ms/ques), relation detection: 894.69 ms/ques (similarity: 704.75 ms/ques), EDG generation: 51.36 ms/ques +[INFO] Cumulative metrics, sample: 513, P: 0.525, R: 0.577, macro F1: 0.550 +[INFO] Current question consumed time: 678 ms, average: 2508.46 ms/ques, entity detection: 384.89 ms/ques (linking tool: 304.12 ms/ques), relation detection: 837.17 ms/ques (similarity: 691.41 ms/ques), EDG generation: 51.32 ms/ques [INFO] Entity Linking P: 0.7504873294346979, R: 0.7056530214424951, micro F1: 0.7199480181936327, macro F1: 0.7273799551683015 [INFO] Relation Linking P: 0.606562703053931, R: 0.5565302144249513, micro F1: 0.5702404158544507, macro F1: 0.5804703409672535 [INFO] Type Linking P: 0.5341130604288499, R: 0.5341130604288499, micro F1: 0.5341130604288499, macro F1: 0.5341130604288499 @@ -7194,8 +7194,8 @@ Question 513: Is George Lucas the cinematographer of 6-18-67? [INFO] Golden Answer: [true] [INFO] Predicted Answer: [true] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 514, P: 0.526, R: 0.578, macro F1: 0.533, macro F1*: 0.551 -[INFO] Current time: 979 ms, average: 2734.15 ms/ques, entity detection: 425.20 ms/ques (linking tool: 333.24 ms/ques), relation detection: 893.09 ms/ques (similarity: 703.47 ms/ques), EDG generation: 51.28 ms/ques +[INFO] Cumulative metrics, sample: 514, P: 0.526, R: 0.578, macro F1: 0.551 +[INFO] Current question consumed time: 777 ms, average: 2505.09 ms/ques, entity detection: 384.71 ms/ques (linking tool: 303.88 ms/ques), relation detection: 835.67 ms/ques (similarity: 690.15 ms/ques), EDG generation: 51.24 ms/ques [INFO] Entity Linking P: 0.7509727626459144, R: 0.7062256809338522, micro F1: 0.7204928664072638, macro F1: 0.7279121838196713 [INFO] Relation Linking P: 0.6073281452658884, R: 0.5573929961089494, micro F1: 0.5710765239948118, macro F1: 0.5812901345835536 [INFO] Type Linking P: 0.5350194552529183, R: 0.5350194552529183, micro F1: 0.5350194552529183, macro F1: 0.5350194552529183 @@ -7208,8 +7208,8 @@ Question 514: Whose predecessor had Jenny Macklin as a deputy ? [INFO] Golden Answer: [http://dbpedia.org/resource/Anthony_Albanese, http://dbpedia.org/resource/Chris_Evans_(Australian_politician), http://dbpedia.org/resource/Clare_O'Neil, http://dbpedia.org/resource/Bob_McMullan, http://dbpedia.org/resource/Mark_Latham, http://dbpedia.org/resource/Peter_Garrett, http://dbpedia.org/resource/Tanya_Plibersek, http://dbpedia.org/resource/Tony_Burke, http://dbpedia.org/resource/Jenny_Macklin, http://dbpedia.org/resource/Stephen_Smith_(Australian_politician), http://dbpedia.org/resource/Amanda_Vanstone, http://dbpedia.org/resource/Robert_Ray_(Australian_politician), http://dbpedia.org/resource/Joe_Hockey, http://dbpedia.org/resource/Simon_Crean, http://dbpedia.org/resource/Gareth_Evans_(politician), http://dbpedia.org/resource/Kevin_Rudd, http://dbpedia.org/resource/Peter_Reith, http://dbpedia.org/resource/Don_Randall, http://dbpedia.org/resource/Graham_Richardson, http://dbpedia.org/resource/Tim_Fischer, http://dbpedia.org/resource/Gary_Gray_(politician), http://dbpedia.org/resource/Chris_Hayes_(politician), http://dbpedia.org/resource/Julia_Gillard, http://dbpedia.org/resource/Kim_Beazley] [INFO] Predicted Answer: [http://dbpedia.org/resource/Simon_Crean, http://dbpedia.org/resource/Kim_Beazley, http://dbpedia.org/resource/Mark_Latham] [INFO] P: 1.0, R: 0.125, F: 0.2222222222222222 -[INFO] Cumulative metrics, sample: 515, P: 0.526, R: 0.577, macro F1: 0.532, macro F1*: 0.551 -[INFO] Current time: 850 ms, average: 2730.49 ms/ques, entity detection: 424.73 ms/ques (linking tool: 332.78 ms/ques), relation detection: 891.86 ms/ques (similarity: 702.42 ms/ques), EDG generation: 51.21 ms/ques +[INFO] Cumulative metrics, sample: 515, P: 0.526, R: 0.577, macro F1: 0.551 +[INFO] Current question consumed time: 735 ms, average: 2501.65 ms/ques, entity detection: 384.27 ms/ques (linking tool: 303.46 ms/ques), relation detection: 834.52 ms/ques (similarity: 689.12 ms/ques), EDG generation: 51.17 ms/ques [INFO] Entity Linking P: 0.7514563106796116, R: 0.7067961165048544, micro F1: 0.7210355987055022, macro F1: 0.728442335785296 [INFO] Relation Linking P: 0.6080906148867313, R: 0.5572815533980583, micro F1: 0.57126213592233, macro F1: 0.5815784719994176 [INFO] Type Linking P: 0.5359223300970873, R: 0.5359223300970873, micro F1: 0.5359223300970873, macro F1: 0.5359223300970873 @@ -7222,8 +7222,8 @@ Question 515: How many cars are similar to the ones assembled in the Jefferson N [INFO] Golden Answer: [29] [INFO] Predicted Answer: [33] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 516, P: 0.525, R: 0.576, macro F1: 0.531, macro F1*: 0.550 -[INFO] Current time: 1640 ms, average: 2728.37 ms/ques, entity detection: 424.26 ms/ques (linking tool: 332.33 ms/ques), relation detection: 891.04 ms/ques (similarity: 701.92 ms/ques), EDG generation: 51.30 ms/ques +[INFO] Cumulative metrics, sample: 516, P: 0.525, R: 0.576, macro F1: 0.550 +[INFO] Current question consumed time: 1455 ms, average: 2499.63 ms/ques, entity detection: 383.82 ms/ques (linking tool: 303.06 ms/ques), relation detection: 833.76 ms/ques (similarity: 688.64 ms/ques), EDG generation: 51.26 ms/ques [INFO] Entity Linking P: 0.751937984496124, R: 0.7073643410852714, micro F1: 0.7215762273901813, macro F1: 0.7289704232166941 [INFO] Relation Linking P: 0.6088501291989663, R: 0.5581395348837209, micro F1: 0.5720930232558138, macro F1: 0.5823930380602347 [INFO] Type Linking P: 0.5368217054263565, R: 0.5368217054263565, micro F1: 0.5368217054263565, macro F1: 0.5368217054263565 @@ -7236,8 +7236,8 @@ Question 516: Count the number of region of AN/FSQ-7 Combat Direction Central? [INFO] Golden Answer: [19] [INFO] Predicted Answer: [19] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 517, P: 0.526, R: 0.577, macro F1: 0.532, macro F1*: 0.550 -[INFO] Current time: 730 ms, average: 2724.51 ms/ques, entity detection: 423.67 ms/ques (linking tool: 331.86 ms/ques), relation detection: 889.65 ms/ques (similarity: 700.78 ms/ques), EDG generation: 51.25 ms/ques +[INFO] Cumulative metrics, sample: 517, P: 0.526, R: 0.577, macro F1: 0.550 +[INFO] Current question consumed time: 627 ms, average: 2496.00 ms/ques, entity detection: 383.28 ms/ques (linking tool: 302.62 ms/ques), relation detection: 832.45 ms/ques (similarity: 687.52 ms/ques), EDG generation: 51.21 ms/ques [INFO] Entity Linking P: 0.7504835589941973, R: 0.7059961315280464, micro F1: 0.7201805286911676, macro F1: 0.7275604223980929 [INFO] Relation Linking P: 0.6096067053513862, R: 0.558027079303675, micro F1: 0.5722759509993551, macro F1: 0.5826776405098009 [INFO] Type Linking P: 0.5377176015473888, R: 0.5377176015473888, micro F1: 0.5377176015473888, macro F1: 0.5377176015473888 @@ -7250,8 +7250,8 @@ Question 517: What are the common houses of US Congress and the Kansas Legislatu [INFO] Golden Answer: [http://dbpedia.org/resource/United_States_Senate] [INFO] Predicted Answer: [http://dbpedia.org/resource/United_States_Senate, http://dbpedia.org/resource/Kansas_House_of_Representatives, http://dbpedia.org/resource/Kansas_Senate, House@en, Senate@en] [INFO] P: 0.2, R: 1.0, F: 0.33333333333333337 -[INFO] Cumulative metrics, sample: 518, P: 0.526, R: 0.578, macro F1: 0.532, macro F1*: 0.550 -[INFO] Current time: 5051 ms, average: 2729.00 ms/ques, entity detection: 424.36 ms/ques (linking tool: 332.22 ms/ques), relation detection: 892.13 ms/ques (similarity: 701.74 ms/ques), EDG generation: 51.25 ms/ques +[INFO] Cumulative metrics, sample: 518, P: 0.526, R: 0.578, macro F1: 0.550 +[INFO] Current question consumed time: 4229 ms, average: 2499.35 ms/ques, entity detection: 383.72 ms/ques (linking tool: 302.84 ms/ques), relation detection: 834.59 ms/ques (similarity: 688.50 ms/ques), EDG generation: 51.20 ms/ques [INFO] Entity Linking P: 0.750965250965251, R: 0.7055984555984556, micro F1: 0.7200772200772206, macro F1: 0.7275753458656042 [INFO] Relation Linking P: 0.6103603603603602, R: 0.5579150579150579, micro F1: 0.5724581724581724, macro F1: 0.5829605424758274 [INFO] Type Linking P: 0.5386100386100386, R: 0.5386100386100386, micro F1: 0.5386100386100386, macro F1: 0.5386100386100386 @@ -7264,8 +7264,8 @@ Question 518: Was John Muir the first one to ascend one of the peaks in the cath [INFO] Golden Answer: [true] [INFO] Predicted Answer: [true] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 519, P: 0.527, R: 0.579, macro F1: 0.533, macro F1*: 0.551 -[INFO] Current time: 982 ms, average: 2725.63 ms/ques, entity detection: 423.55 ms/ques (linking tool: 331.58 ms/ques), relation detection: 890.41 ms/ques (similarity: 700.43 ms/ques), EDG generation: 51.39 ms/ques +[INFO] Cumulative metrics, sample: 519, P: 0.527, R: 0.579, macro F1: 0.551 +[INFO] Current question consumed time: 860 ms, average: 2496.19 ms/ques, entity detection: 382.98 ms/ques (linking tool: 302.25 ms/ques), relation detection: 832.98 ms/ques (similarity: 687.21 ms/ques), EDG generation: 51.38 ms/ques [INFO] Entity Linking P: 0.7514450867052023, R: 0.7061657032755299, micro F1: 0.7206165703275536, macro F1: 0.7281021130930777 [INFO] Relation Linking P: 0.611111111111111, R: 0.558766859344894, micro F1: 0.5732819524727039, macro F1: 0.5837679568121511 [INFO] Type Linking P: 0.5394990366088632, R: 0.5394990366088632, micro F1: 0.5394990366088632, macro F1: 0.5394990366088632 @@ -7278,8 +7278,8 @@ Question 519: From how many different places have people gone to the Harward Med [INFO] Golden Answer: [16] [INFO] Predicted Answer: [1] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 520, P: 0.526, R: 0.577, macro F1: 0.532, macro F1*: 0.550 -[INFO] Current time: 1259 ms, average: 2722.81 ms/ques, entity detection: 423.10 ms/ques (linking tool: 331.17 ms/ques), relation detection: 889.28 ms/ques (similarity: 699.66 ms/ques), EDG generation: 51.32 ms/ques +[INFO] Cumulative metrics, sample: 520, P: 0.526, R: 0.577, macro F1: 0.550 +[INFO] Current question consumed time: 1047 ms, average: 2493.40 ms/ques, entity detection: 382.53 ms/ques (linking tool: 301.85 ms/ques), relation detection: 831.90 ms/ques (similarity: 686.47 ms/ques), EDG generation: 51.32 ms/ques [INFO] Entity Linking P: 0.75, R: 0.7048076923076924, micro F1: 0.7192307692307698, macro F1: 0.726701916721745 [INFO] Relation Linking P: 0.6099358974358974, R: 0.5576923076923077, micro F1: 0.5721794871794871, macro F1: 0.5826453261259739 [INFO] Type Linking P: 0.5403846153846154, R: 0.5403846153846154, micro F1: 0.5403846153846154, macro F1: 0.5403846153846154 @@ -7292,8 +7292,8 @@ Question 520: Name the television show where Mona Marshall has given voice and i [INFO] Golden Answer: [http://dbpedia.org/resource/South_Park] [INFO] Predicted Answer: [http://dbpedia.org/resource/South_Park, http://dbpedia.org/resource/Key_&_Peele, http://dbpedia.org/resource/Patton_Oswalt:_Tragedy_Plus_Comedy_Equals_Time, http://dbpedia.org/resource/No_Reason_to_Complain, http://dbpedia.org/resource/Jeff_Dunham's_Very_Special_Christmas_Special, http://dbpedia.org/resource/Jeff_Dunham:_Arguing_with_Myself, http://dbpedia.org/resource/Jeff_Dunham:_Spark_of_Insanity, http://dbpedia.org/resource/Make_Me_Laugh] [INFO] P: 0.125, R: 1.0, F: 0.2222222222222222 -[INFO] Cumulative metrics, sample: 521, P: 0.525, R: 0.578, macro F1: 0.531, macro F1*: 0.550 -[INFO] Current time: 2531 ms, average: 2722.45 ms/ques, entity detection: 422.97 ms/ques (linking tool: 331.00 ms/ques), relation detection: 888.91 ms/ques (similarity: 700.00 ms/ques), EDG generation: 51.33 ms/ques +[INFO] Cumulative metrics, sample: 521, P: 0.525, R: 0.578, macro F1: 0.550 +[INFO] Current question consumed time: 2184 ms, average: 2492.81 ms/ques, entity detection: 382.35 ms/ques (linking tool: 301.65 ms/ques), relation detection: 831.55 ms/ques (similarity: 686.83 ms/ques), EDG generation: 51.32 ms/ques [INFO] Entity Linking P: 0.7504798464491362, R: 0.7044145873320538, micro F1: 0.7191298784389001, macro F1: 0.7267179515747573 [INFO] Relation Linking P: 0.610684580934101, R: 0.5575815738963532, micro F1: 0.5723608445297504, macro F1: 0.582926190891642 [INFO] Type Linking P: 0.5393474088291746, R: 0.5393474088291746, micro F1: 0.5393474088291746, macro F1: 0.5393474088291746 @@ -7306,8 +7306,8 @@ Question 521: What are Jerry Bock's musicals based upon? [INFO] Golden Answer: [http://dbpedia.org/resource/Mark_Twain, http://dbpedia.org/resource/Jules_Feiffer, http://dbpedia.org/resource/Frank_R._Stockton, http://dbpedia.org/resource/Tevye, http://dbpedia.org/resource/Frederic_Morton, http://dbpedia.org/resource/New_York_City, http://dbpedia.org/resource/Miklós_László, http://dbpedia.org/resource/Samuel_Hopkins_Adams, http://dbpedia.org/resource/Tenderloin_(novel)] [INFO] Predicted Answer: [http://dbpedia.org/resource/A_Stranger_Among_Us, http://dbpedia.org/resource/Fiddler_on_the_Roof_(film), http://dbpedia.org/resource/The_Apple_Tree, http://dbpedia.org/resource/Fiddler_on_the_Roof, http://dbpedia.org/resource/The_Rothschilds_(musical), http://dbpedia.org/resource/Mr._Wonderful_(musical), http://dbpedia.org/resource/The_Body_Beautiful, http://dbpedia.org/resource/Fiorello!, http://dbpedia.org/resource/She_Loves_Me, http://dbpedia.org/resource/Tenderloin_(musical)] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 522, P: 0.524, R: 0.577, macro F1: 0.530, macro F1*: 0.549 -[INFO] Current time: 1245 ms, average: 2719.61 ms/ques, entity detection: 422.69 ms/ques (linking tool: 330.63 ms/ques), relation detection: 887.62 ms/ques (similarity: 698.91 ms/ques), EDG generation: 51.25 ms/ques +[INFO] Cumulative metrics, sample: 522, P: 0.524, R: 0.577, macro F1: 0.549 +[INFO] Current question consumed time: 1005 ms, average: 2489.96 ms/ques, entity detection: 382.03 ms/ques (linking tool: 301.28 ms/ques), relation detection: 830.31 ms/ques (similarity: 685.75 ms/ques), EDG generation: 51.25 ms/ques [INFO] Entity Linking P: 0.7509578544061303, R: 0.7049808429118773, micro F1: 0.7196679438058754, macro F1: 0.7272433958459367 [INFO] Relation Linking P: 0.6114303959131545, R: 0.5574712643678161, micro F1: 0.5725415070242657, macro F1: 0.5832053926601298 [INFO] Type Linking P: 0.5383141762452107, R: 0.5383141762452107, micro F1: 0.5383141762452107, macro F1: 0.5383141762452107 @@ -7320,8 +7320,8 @@ Question 522: Give me list of people who were the first to climb a peak in the Y [INFO] Golden Answer: [http://dbpedia.org/resource/John_Muir, http://dbpedia.org/resource/John_Salathé, http://dbpedia.org/resource/Allen_Steck, http://dbpedia.org/resource/Clarence_King, http://dbpedia.org/resource/James_Terry_Gardiner, http://dbpedia.org/resource/Glen_Dawson, http://dbpedia.org/resource/Jules_Eichorn, http://dbpedia.org/resource/François_E._Matthes] [INFO] Predicted Answer: [http://dbpedia.org/resource/Tobias_Wolff] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 523, P: 0.523, R: 0.576, macro F1: 0.529, macro F1*: 0.548 -[INFO] Current time: 2048 ms, average: 2718.33 ms/ques, entity detection: 422.26 ms/ques (linking tool: 330.24 ms/ques), relation detection: 887.16 ms/ques (similarity: 698.31 ms/ques), EDG generation: 51.39 ms/ques +[INFO] Cumulative metrics, sample: 523, P: 0.523, R: 0.576, macro F1: 0.548 +[INFO] Current question consumed time: 1626 ms, average: 2488.31 ms/ques, entity detection: 381.56 ms/ques (linking tool: 300.87 ms/ques), relation detection: 829.80 ms/ques (similarity: 685.18 ms/ques), EDG generation: 51.41 ms/ques [INFO] Entity Linking P: 0.7495219885277247, R: 0.7036328871892925, micro F1: 0.7182919056724034, macro F1: 0.7258528731005334 [INFO] Relation Linking P: 0.6102613129381771, R: 0.5564053537284895, micro F1: 0.5714467813894201, macro F1: 0.5820902771865923 [INFO] Type Linking P: 0.5391969407265774, R: 0.5391969407265774, micro F1: 0.5391969407265774, macro F1: 0.5391969407265774 @@ -7334,8 +7334,8 @@ Question 523: What is Naval Support Activity Naples a part of? [INFO] Golden Answer: [http://dbpedia.org/resource/United_States_Navy] [INFO] Predicted Answer: [IT@en] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 524, P: 0.522, R: 0.575, macro F1: 0.528, macro F1*: 0.547 -[INFO] Current time: 924 ms, average: 2714.91 ms/ques, entity detection: 421.76 ms/ques (linking tool: 329.79 ms/ques), relation detection: 885.74 ms/ques (similarity: 697.14 ms/ques), EDG generation: 51.31 ms/ques +[INFO] Cumulative metrics, sample: 524, P: 0.522, R: 0.575, macro F1: 0.547 +[INFO] Current question consumed time: 721 ms, average: 2484.94 ms/ques, entity detection: 381.05 ms/ques (linking tool: 300.43 ms/ques), relation detection: 828.47 ms/ques (similarity: 684.03 ms/ques), EDG generation: 51.35 ms/ques [INFO] Entity Linking P: 0.75, R: 0.7041984732824428, micro F1: 0.7188295165394408, macro F1: 0.7263779527559054 [INFO] Relation Linking P: 0.6090966921119593, R: 0.5553435114503816, micro F1: 0.5703562340966921, macro F1: 0.5809794178789843 [INFO] Type Linking P: 0.5400763358778626, R: 0.5400763358778626, micro F1: 0.5400763358778626, macro F1: 0.5400763358778626 @@ -7348,8 +7348,8 @@ Question 524: Who has allegiance is Colony of Virginia and buried in Montpelier? [INFO] Golden Answer: [http://dbpedia.org/resource/James_Madison] [INFO] Predicted Answer: [http://dbpedia.org/resource/Alexander_Spotswood, http://dbpedia.org/resource/Francis_Dade_(politician), http://dbpedia.org/resource/George_Bowman_(pioneer), http://dbpedia.org/resource/Edward_Major, http://dbpedia.org/resource/Alexander_White_(Virginia)] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 525, P: 0.521, R: 0.574, macro F1: 0.527, macro F1*: 0.546 -[INFO] Current time: 6299 ms, average: 2721.73 ms/ques, entity detection: 422.92 ms/ques (linking tool: 330.84 ms/ques), relation detection: 888.69 ms/ques (similarity: 698.47 ms/ques), EDG generation: 51.24 ms/ques +[INFO] Cumulative metrics, sample: 525, P: 0.521, R: 0.574, macro F1: 0.546 +[INFO] Current question consumed time: 5091 ms, average: 2489.90 ms/ques, entity detection: 381.99 ms/ques (linking tool: 301.29 ms/ques), relation detection: 830.84 ms/ques (similarity: 685.37 ms/ques), EDG generation: 51.27 ms/ques [INFO] Entity Linking P: 0.7504761904761905, R: 0.7038095238095238, micro F1: 0.7187301587301593, macro F1: 0.7263941123273148 [INFO] Relation Linking P: 0.6079365079365079, R: 0.5542857142857143, micro F1: 0.5692698412698413, macro F1: 0.5798727904163576 [INFO] Type Linking P: 0.540952380952381, R: 0.540952380952381, micro F1: 0.540952380952381, macro F1: 0.540952380952381 @@ -7362,8 +7362,8 @@ Question 525: Was Tim Gunn a guest on The Broken Code? [INFO] Golden Answer: [true] [INFO] Predicted Answer: [true] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 526, P: 0.522, R: 0.575, macro F1: 0.528, macro F1*: 0.547 -[INFO] Current time: 879 ms, average: 2718.23 ms/ques, entity detection: 422.71 ms/ques (linking tool: 330.56 ms/ques), relation detection: 887.24 ms/ques (similarity: 697.29 ms/ques), EDG generation: 51.16 ms/ques +[INFO] Cumulative metrics, sample: 526, P: 0.522, R: 0.575, macro F1: 0.547 +[INFO] Current question consumed time: 631 ms, average: 2486.37 ms/ques, entity detection: 381.68 ms/ques (linking tool: 300.98 ms/ques), relation detection: 829.46 ms/ques (similarity: 684.21 ms/ques), EDG generation: 51.20 ms/ques [INFO] Entity Linking P: 0.7509505703422054, R: 0.7043726235741445, micro F1: 0.7192648922686952, macro F1: 0.7269162281170138 [INFO] Relation Linking P: 0.6067807351077312, R: 0.5532319391634981, micro F1: 0.5681875792141952, macro F1: 0.5787703706627144 [INFO] Type Linking P: 0.5418250950570342, R: 0.5418250950570342, micro F1: 0.5418250950570342, macro F1: 0.5418250950570342 @@ -7376,8 +7376,8 @@ Question 526: What is the successor of PlayStation 4 [INFO] Golden Answer: [http://dbpedia.org/resource/PlayStation_3] [INFO] Predicted Answer: [http://dbpedia.org/resource/PlayStation_3] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 527, P: 0.523, R: 0.575, macro F1: 0.529, macro F1*: 0.548 -[INFO] Current time: 1883 ms, average: 2716.65 ms/ques, entity detection: 422.87 ms/ques (linking tool: 330.70 ms/ques), relation detection: 886.53 ms/ques (similarity: 696.49 ms/ques), EDG generation: 51.09 ms/ques +[INFO] Cumulative metrics, sample: 527, P: 0.523, R: 0.575, macro F1: 0.548 +[INFO] Current question consumed time: 1518 ms, average: 2484.53 ms/ques, entity detection: 381.71 ms/ques (linking tool: 301.02 ms/ques), relation detection: 828.75 ms/ques (similarity: 683.44 ms/ques), EDG generation: 51.12 ms/ques [INFO] Entity Linking P: 0.7514231499051234, R: 0.7049335863377609, micro F1: 0.7197975964579386, macro F1: 0.7274363522860021 [INFO] Relation Linking P: 0.60752688172043, R: 0.5540796963946869, micro F1: 0.5690069576217583, macro F1: 0.5795736982162774 [INFO] Type Linking P: 0.5426944971537002, R: 0.5426944971537002, micro F1: 0.5426944971537002, macro F1: 0.5426944971537002 @@ -7390,8 +7390,8 @@ Question 527: What is the sovereign state of the Dafydd Benfras where Jasmine Lo [INFO] Golden Answer: [http://dbpedia.org/resource/United_Kingdom] [INFO] Predicted Answer: [] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 528, P: 0.522, R: 0.574, macro F1: 0.528, macro F1*: 0.547 -[INFO] Current time: 1398 ms, average: 2714.15 ms/ques, entity detection: 422.75 ms/ques (linking tool: 330.43 ms/ques), relation detection: 885.07 ms/ques (similarity: 695.33 ms/ques), EDG generation: 51.12 ms/ques +[INFO] Cumulative metrics, sample: 528, P: 0.522, R: 0.574, macro F1: 0.547 +[INFO] Current question consumed time: 1205 ms, average: 2482.10 ms/ques, entity detection: 381.49 ms/ques (linking tool: 300.70 ms/ques), relation detection: 827.36 ms/ques (similarity: 682.28 ms/ques), EDG generation: 51.14 ms/ques [INFO] Entity Linking P: 0.75, R: 0.7035984848484849, micro F1: 0.7184343434343441, macro F1: 0.7260586319218241 [INFO] Relation Linking P: 0.6063762626262625, R: 0.553030303030303, micro F1: 0.567929292929293, macro F1: 0.5784760207575345 [INFO] Type Linking P: 0.5435606060606061, R: 0.5435606060606061, micro F1: 0.5435606060606061, macro F1: 0.5435606060606061 @@ -7404,8 +7404,8 @@ Question 528: Who did their highschool from a school whose mascot was a fighting [INFO] Golden Answer: [http://dbpedia.org/resource/Joe_Namath] [INFO] Predicted Answer: [] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 529, P: 0.521, R: 0.573, macro F1: 0.527, macro F1*: 0.546 -[INFO] Current time: 1172 ms, average: 2711.23 ms/ques, entity detection: 422.12 ms/ques (linking tool: 329.88 ms/ques), relation detection: 884.06 ms/ques (similarity: 694.69 ms/ques), EDG generation: 51.08 ms/ques +[INFO] Cumulative metrics, sample: 529, P: 0.521, R: 0.573, macro F1: 0.546 +[INFO] Current question consumed time: 1007 ms, average: 2479.32 ms/ques, entity detection: 380.90 ms/ques (linking tool: 300.20 ms/ques), relation detection: 826.38 ms/ques (similarity: 681.62 ms/ques), EDG generation: 51.11 ms/ques [INFO] Entity Linking P: 0.7485822306238186, R: 0.7022684310018904, micro F1: 0.717076244486453, macro F1: 0.7246861203302896 [INFO] Relation Linking P: 0.6052299936988027, R: 0.5519848771266541, micro F1: 0.5668557025834908, macro F1: 0.5773824933080873 [INFO] Type Linking P: 0.5444234404536862, R: 0.5444234404536862, micro F1: 0.5444234404536862, macro F1: 0.5444234404536862 @@ -7418,8 +7418,8 @@ Question 529: Which destination of Emerald Air is also the city served by housto [INFO] Golden Answer: [http://dbpedia.org/resource/Houston] [INFO] Predicted Answer: [http://dbpedia.org/resource/Houston] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 530, P: 0.522, R: 0.574, macro F1: 0.528, macro F1*: 0.547 -[INFO] Current time: 2043 ms, average: 2709.97 ms/ques, entity detection: 422.18 ms/ques (linking tool: 329.62 ms/ques), relation detection: 883.65 ms/ques (similarity: 694.14 ms/ques), EDG generation: 51.16 ms/ques +[INFO] Cumulative metrics, sample: 530, P: 0.522, R: 0.574, macro F1: 0.547 +[INFO] Current question consumed time: 1662 ms, average: 2477.78 ms/ques, entity detection: 380.85 ms/ques (linking tool: 299.94 ms/ques), relation detection: 825.90 ms/ques (similarity: 681.06 ms/ques), EDG generation: 51.16 ms/ques [INFO] Entity Linking P: 0.7490566037735849, R: 0.7028301886792453, micro F1: 0.7176100628930824, macro F1: 0.7252074981303714 [INFO] Relation Linking P: 0.6059748427672955, R: 0.5528301886792453, micro F1: 0.5676729559748428, macro F1: 0.5781838662604644 [INFO] Type Linking P: 0.5452830188679245, R: 0.5452830188679245, micro F1: 0.5452830188679245, macro F1: 0.5452830188679245 @@ -7432,8 +7432,8 @@ Question 530: name the region served by Jerusalem Institue of Justice which was [INFO] Golden Answer: [http://dbpedia.org/resource/Israel] [INFO] Predicted Answer: [http://dbpedia.org/resource/Israel] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 531, P: 0.522, R: 0.575, macro F1: 0.529, macro F1*: 0.547 -[INFO] Current time: 7451 ms, average: 2718.90 ms/ques, entity detection: 422.50 ms/ques (linking tool: 330.04 ms/ques), relation detection: 886.60 ms/ques (similarity: 700.10 ms/ques), EDG generation: 51.28 ms/ques +[INFO] Cumulative metrics, sample: 531, P: 0.522, R: 0.575, macro F1: 0.547 +[INFO] Current question consumed time: 6703 ms, average: 2485.73 ms/ques, entity detection: 381.06 ms/ques (linking tool: 300.26 ms/ques), relation detection: 828.47 ms/ques (similarity: 686.97 ms/ques), EDG generation: 51.31 ms/ques [INFO] Entity Linking P: 0.7495291902071564, R: 0.7024482109227872, micro F1: 0.7175141242937859, macro F1: 0.72522539026529 [INFO] Relation Linking P: 0.6067168863779032, R: 0.5527306967984934, micro F1: 0.5678593848085374, macro F1: 0.5784669393132039 [INFO] Type Linking P: 0.5461393596986818, R: 0.5461393596986818, micro F1: 0.5461393596986818, macro F1: 0.5461393596986818 @@ -7446,8 +7446,8 @@ Question 531: Name the company whose key person is Raymond Conner and produces B [INFO] Golden Answer: [http://dbpedia.org/resource/Boeing] [INFO] Predicted Answer: [1962] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 532, P: 0.522, R: 0.574, macro F1: 0.528, macro F1*: 0.546 -[INFO] Current time: 2672 ms, average: 2718.81 ms/ques, entity detection: 422.51 ms/ques (linking tool: 329.93 ms/ques), relation detection: 886.53 ms/ques (similarity: 700.58 ms/ques), EDG generation: 51.23 ms/ques +[INFO] Cumulative metrics, sample: 532, P: 0.522, R: 0.574, macro F1: 0.546 +[INFO] Current question consumed time: 2328 ms, average: 2485.44 ms/ques, entity detection: 380.98 ms/ques (linking tool: 300.11 ms/ques), relation detection: 828.38 ms/ques (similarity: 687.45 ms/ques), EDG generation: 51.27 ms/ques [INFO] Entity Linking P: 0.75, R: 0.7020676691729323, micro F1: 0.7174185463659155, macro F1: 0.725242718446602 [INFO] Relation Linking P: 0.6055764411027568, R: 0.5516917293233082, micro F1: 0.5667919799498747, macro F1: 0.5773795954423144 [INFO] Type Linking P: 0.5451127819548872, R: 0.5451127819548872, micro F1: 0.5451127819548872, macro F1: 0.5451127819548872 @@ -7460,8 +7460,8 @@ Question 532: What is the total number of professions in which someone who has s [INFO] Golden Answer: [97] [INFO] Predicted Answer: [941] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 533, P: 0.521, R: 0.573, macro F1: 0.527, macro F1*: 0.545 -[INFO] Current time: 512 ms, average: 2714.67 ms/ques, entity detection: 421.72 ms/ques (linking tool: 329.32 ms/ques), relation detection: 885.25 ms/ques (similarity: 699.52 ms/ques), EDG generation: 51.22 ms/ques +[INFO] Cumulative metrics, sample: 533, P: 0.521, R: 0.573, macro F1: 0.545 +[INFO] Current question consumed time: 484 ms, average: 2481.68 ms/ques, entity detection: 380.27 ms/ques (linking tool: 299.55 ms/ques), relation detection: 827.16 ms/ques (similarity: 686.40 ms/ques), EDG generation: 51.38 ms/ques [INFO] Entity Linking P: 0.7485928705440901, R: 0.700750469043152, micro F1: 0.7160725453408388, macro F1: 0.723882037924188 [INFO] Relation Linking P: 0.6044402751719824, R: 0.550656660412758, micro F1: 0.5657285803627268, macro F1: 0.5762963316609968 [INFO] Type Linking P: 0.5440900562851783, R: 0.5440900562851783, micro F1: 0.5440900562851783, macro F1: 0.5440900562851783 @@ -7474,8 +7474,8 @@ Question 533: How many golf players are there in Arizona State Sun Devils ? [INFO] Golden Answer: [5] [INFO] Predicted Answer: [1] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 534, P: 0.520, R: 0.572, macro F1: 0.526, macro F1*: 0.544 -[INFO] Current time: 1781 ms, average: 2712.92 ms/ques, entity detection: 421.34 ms/ques (linking tool: 329.00 ms/ques), relation detection: 884.71 ms/ques (similarity: 699.42 ms/ques), EDG generation: 51.15 ms/ques +[INFO] Cumulative metrics, sample: 534, P: 0.520, R: 0.572, macro F1: 0.544 +[INFO] Current question consumed time: 1579 ms, average: 2479.99 ms/ques, entity detection: 379.91 ms/ques (linking tool: 299.25 ms/ques), relation detection: 826.63 ms/ques (similarity: 686.37 ms/ques), EDG generation: 51.28 ms/ques [INFO] Entity Linking P: 0.7490636704119851, R: 0.701310861423221, micro F1: 0.716604244694133, macro F1: 0.7244011480162386 [INFO] Relation Linking P: 0.6033083645443196, R: 0.549625468164794, micro F1: 0.5646691635455681, macro F1: 0.5752171250473994 [INFO] Type Linking P: 0.5430711610486891, R: 0.5430711610486891, micro F1: 0.5430711610486891, macro F1: 0.5430711610486891 @@ -7488,8 +7488,8 @@ Question 534: Count the number of characters in Batman Live? [INFO] Golden Answer: [11] [INFO] Predicted Answer: [0] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 535, P: 0.519, R: 0.571, macro F1: 0.525, macro F1*: 0.543 -[INFO] Current time: 1083 ms, average: 2709.88 ms/ques, entity detection: 421.04 ms/ques (linking tool: 328.73 ms/ques), relation detection: 883.27 ms/ques (similarity: 698.25 ms/ques), EDG generation: 51.07 ms/ques +[INFO] Cumulative metrics, sample: 535, P: 0.519, R: 0.571, macro F1: 0.543 +[INFO] Current question consumed time: 942 ms, average: 2477.12 ms/ques, entity detection: 379.61 ms/ques (linking tool: 298.99 ms/ques), relation detection: 825.28 ms/ques (similarity: 685.23 ms/ques), EDG generation: 51.21 ms/ques [INFO] Entity Linking P: 0.7476635514018691, R: 0.7, micro F1: 0.7152647975077888, macro F1: 0.723047127178825 [INFO] Relation Linking P: 0.6021806853582554, R: 0.5485981308411215, micro F1: 0.5636137071651091, macro F1: 0.5741419528510493 [INFO] Type Linking P: 0.5420560747663551, R: 0.5420560747663551, micro F1: 0.5420560747663551, macro F1: 0.5420560747663551 @@ -7502,8 +7502,8 @@ Question 535: List the services provided by the company which provides Microsoft [INFO] Golden Answer: [http://dbpedia.org/resource/Microsoft_Office, http://dbpedia.org/resource/Office_365, http://dbpedia.org/resource/Microsoft_Exchange_Server, http://dbpedia.org/resource/Skype, http://dbpedia.org/resource/Microsoft_Azure, http://dbpedia.org/resource/Web_hosting_service, http://dbpedia.org/resource/OneDrive, http://dbpedia.org/resource/Data_center_services, *@en] [INFO] Predicted Answer: [http://dbpedia.org/resource/Microsoft_Office, http://dbpedia.org/resource/Office_365, http://dbpedia.org/resource/Microsoft_Exchange_Server, http://dbpedia.org/resource/Skype, http://dbpedia.org/resource/Microsoft_Azure, http://dbpedia.org/resource/Web_hosting_service, http://dbpedia.org/resource/OneDrive, http://dbpedia.org/resource/Data_center_services, *@en] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 536, P: 0.519, R: 0.571, macro F1: 0.526, macro F1*: 0.544 -[INFO] Current time: 1943 ms, average: 2708.45 ms/ques, entity detection: 420.85 ms/ques (linking tool: 328.64 ms/ques), relation detection: 882.50 ms/ques (similarity: 697.82 ms/ques), EDG generation: 51.05 ms/ques +[INFO] Cumulative metrics, sample: 536, P: 0.519, R: 0.571, macro F1: 0.544 +[INFO] Current question consumed time: 1667 ms, average: 2475.60 ms/ques, entity detection: 379.40 ms/ques (linking tool: 298.87 ms/ques), relation detection: 824.55 ms/ques (similarity: 684.86 ms/ques), EDG generation: 51.21 ms/ques [INFO] Entity Linking P: 0.7481343283582089, R: 0.7005597014925373, micro F1: 0.7157960199004982, macro F1: 0.723565847517083 [INFO] Relation Linking P: 0.6019900497512437, R: 0.5494402985074627, micro F1: 0.5638059701492538, macro F1: 0.5745160237161479 [INFO] Type Linking P: 0.5410447761194029, R: 0.5410447761194029, micro F1: 0.5410447761194029, macro F1: 0.5410447761194029 @@ -7516,8 +7516,8 @@ Question 536: What are some team members of the team based in Camp Nou? [INFO] Golden Answer: [http://dbpedia.org/resource/Santiago_Ezquerro, http://dbpedia.org/resource/Albert_Jorquera, http://dbpedia.org/resource/Damià_Abella, http://dbpedia.org/resource/Jordi_Gómez, http://dbpedia.org/resource/Jesús_Olmo, http://dbpedia.org/resource/Sergio_Rodríguez_García, http://dbpedia.org/resource/Deco, http://dbpedia.org/resource/Sylvinho, http://dbpedia.org/resource/Giovanni_van_Bronckhorst, http://dbpedia.org/resource/Rafael_Márquez, http://dbpedia.org/resource/Henrik_Larsson, http://dbpedia.org/resource/Gabri_García, http://dbpedia.org/resource/Ramón_Masó, http://dbpedia.org/resource/Carles_Puyol, http://dbpedia.org/resource/Mark_van_Bommel, http://dbpedia.org/resource/Thiago_Motta, http://dbpedia.org/resource/Francisco_Martos, http://dbpedia.org/resource/Maxi_López, http://dbpedia.org/resource/Samuel_Eto'o, http://dbpedia.org/resource/Andrés_Iniesta, http://dbpedia.org/resource/Lionel_Messi, http://dbpedia.org/resource/Ronaldinho, http://dbpedia.org/resource/Víctor_Valdés, http://dbpedia.org/resource/Carlos_Peña_(footballer), http://dbpedia.org/resource/Joan_Verdú, http://dbpedia.org/resource/Francisco_Montañés, http://dbpedia.org/resource/Edmílson_(footballer,_born_1976), http://dbpedia.org/resource/Juliano_Belletti, http://dbpedia.org/resource/Ludovic_Giuly, http://dbpedia.org/resource/Rubén_Iván_Martínez, http://dbpedia.org/resource/Xavi, http://dbpedia.org/resource/Pitu, http://dbpedia.org/resource/Ludovic_Sylvestre, http://dbpedia.org/resource/Andrea_Orlandi, http://dbpedia.org/resource/Oleguer_Presas, http://dbpedia.org/resource/Cristian_Hidalgo, http://dbpedia.org/resource/Edgar_Ié, http://dbpedia.org/resource/Diawandou_Diagne, http://dbpedia.org/resource/Munir_El_Haddadi, http://dbpedia.org/resource/Sandro_Ramírez, http://dbpedia.org/resource/Alen_Halilović, http://dbpedia.org/resource/Fabrice_Ondoa, http://dbpedia.org/resource/Adama_Traoré_(footballer,_born_1996), http://dbpedia.org/resource/Álex_Grimaldo, http://dbpedia.org/resource/Sergi_Samper, http://dbpedia.org/resource/Gerard_Gumbau, http://dbpedia.org/resource/Luís_Figo, http://dbpedia.org/resource/Sonny_Anderson, http://dbpedia.org/resource/Patrick_Kluivert, http://dbpedia.org/resource/Óscar_García_Junyent, http://dbpedia.org/resource/Dragan_Ćirić, http://dbpedia.org/resource/Samuel_Okunowo, http://dbpedia.org/resource/Boudewijn_Zenden, http://dbpedia.org/resource/Phillip_Cocu, http://dbpedia.org/resource/Mauricio_Pellegrino, http://dbpedia.org/resource/Vítor_Baía, http://dbpedia.org/resource/Francesc_Arnau, http://dbpedia.org/resource/Frank_de_Boer, http://dbpedia.org/resource/Luis_Enrique_(footballer), http://dbpedia.org/resource/Michael_Reiziger, http://dbpedia.org/resource/Rivaldo, http://dbpedia.org/resource/Roger_García_Junyent, http://dbpedia.org/resource/Ronald_de_Boer, http://dbpedia.org/resource/Winston_Bogarde, http://dbpedia.org/resource/Emmanuel_Amunike, http://dbpedia.org/resource/Pep_Guardiola, http://dbpedia.org/resource/Sergi_Barjuán, http://dbpedia.org/resource/Luis_García_(footballer,_born_1978), http://dbpedia.org/resource/Miguel_Ángel_Nadal, http://dbpedia.org/resource/Carles_Busquets, http://dbpedia.org/resource/Giovanni_Silva_de_Oliveira, http://dbpedia.org/resource/Abelardo_Fernández, http://dbpedia.org/resource/Ruud_Hesp, http://dbpedia.org/resource/Albert_Celades, http://dbpedia.org/resource/Ibán_Cuadrado, http://dbpedia.org/resource/Rüştü_Reçber, http://dbpedia.org/resource/Ramón_Ros, http://dbpedia.org/resource/Edgar_Davids, http://dbpedia.org/resource/Marc_Overmars, http://dbpedia.org/resource/Javier_Saviola, http://dbpedia.org/resource/Pedro_Mario_Álvarez, http://dbpedia.org/resource/Gerard_López, http://dbpedia.org/resource/Oriol_Riera, http://dbpedia.org/resource/Sergio_García_(footballer,_born_1983), http://dbpedia.org/resource/Sergio_Santamaría, http://dbpedia.org/resource/Óscar_López_(footballer), http://dbpedia.org/resource/Ricardo_Quaresma, http://dbpedia.org/resource/Ronnie_Ekelund, http://dbpedia.org/resource/Iván_Iglesias, http://dbpedia.org/resource/Andoni_Zubizarreta, http://dbpedia.org/resource/Ion_Andoni_Goikoetxea, http://dbpedia.org/resource/Eusebio_Sacristán, http://dbpedia.org/resource/Txiki_Begiristain, http://dbpedia.org/resource/Guillermo_Amor, http://dbpedia.org/resource/Ronald_Koeman, http://dbpedia.org/resource/Jesús_Angoy, http://dbpedia.org/resource/Hristo_Stoichkov, http://dbpedia.org/resource/Albert_Ferrer, http://dbpedia.org/resource/Michael_Laudrup, http://dbpedia.org/resource/Romário, http://dbpedia.org/resource/José_Mari_Bakero, http://dbpedia.org/resource/Quique_Estebaranz, http://dbpedia.org/resource/Goran_Vučević, http://dbpedia.org/resource/Julio_Salinas, http://dbpedia.org/resource/Juan_Carlos_(footballer,_born_1965), http://dbpedia.org/resource/Jonathan_dos_Santos, http://dbpedia.org/resource/Gai_Assulin, http://dbpedia.org/resource/Thiago_Alcântara, http://dbpedia.org/resource/Rubén_Miño, http://dbpedia.org/resource/Marc_Bartra, http://dbpedia.org/resource/Jonathan_Soriano, http://dbpedia.org/resource/Víctor_Vázquez, http://dbpedia.org/resource/Andreu_Fontàs, http://dbpedia.org/resource/Marc_Muniesa, http://dbpedia.org/resource/Sergi_Roberto, http://dbpedia.org/resource/Rafinha_(footballer,_born_1993), http://dbpedia.org/resource/Kiko_Femenía, http://dbpedia.org/resource/Cristian_Tello, http://dbpedia.org/resource/Martín_Montoya, http://dbpedia.org/resource/Gerard_Deulofeu, http://dbpedia.org/resource/Martí_Riverola, http://dbpedia.org/resource/Sergi_Gómez, http://dbpedia.org/resource/Oier_Olazábal, http://dbpedia.org/resource/Carles_Planas, http://dbpedia.org/resource/Jordi_Masip, http://dbpedia.org/resource/Dani_Romera, http://dbpedia.org/resource/Juan_Cámara, http://dbpedia.org/resource/Wilfrid_Kaptoum, Carles Alena@en, http://dbpedia.org/resource/Demetrio_Albertini, http://dbpedia.org/resource/Fernando_Navarro, http://dbpedia.org/resource/Javito, http://dbpedia.org/resource/Oriol_Romeu, http://dbpedia.org/resource/Nolito, http://dbpedia.org/resource/Fábio_Rochemback, http://dbpedia.org/resource/Gaizka_Mendieta, http://dbpedia.org/resource/Patrik_Andersson, http://dbpedia.org/resource/Philippe_Christanval, http://dbpedia.org/resource/Robert_Enke, http://dbpedia.org/resource/Roberto_Bonano, http://dbpedia.org/resource/David_Sánchez_(footballer), http://dbpedia.org/resource/Daniel_García_(footballer,_born_1974), http://dbpedia.org/resource/Juan_Román_Riquelme, http://dbpedia.org/resource/Dani_Tortolero, http://dbpedia.org/resource/Juan_Pablo_Sorín, http://dbpedia.org/resource/Nano_(Spanish_footballer,_born_1982)] [INFO] Predicted Answer: [] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 537, P: 0.519, R: 0.570, macro F1: 0.525, macro F1*: 0.543 -[INFO] Current time: 10601 ms, average: 2723.14 ms/ques, entity detection: 420.43 ms/ques (linking tool: 328.27 ms/ques), relation detection: 892.91 ms/ques (similarity: 702.30 ms/ques), EDG generation: 51.14 ms/ques +[INFO] Cumulative metrics, sample: 537, P: 0.519, R: 0.570, macro F1: 0.543 +[INFO] Current question consumed time: 9657 ms, average: 2488.98 ms/ques, entity detection: 378.99 ms/ques (linking tool: 298.52 ms/ques), relation detection: 834.17 ms/ques (similarity: 689.34 ms/ques), EDG generation: 51.30 ms/ques [INFO] Entity Linking P: 0.7467411545623837, R: 0.6992551210428305, micro F1: 0.7144630664183743, macro F1: 0.722218425082228 [INFO] Relation Linking P: 0.600869025450031, R: 0.5484171322160148, micro F1: 0.5627560521415271, macro F1: 0.5734461614745907 [INFO] Type Linking P: 0.5418994413407822, R: 0.5418994413407822, micro F1: 0.5418994413407822, macro F1: 0.5418994413407822 @@ -7530,8 +7530,8 @@ Question 537: Which universities are also known as the Tulane Green wave? [INFO] Golden Answer: [http://dbpedia.org/resource/Tulane_University, http://dbpedia.org/resource/Tulane_Green_Wave_women's_basketball, http://dbpedia.org/resource/H._Sophie_Newcomb_Memorial_College] [INFO] Predicted Answer: [http://dbpedia.org/resource/Tulane_University] [INFO] P: 1.0, R: 0.3333333333333333, F: 0.5 -[INFO] Cumulative metrics, sample: 538, P: 0.519, R: 0.570, macro F1: 0.525, macro F1*: 0.543 -[INFO] Current time: 1777 ms, average: 2721.38 ms/ques, entity detection: 420.00 ms/ques (linking tool: 327.89 ms/ques), relation detection: 892.47 ms/ques (similarity: 702.40 ms/ques), EDG generation: 51.08 ms/ques +[INFO] Cumulative metrics, sample: 538, P: 0.519, R: 0.570, macro F1: 0.543 +[INFO] Current question consumed time: 1672 ms, average: 2487.46 ms/ques, entity detection: 378.64 ms/ques (linking tool: 298.20 ms/ques), relation detection: 833.75 ms/ques (similarity: 689.49 ms/ques), EDG generation: 51.23 ms/ques [INFO] Entity Linking P: 0.7472118959107806, R: 0.699814126394052, micro F1: 0.7149938042131357, macro F1: 0.7227367471044545 [INFO] Relation Linking P: 0.5997521685254027, R: 0.5473977695167286, micro F1: 0.5617100371747212, macro F1: 0.5723802764160878 [INFO] Type Linking P: 0.5408921933085502, R: 0.5408921933085502, micro F1: 0.5408921933085502, macro F1: 0.5408921933085502 @@ -7544,8 +7544,8 @@ Question 538: What is the branch of the politicians whose governor is Bill Cleme [INFO] Golden Answer: [http://dbpedia.org/resource/United_States_Army, http://dbpedia.org/resource/Texas_Military_Forces, http://dbpedia.org/resource/United_States_Navy, United States Navy@en] [INFO] Predicted Answer: [http://dbpedia.org/resource/United_States_Army, http://dbpedia.org/resource/Texas_Military_Forces, http://dbpedia.org/resource/United_States_Navy, United States Navy@en, http://dbpedia.org/resource/United_States_Air_Force] [INFO] P: 0.75, R: 1.0, F: 0.8571428571428571 -[INFO] Cumulative metrics, sample: 539, P: 0.520, R: 0.571, macro F1: 0.525, macro F1*: 0.544 -[INFO] Current time: 2615 ms, average: 2721.19 ms/ques, entity detection: 419.51 ms/ques (linking tool: 327.46 ms/ques), relation detection: 893.15 ms/ques (similarity: 703.06 ms/ques), EDG generation: 51.03 ms/ques +[INFO] Cumulative metrics, sample: 539, P: 0.520, R: 0.571, macro F1: 0.544 +[INFO] Current question consumed time: 2389 ms, average: 2487.28 ms/ques, entity detection: 378.20 ms/ques (linking tool: 297.80 ms/ques), relation detection: 834.37 ms/ques (similarity: 690.22 ms/ques), EDG generation: 51.16 ms/ques [INFO] Entity Linking P: 0.7476808905380334, R: 0.7003710575139147, micro F1: 0.7155225726654305, macro F1: 0.7232531356261567 [INFO] Relation Linking P: 0.6004947433518861, R: 0.5482374768089053, micro F1: 0.5625231910946198, macro F1: 0.573177485848126 [INFO] Type Linking P: 0.5398886827458256, R: 0.5398886827458256, micro F1: 0.5398886827458256, macro F1: 0.5398886827458256 @@ -7558,8 +7558,8 @@ Question 539: Who composed the episode after which came Butter's Very own Episod [INFO] Golden Answer: [http://dbpedia.org/resource/Willie_Nelson, http://dbpedia.org/resource/On_the_Road_Again_(Willie_Nelson_song)] [INFO] Predicted Answer: [http://dbpedia.org/resource/Cornell_University] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 540, P: 0.519, R: 0.570, macro F1: 0.524, macro F1*: 0.543 -[INFO] Current time: 2347 ms, average: 2720.49 ms/ques, entity detection: 419.48 ms/ques (linking tool: 327.33 ms/ques), relation detection: 893.19 ms/ques (similarity: 702.73 ms/ques), EDG generation: 50.98 ms/ques +[INFO] Cumulative metrics, sample: 540, P: 0.519, R: 0.570, macro F1: 0.543 +[INFO] Current question consumed time: 2000 ms, average: 2486.37 ms/ques, entity detection: 378.08 ms/ques (linking tool: 297.65 ms/ques), relation detection: 834.36 ms/ques (similarity: 689.90 ms/ques), EDG generation: 51.09 ms/ques [INFO] Entity Linking P: 0.7462962962962963, R: 0.6990740740740741, micro F1: 0.7141975308641982, macro F1: 0.7219137779675896 [INFO] Relation Linking P: 0.5993827160493826, R: 0.5472222222222223, micro F1: 0.5614814814814816, macro F1: 0.5721160460595186 [INFO] Type Linking P: 0.5388888888888889, R: 0.5388888888888889, micro F1: 0.5388888888888889, macro F1: 0.5388888888888889 @@ -7572,8 +7572,8 @@ Question 540: What is the location country of Teperberg 1870 ? [INFO] Golden Answer: [http://dbpedia.org/resource/Israel] [INFO] Predicted Answer: [http://dbpedia.org/resource/Israel] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 541, P: 0.520, R: 0.570, macro F1: 0.525, macro F1*: 0.544 -[INFO] Current time: 426 ms, average: 2716.25 ms/ques, entity detection: 418.87 ms/ques (linking tool: 326.88 ms/ques), relation detection: 891.70 ms/ques (similarity: 701.56 ms/ques), EDG generation: 50.92 ms/ques +[INFO] Cumulative metrics, sample: 541, P: 0.520, R: 0.570, macro F1: 0.544 +[INFO] Current question consumed time: 376 ms, average: 2482.47 ms/ques, entity detection: 377.53 ms/ques (linking tool: 297.25 ms/ques), relation detection: 832.99 ms/ques (similarity: 688.77 ms/ques), EDG generation: 51.06 ms/ques [INFO] Entity Linking P: 0.7467652495378928, R: 0.699630314232902, micro F1: 0.714725816389403, macro F1: 0.7224297685625365 [INFO] Relation Linking P: 0.6001232285890326, R: 0.5480591497227357, micro F1: 0.5622920517560075, macro F1: 0.5729107720203321 [INFO] Type Linking P: 0.5397412199630314, R: 0.5397412199630314, micro F1: 0.5397412199630314, macro F1: 0.5397412199630314 @@ -7586,8 +7586,8 @@ Question 541: List the software developed by Shawn Fanning ? [INFO] Golden Answer: [http://dbpedia.org/resource/Napster] [INFO] Predicted Answer: [http://dbpedia.org/resource/Napster] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 542, P: 0.521, R: 0.571, macro F1: 0.526, macro F1*: 0.545 -[INFO] Current time: 1021 ms, average: 2713.13 ms/ques, entity detection: 418.39 ms/ques (linking tool: 326.44 ms/ques), relation detection: 890.71 ms/ques (similarity: 700.90 ms/ques), EDG generation: 50.84 ms/ques +[INFO] Cumulative metrics, sample: 542, P: 0.521, R: 0.571, macro F1: 0.545 +[INFO] Current question consumed time: 906 ms, average: 2479.56 ms/ques, entity detection: 377.06 ms/ques (linking tool: 296.82 ms/ques), relation detection: 832.05 ms/ques (similarity: 688.15 ms/ques), EDG generation: 50.99 ms/ques [INFO] Entity Linking P: 0.7472324723247232, R: 0.7001845018450185, micro F1: 0.7152521525215259, macro F1: 0.7229438451172274 [INFO] Relation Linking P: 0.6008610086100861, R: 0.5488929889298892, micro F1: 0.56309963099631, macro F1: 0.5737025409836873 [INFO] Type Linking P: 0.5387453874538746, R: 0.5387453874538746, micro F1: 0.5387453874538746, macro F1: 0.5387453874538746 @@ -7600,8 +7600,8 @@ Question 542: What is the stylistic origin of the music genres where the Bass gu [INFO] Golden Answer: [http://dbpedia.org/resource/Disco, http://dbpedia.org/resource/Jazz, http://dbpedia.org/resource/Funk, http://dbpedia.org/resource/Soul_music, http://dbpedia.org/resource/Classical_music, http://dbpedia.org/resource/Heavy_metal_music, http://dbpedia.org/resource/Art_rock, http://dbpedia.org/resource/Baroque_pop, http://dbpedia.org/resource/Contemporary_Christian_music, http://dbpedia.org/resource/Jesus_music, http://dbpedia.org/resource/Hip_hop_music, http://dbpedia.org/resource/Electronic_music, http://dbpedia.org/resource/Ambient_music, http://dbpedia.org/resource/Chill-out_music, http://dbpedia.org/resource/Dub_music, http://dbpedia.org/resource/Groove_(music), http://dbpedia.org/resource/House_music, http://dbpedia.org/resource/Hi-NRG, http://dbpedia.org/resource/Glam_rock, http://dbpedia.org/resource/Pop_rock, http://dbpedia.org/resource/Punk_rock, http://dbpedia.org/resource/Hard_rock, http://dbpedia.org/resource/Avant-garde, http://dbpedia.org/resource/Experimental_rock, http://dbpedia.org/resource/Progressive_rock, http://dbpedia.org/resource/Psychedelic_rock, http://dbpedia.org/resource/Experimental_music, http://dbpedia.org/resource/Emo, http://dbpedia.org/resource/Hardcore_punk, http://dbpedia.org/resource/Krautrock, http://dbpedia.org/resource/Proto-punk, http://dbpedia.org/resource/British_rock_and_roll, http://dbpedia.org/resource/Pop_music, http://dbpedia.org/resource/Rhythm_and_blues, http://dbpedia.org/resource/Skiffle, http://dbpedia.org/resource/Doo-wop, http://dbpedia.org/resource/Beat_music, http://dbpedia.org/resource/Blues, http://dbpedia.org/resource/Rockabilly, http://dbpedia.org/resource/Rock_and_roll, http://dbpedia.org/resource/Instrumental_rock, http://dbpedia.org/resource/Surf_music, http://dbpedia.org/resource/New_Wave_of_British_Heavy_Metal, http://dbpedia.org/resource/Speed_metal, http://dbpedia.org/resource/Neoclassical_metal, http://dbpedia.org/resource/Garage_rock, http://dbpedia.org/resource/Pub_rock_(United_Kingdom), http://dbpedia.org/resource/Power_pop, http://dbpedia.org/resource/Horror_film_score, http://dbpedia.org/resource/Acid_rock, http://dbpedia.org/resource/Post-punk, http://dbpedia.org/resource/Folk_music, http://dbpedia.org/resource/Dance_music, http://dbpedia.org/resource/Music_of_Korea, http://dbpedia.org/resource/Synthpop, http://dbpedia.org/resource/Electronic_rock, http://dbpedia.org/resource/Son_(music), http://dbpedia.org/resource/Jazz_fusion, http://dbpedia.org/resource/Ska, http://dbpedia.org/resource/Japanese_jazz, http://dbpedia.org/resource/J-pop, http://dbpedia.org/resource/Yé-yé, http://dbpedia.org/resource/Lounge_music, http://dbpedia.org/resource/Bossa_nova, http://dbpedia.org/resource/Rocksteady, http://dbpedia.org/resource/New_wave_music, http://dbpedia.org/resource/Reggae, http://dbpedia.org/resource/Folk_rock, http://dbpedia.org/resource/British_Invasion, http://dbpedia.org/resource/Blues_rock, http://dbpedia.org/resource/Boogie-woogie, http://dbpedia.org/resource/Rock_music, http://dbpedia.org/resource/Music_of_Brazil, http://dbpedia.org/resource/British_rock, http://dbpedia.org/resource/Kayōkyoku, http://dbpedia.org/resource/Dancehall, http://dbpedia.org/resource/Indie_rock, http://dbpedia.org/resource/Shoegazing, http://dbpedia.org/resource/Alternative_country, http://dbpedia.org/resource/Reggae_fusion, http://dbpedia.org/resource/Samba, http://dbpedia.org/resource/Brazilian_rock, http://dbpedia.org/resource/Bebop, http://dbpedia.org/resource/Sufi_music, http://dbpedia.org/resource/Hindustani_classical_music, http://dbpedia.org/resource/Alternative_rock, http://dbpedia.org/resource/Post-rock, http://dbpedia.org/resource/Acoustic_music, http://dbpedia.org/resource/Dream_pop, http://dbpedia.org/resource/Musical_theatre, http://dbpedia.org/resource/Concept_album, http://dbpedia.org/resource/Opera, http://dbpedia.org/resource/Rock_opera, http://dbpedia.org/resource/Contemporary_R&B, http://dbpedia.org/resource/Spoken_word, http://dbpedia.org/resource/Frevo, http://dbpedia.org/resource/Coco_(music), http://dbpedia.org/resource/Forró, http://dbpedia.org/resource/Maracatu, http://dbpedia.org/resource/Crossover_thrash, http://dbpedia.org/resource/Powerviolence, http://dbpedia.org/resource/Thrashcore, http://dbpedia.org/resource/Youth_crew, http://dbpedia.org/resource/Electronic_dance_music, http://dbpedia.org/resource/Music_of_Africa, http://dbpedia.org/resource/Work_song, http://dbpedia.org/resource/Spiritual_(music), http://dbpedia.org/resource/Ballad, http://dbpedia.org/resource/Celtic_music, http://dbpedia.org/resource/Appalachian_music, http://dbpedia.org/resource/Old-time_music, http://dbpedia.org/resource/Electro_(music), http://dbpedia.org/resource/Post-disco, http://dbpedia.org/resource/Electropop, http://dbpedia.org/resource/Black_metal, http://dbpedia.org/resource/Thrash_metal, http://dbpedia.org/resource/Philadelphia_soul, http://dbpedia.org/resource/Psychedelic_soul, http://dbpedia.org/resource/Bubblegum_pop, http://dbpedia.org/resource/Latin_music_(genre), http://dbpedia.org/resource/Post-hardcore, http://dbpedia.org/resource/Uplifting_trance, http://dbpedia.org/resource/Vocal_trance, http://dbpedia.org/resource/Disco_polo, http://dbpedia.org/resource/Italo_disco, http://dbpedia.org/resource/Euro_disco, http://dbpedia.org/resource/Eurobeat, http://dbpedia.org/resource/Noise_rock, http://dbpedia.org/resource/Extreme_metal, http://dbpedia.org/resource/Mento, http://dbpedia.org/resource/Calypso_music, http://dbpedia.org/resource/Gospel_music, http://dbpedia.org/resource/Country_music, http://dbpedia.org/resource/Electric_blues, http://dbpedia.org/resource/Cadence_rampa, http://dbpedia.org/resource/Kaiso, http://dbpedia.org/resource/Breakbeat, http://dbpedia.org/resource/Downtempo, http://dbpedia.org/resource/Psychedelic_music, http://dbpedia.org/resource/Minimal_music, http://dbpedia.org/resource/Acid_house, http://dbpedia.org/resource/Music_of_Colombia, http://dbpedia.org/resource/Latin_pop, http://dbpedia.org/resource/Vallenato, http://dbpedia.org/resource/Merengue_music, http://dbpedia.org/resource/Salsa_music, http://dbpedia.org/resource/African_Americans, http://dbpedia.org/resource/Cajun_music, http://dbpedia.org/resource/Creole_music, http://dbpedia.org/resource/Highlife, http://dbpedia.org/resource/Yoruba_music, http://dbpedia.org/resource/Contemporary_folk_music, http://dbpedia.org/resource/Jazz-funk, http://dbpedia.org/resource/Free_improvisation, http://dbpedia.org/resource/Avant-garde_jazz, http://dbpedia.org/resource/Free_jazz, http://dbpedia.org/resource/Funk_rock, http://dbpedia.org/resource/Dance-punk, http://dbpedia.org/resource/Alternative_metal, http://dbpedia.org/resource/Soul_jazz, http://dbpedia.org/resource/Acid_jazz, http://dbpedia.org/resource/Electronica, http://dbpedia.org/resource/Trip_hop, http://dbpedia.org/resource/Big_beat, http://dbpedia.org/resource/P-Funk, http://dbpedia.org/resource/Neo_soul, http://dbpedia.org/resource/Hard_bop, http://dbpedia.org/resource/Music_of_Serbia, http://dbpedia.org/resource/Arabic_music, http://dbpedia.org/resource/Music_of_Greece, http://dbpedia.org/resource/Music_of_Turkey, http://dbpedia.org/resource/Music_of_Bulgaria, http://dbpedia.org/resource/Music_of_Romania, http://dbpedia.org/resource/Southern_soul, http://dbpedia.org/resource/Richard_Wagner, http://dbpedia.org/resource/Wall_of_Sound, http://dbpedia.org/resource/Rock_in_Opposition, http://dbpedia.org/resource/New_Orleans_rhythm_and_blues, http://dbpedia.org/resource/Gothic_rock, http://dbpedia.org/resource/Soukous, http://dbpedia.org/resource/Chimurenga, http://dbpedia.org/resource/Music_of_Tanzania, http://dbpedia.org/resource/Soca_music, http://dbpedia.org/resource/Zouk, http://dbpedia.org/resource/Tuk_band, http://dbpedia.org/resource/Hymn, http://dbpedia.org/resource/Industrial_music, http://dbpedia.org/resource/Crust_punk, http://dbpedia.org/resource/Progressive_folk, http://dbpedia.org/resource/Avant-garde_music, http://dbpedia.org/resource/Smooth_jazz, http://dbpedia.org/resource/Christian_rock, http://dbpedia.org/resource/Space_rock, http://dbpedia.org/resource/Symphonic_rock, http://dbpedia.org/resource/Popular_music, http://dbpedia.org/resource/Grunge, http://dbpedia.org/resource/Street_punk, http://dbpedia.org/resource/2_Tone_(music_genre), http://dbpedia.org/resource/Oi!, http://dbpedia.org/resource/Sega_(genre), http://dbpedia.org/resource/Garage_punk, http://dbpedia.org/resource/Benga_music, http://dbpedia.org/resource/Isukuti, http://dbpedia.org/resource/Rap_rock, http://dbpedia.org/resource/British_blues, http://dbpedia.org/resource/British_jazz, http://dbpedia.org/resource/British_rhythm_and_blues, http://dbpedia.org/resource/Delta_blues, http://dbpedia.org/resource/Christian_music, http://dbpedia.org/resource/Roots_rock, http://dbpedia.org/resource/Southern_rock, http://dbpedia.org/resource/Country_rock, http://dbpedia.org/resource/Alternative_hip_hop, http://dbpedia.org/resource/Swing_music, http://dbpedia.org/resource/Chicago_blues, http://dbpedia.org/resource/Jump_blues, http://dbpedia.org/resource/Western_swing, http://dbpedia.org/resource/Electric_folk, http://dbpedia.org/resource/Folk_punk, http://dbpedia.org/resource/Celtic_rock, http://dbpedia.org/resource/Anarcho-punk, http://dbpedia.org/resource/D-beat, http://dbpedia.org/resource/Schlager_music, http://dbpedia.org/resource/Glam_punk, http://dbpedia.org/resource/Death-doom, http://dbpedia.org/resource/Doom_metal, http://dbpedia.org/resource/Instrumental, http://dbpedia.org/resource/No_wave, http://dbpedia.org/resource/Noise_music, http://dbpedia.org/resource/Skinhead, http://dbpedia.org/resource/Football_chant, http://dbpedia.org/resource/Drinking_song, http://dbpedia.org/resource/Progressive_country, http://dbpedia.org/resource/Dark_ambient, http://dbpedia.org/resource/Funk_metal, http://dbpedia.org/resource/Bluegrass_music, http://dbpedia.org/resource/Texas_country_music, http://dbpedia.org/resource/Tulsa_Sound, http://dbpedia.org/resource/American_folk_music, http://dbpedia.org/resource/Honky-tonk, http://dbpedia.org/resource/Outlaw_country, http://dbpedia.org/resource/Drum_and_bass, http://dbpedia.org/resource/Breakbeat_hardcore, http://dbpedia.org/resource/Alternative_dance, http://dbpedia.org/resource/Neo-psychedelia, http://dbpedia.org/resource/Electronic_body_music, http://dbpedia.org/resource/Death_metal, http://dbpedia.org/resource/Metalcore, http://dbpedia.org/resource/Grindcore, http://dbpedia.org/resource/Kundiman, http://dbpedia.org/resource/Taglish, http://dbpedia.org/resource/Philippine_folk_music, http://dbpedia.org/resource/Techno, http://dbpedia.org/resource/Rave_music, http://dbpedia.org/resource/Post-punk_revival, http://dbpedia.org/resource/Electro_house, http://dbpedia.org/resource/Mashup_(music), http://dbpedia.org/resource/Grindie, http://dbpedia.org/resource/Lovers_rock, http://dbpedia.org/resource/Ragga, http://dbpedia.org/resource/Jamaica, http://dbpedia.org/resource/United_States, http://dbpedia.org/resource/Music_of_Morocco, http://dbpedia.org/resource/Film_score, http://dbpedia.org/resource/Exotica, http://dbpedia.org/resource/Baganda_music, http://dbpedia.org/resource/Kadongo_Kamu, http://dbpedia.org/resource/Neotraditional_country, http://dbpedia.org/resource/Dance-rock, http://dbpedia.org/resource/Baroque_music, http://dbpedia.org/resource/Children's_music, http://dbpedia.org/resource/Nursery_rhyme, http://dbpedia.org/resource/Psychedelic_pop, http://dbpedia.org/resource/Novelty_song, http://dbpedia.org/resource/Comedy_music, http://dbpedia.org/resource/Deejay_(Jamaican), http://dbpedia.org/resource/Hardcore_(electronic_dance_music_genre), http://dbpedia.org/resource/Industrial_rock, http://dbpedia.org/resource/20th-century_classical_music, http://dbpedia.org/resource/Zydeco, http://dbpedia.org/resource/Dixieland, http://dbpedia.org/resource/Afrobeat, http://dbpedia.org/resource/Rock_Against_Communism, http://dbpedia.org/resource/Roots_revival, http://dbpedia.org/resource/Urban_contemporary, http://dbpedia.org/resource/Swamp_pop, http://dbpedia.org/resource/Country_blues, http://dbpedia.org/resource/Louisiana_blues, http://dbpedia.org/resource/Math_rock, http://dbpedia.org/resource/Blast_beat, http://dbpedia.org/resource/Méringue, http://dbpedia.org/resource/Melodic_hardcore, http://dbpedia.org/resource/Straight_edge, http://dbpedia.org/resource/Luo_peoples, http://dbpedia.org/resource/Blue-eyed_soul, http://dbpedia.org/resource/Space_Age, http://dbpedia.org/resource/Dance-pop, http://dbpedia.org/resource/Madchester, http://dbpedia.org/resource/Polka, http://dbpedia.org/resource/Waltz, http://dbpedia.org/resource/Huapango, http://dbpedia.org/resource/Ranchera, http://dbpedia.org/resource/Redova, http://dbpedia.org/resource/Bolero, http://dbpedia.org/resource/Schottische, http://dbpedia.org/resource/Corrido, http://dbpedia.org/resource/Harry_Potter, http://dbpedia.org/resource/Nederpop, http://dbpedia.org/resource/Neue_Deutsche_Härte, http://dbpedia.org/resource/Folk_metal, http://dbpedia.org/resource/Neo-Medieval_music, http://dbpedia.org/resource/Medieval_folk_rock, http://dbpedia.org/resource/Chiptune, http://dbpedia.org/resource/Video_game_music, http://dbpedia.org/resource/Melodic_metalcore, http://dbpedia.org/resource/Ryūkōka, http://dbpedia.org/resource/Hip_hop, http://dbpedia.org/resource/Psychedelic_folk, http://dbpedia.org/resource/Turkish_folk_music, http://dbpedia.org/resource/Post-metal, http://dbpedia.org/resource/Mod_revival, http://dbpedia.org/resource/Baggy, http://dbpedia.org/resource/Country_folk, http://dbpedia.org/resource/Viking_metal, http://dbpedia.org/resource/Progressive_metal, http://dbpedia.org/resource/Avant-garde_metal, http://dbpedia.org/resource/Nordic_folk_music, http://dbpedia.org/resource/Sea_shanty, http://dbpedia.org/resource/Shock_rock, http://dbpedia.org/resource/Punjab_(region), http://dbpedia.org/resource/Shred_guitar, http://dbpedia.org/resource/Traditional_heavy_metal, http://dbpedia.org/resource/Nu_metal, http://dbpedia.org/resource/Groove_metal, http://dbpedia.org/resource/Music_of_India, http://dbpedia.org/resource/Raga, http://dbpedia.org/resource/Ska_punk, http://dbpedia.org/resource/Christian_alternative_rock, http://dbpedia.org/resource/Visual_kei, http://dbpedia.org/resource/Geek_rock, http://dbpedia.org/resource/Wizard_rock, http://dbpedia.org/resource/Britpop, http://dbpedia.org/resource/Dark_wave, http://dbpedia.org/resource/Neoclassical_dark_wave, http://dbpedia.org/resource/German_philosophy, http://dbpedia.org/resource/German_rock, http://dbpedia.org/resource/Spanish_rock, http://dbpedia.org/resource/World_music, http://dbpedia.org/resource/Rock_en_español, http://dbpedia.org/resource/Music_of_Latin_America, http://dbpedia.org/resource/Pop_punk, http://dbpedia.org/resource/Reggaeton, http://dbpedia.org/resource/Dirty_rap, http://dbpedia.org/resource/Funk_carioca, http://dbpedia.org/resource/Gangsta_rap, http://dbpedia.org/resource/Tango_music, http://dbpedia.org/resource/Min'yō, http://dbpedia.org/resource/Rōkyoku, http://dbpedia.org/resource/Third-wave_feminism, http://dbpedia.org/resource/Queer_theory, http://dbpedia.org/resource/Music_hall, http://dbpedia.org/resource/Ribaldry, http://dbpedia.org/resource/British_comedy] [INFO] Predicted Answer: [http://dbpedia.org/resource/Latvia, http://dbpedia.org/resource/Valmiera] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 543, P: 0.520, R: 0.570, macro F1: 0.525, macro F1*: 0.544 -[INFO] Current time: 4898 ms, average: 2717.15 ms/ques, entity detection: 420.39 ms/ques (linking tool: 328.48 ms/ques), relation detection: 891.07 ms/ques (similarity: 700.74 ms/ques), EDG generation: 50.82 ms/ques +[INFO] Cumulative metrics, sample: 543, P: 0.520, R: 0.570, macro F1: 0.544 +[INFO] Current question consumed time: 4493 ms, average: 2483.27 ms/ques, entity detection: 379.00 ms/ques (linking tool: 298.81 ms/ques), relation detection: 832.35 ms/ques (similarity: 688.00 ms/ques), EDG generation: 50.96 ms/ques [INFO] Entity Linking P: 0.7476979742173112, R: 0.7007366482504604, micro F1: 0.7157765500306943, macro F1: 0.7234560182827385 [INFO] Relation Linking P: 0.5997544505831798, R: 0.5478821362799263, micro F1: 0.5620626151012892, macro F1: 0.5726459985509367 [INFO] Type Linking P: 0.5377532228360957, R: 0.5377532228360957, micro F1: 0.5377532228360957, macro F1: 0.5377532228360957 @@ -7614,8 +7614,8 @@ Question 543: What are the other notable work of the executive producer of The R [INFO] Golden Answer: [http://dbpedia.org/resource/Doctor_Who, http://dbpedia.org/resource/Torchwood, http://dbpedia.org/resource/Cucumber_(TV_series), http://dbpedia.org/resource/Queer_as_Folk_(UK_TV_series), http://dbpedia.org/resource/The_Sarah_Jane_Adventures, http://dbpedia.org/resource/Casanova_(2005_TV_serial), http://dbpedia.org/resource/Banana_(TV_series), http://dbpedia.org/resource/Bob_&_Rose, http://dbpedia.org/resource/The_Second_Coming_(TV_serial), *@en, ''@en, *''@en] [INFO] Predicted Answer: [http://dbpedia.org/resource/Lars_Pearson, http://dbpedia.org/resource/Neil_Cross, http://dbpedia.org/resource/Paul_Magrs, http://dbpedia.org/resource/Russell_T_Davies, http://dbpedia.org/resource/Johnny_Byrne_(writer)] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 544, P: 0.519, R: 0.569, macro F1: 0.524, macro F1*: 0.543 -[INFO] Current time: 5412 ms, average: 2722.10 ms/ques, entity detection: 419.99 ms/ques (linking tool: 328.09 ms/ques), relation detection: 893.61 ms/ques (similarity: 701.66 ms/ques), EDG generation: 50.83 ms/ques +[INFO] Cumulative metrics, sample: 544, P: 0.519, R: 0.569, macro F1: 0.543 +[INFO] Current question consumed time: 4678 ms, average: 2487.31 ms/ques, entity detection: 378.61 ms/ques (linking tool: 298.43 ms/ques), relation detection: 834.60 ms/ques (similarity: 688.96 ms/ques), EDG generation: 50.96 ms/ques [INFO] Entity Linking P: 0.7463235294117647, R: 0.6994485294117647, micro F1: 0.7144607843137262, macro F1: 0.7221261358961895 [INFO] Relation Linking P: 0.599111519607843, R: 0.5477941176470589, micro F1: 0.5616421568627451, macro F1: 0.5723047399806723 [INFO] Type Linking P: 0.5386029411764706, R: 0.5386029411764706, micro F1: 0.5386029411764706, macro F1: 0.5386029411764706 @@ -7628,8 +7628,8 @@ Question 544: Who was the voice actor of allen walker also gave voice to kimihir [INFO] Golden Answer: [http://dbpedia.org/resource/Todd_Haberkorn] [INFO] Predicted Answer: [http://dbpedia.org/resource/Todd_Haberkorn] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 545, P: 0.520, R: 0.570, macro F1: 0.525, macro F1*: 0.544 -[INFO] Current time: 975 ms, average: 2718.90 ms/ques, entity detection: 419.59 ms/ques (linking tool: 327.67 ms/ques), relation detection: 892.30 ms/ques (similarity: 700.59 ms/ques), EDG generation: 50.85 ms/ques +[INFO] Cumulative metrics, sample: 545, P: 0.520, R: 0.570, macro F1: 0.544 +[INFO] Current question consumed time: 942 ms, average: 2484.47 ms/ques, entity detection: 378.28 ms/ques (linking tool: 298.06 ms/ques), relation detection: 833.38 ms/ques (similarity: 687.92 ms/ques), EDG generation: 50.99 ms/ques [INFO] Entity Linking P: 0.7467889908256881, R: 0.7, micro F1: 0.7149847094801229, macro F1: 0.7226379201014584 [INFO] Relation Linking P: 0.5998470948012232, R: 0.5486238532110091, micro F1: 0.5624464831804282, macro F1: 0.5730931636657672 [INFO] Type Linking P: 0.5394495412844037, R: 0.5394495412844037, micro F1: 0.5394495412844037, macro F1: 0.5394495412844037 @@ -7642,8 +7642,8 @@ Question 545: Name the office holder whose final resting place is North Bend and [INFO] Golden Answer: [http://dbpedia.org/resource/William_Henry_Harrison] [INFO] Predicted Answer: [http://dbpedia.org/resource/William_Henry_Harrison] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 546, P: 0.521, R: 0.571, macro F1: 0.526, macro F1*: 0.544 -[INFO] Current time: 4933 ms, average: 2722.95 ms/ques, entity detection: 419.79 ms/ques (linking tool: 327.79 ms/ques), relation detection: 893.91 ms/ques (similarity: 703.09 ms/ques), EDG generation: 50.90 ms/ques +[INFO] Cumulative metrics, sample: 546, P: 0.521, R: 0.571, macro F1: 0.544 +[INFO] Current question consumed time: 4522 ms, average: 2488.20 ms/ques, entity detection: 378.39 ms/ques (linking tool: 298.13 ms/ques), relation detection: 834.82 ms/ques (similarity: 690.45 ms/ques), EDG generation: 51.03 ms/ques [INFO] Entity Linking P: 0.7472527472527473, R: 0.7005494505494505, micro F1: 0.7155067155067162, macro F1: 0.7231478199220135 [INFO] Relation Linking P: 0.5996642246642246, R: 0.5485347985347986, micro F1: 0.5623321123321123, macro F1: 0.5729611121741921 [INFO] Type Linking P: 0.5384615384615384, R: 0.5384615384615384, micro F1: 0.5384615384615384, macro F1: 0.5384615384615384 @@ -7656,8 +7656,8 @@ Question 546: What type of engines are utilised by Purvis Eureka? [INFO] Golden Answer: [http://dbpedia.org/resource/Ford_Pinto_engine, http://dbpedia.org/resource/Ford_Motor_Company, http://dbpedia.org/resource/Mazda_Wankel_engine, http://dbpedia.org/resource/Volkswagen_air-cooled_engine] [INFO] Predicted Answer: [http://dbpedia.org/resource/Purvis_Eureka__1, http://dbpedia.org/resource/Purvis_Eureka__2, http://dbpedia.org/resource/Purvis_Eureka__3, http://dbpedia.org/resource/Purvis_Eureka__4, http://dbpedia.org/resource/Purvis_Eureka__5, http://dbpedia.org/resource/Ford_Pinto_engine, http://dbpedia.org/resource/Ford_Motor_Company, http://dbpedia.org/resource/Mazda_Wankel_engine, http://dbpedia.org/resource/Volkswagen_air-cooled_engine] [INFO] P: 0.4444444444444444, R: 1.0, F: 0.6153846153846153 -[INFO] Cumulative metrics, sample: 547, P: 0.520, R: 0.571, macro F1: 0.526, macro F1*: 0.545 -[INFO] Current time: 1145 ms, average: 2720.07 ms/ques, entity detection: 419.33 ms/ques (linking tool: 327.34 ms/ques), relation detection: 892.96 ms/ques (similarity: 702.51 ms/ques), EDG generation: 50.84 ms/ques +[INFO] Cumulative metrics, sample: 547, P: 0.520, R: 0.571, macro F1: 0.545 +[INFO] Current question consumed time: 1046 ms, average: 2485.57 ms/ques, entity detection: 377.97 ms/ques (linking tool: 297.73 ms/ques), relation detection: 833.93 ms/ques (similarity: 689.89 ms/ques), EDG generation: 50.98 ms/ques [INFO] Entity Linking P: 0.7477148080438757, R: 0.7010968921389397, micro F1: 0.7160268129189525, macro F1: 0.7236558457661232 [INFO] Relation Linking P: 0.6003960999390615, R: 0.5493601462522852, micro F1: 0.5631322364411945, macro F1: 0.573745418412682 [INFO] Type Linking P: 0.5393053016453382, R: 0.5393053016453382, micro F1: 0.5393053016453382, macro F1: 0.5393053016453382 @@ -7670,8 +7670,8 @@ Question 547: Who edited Easy street ? [INFO] Golden Answer: [http://dbpedia.org/resource/Charlie_Chaplin] [INFO] Predicted Answer: [http://dbpedia.org/resource/Charlie_Chaplin, Charles Chaplin@en] [INFO] P: 0.5, R: 1.0, F: 0.6666666666666666 -[INFO] Cumulative metrics, sample: 548, P: 0.520, R: 0.572, macro F1: 0.526, macro F1*: 0.545 -[INFO] Current time: 774 ms, average: 2716.52 ms/ques, entity detection: 418.84 ms/ques (linking tool: 326.93 ms/ques), relation detection: 891.75 ms/ques (similarity: 701.49 ms/ques), EDG generation: 50.76 ms/ques +[INFO] Cumulative metrics, sample: 548, P: 0.520, R: 0.572, macro F1: 0.545 +[INFO] Current question consumed time: 688 ms, average: 2482.29 ms/ques, entity detection: 377.53 ms/ques (linking tool: 297.34 ms/ques), relation detection: 832.82 ms/ques (similarity: 688.88 ms/ques), EDG generation: 50.89 ms/ques [INFO] Entity Linking P: 0.7481751824817519, R: 0.7016423357664233, micro F1: 0.7165450121654507, macro F1: 0.7241620079653457 [INFO] Relation Linking P: 0.601125304136253, R: 0.5501824817518248, micro F1: 0.5639294403892945, macro F1: 0.5745268393514631 [INFO] Type Linking P: 0.5401459854014599, R: 0.5401459854014599, micro F1: 0.5401459854014599, macro F1: 0.5401459854014599 @@ -7695,8 +7695,8 @@ Co-belligerents:@en, Hesse-Kassel@en, *15px Brunswick-Wolfenbüttel * Hesse-Hanau@en, http://dbpedia.org/resource/Chickasaw, http://dbpedia.org/resource/Choctaw] [INFO] Predicted Answer: [http://dbpedia.org/resource/Anthony_Wayne, http://dbpedia.org/resource/William_Henry_Harrison, http://dbpedia.org/resource/Legion_of_the_United_States, http://dbpedia.org/resource/Henry_De_Butts, http://dbpedia.org/resource/Jonathan_Haskell, http://dbpedia.org/resource/Simeon_Magruder_Levy, http://dbpedia.org/resource/3rd_U.S._Infantry_Regiment_(The_Old_Guard), http://dbpedia.org/resource/Edward_Butler_(soldier), http://dbpedia.org/resource/David_Wooster, http://dbpedia.org/resource/Israel_Putnam, http://dbpedia.org/resource/Archibald_Campbell_(British_Army_officer,_born_1739), http://dbpedia.org/resource/George_Clinton_(vice_president), http://dbpedia.org/resource/Green_Mountain_Boys, http://dbpedia.org/resource/Richard_Prescott, http://dbpedia.org/resource/Francis_Peirson, http://dbpedia.org/resource/George_Don_(British_Army_officer), http://dbpedia.org/resource/Hugh_Palliser, http://dbpedia.org/resource/Infantry_Branch_(United_States), http://dbpedia.org/resource/John_Ford_(Royal_Navy_officer), http://dbpedia.org/resource/Sir_Albemarle_Bertie,_1st_Baronet, http://dbpedia.org/resource/United_States_Army_Rangers, http://dbpedia.org/resource/John_Gunby, http://dbpedia.org/resource/Robert_Monckton, http://dbpedia.org/resource/Royal_Welch_Fusiliers, http://dbpedia.org/resource/Victor-François,_2nd_duc_de_Broglie, http://dbpedia.org/resource/Abraham_Whipple, http://dbpedia.org/resource/Edward_Preble, http://dbpedia.org/resource/John_Gibson_(soldier), http://dbpedia.org/resource/John_Pitt,_2nd_Earl_of_Chatham, http://dbpedia.org/resource/John_Stricker, http://dbpedia.org/resource/Joshua_Barney, http://dbpedia.org/resource/Master-at-arms, http://dbpedia.org/resource/Provincial_Marine, http://dbpedia.org/resource/Sir_Hyde_Parker,_5th_Baronet, http://dbpedia.org/resource/Wade_Hampton_I, http://dbpedia.org/resource/Charles_Scott_(governor), http://dbpedia.org/resource/James_Vashon, http://dbpedia.org/resource/John_Archer_Elmore, http://dbpedia.org/resource/Henry_Monckton, http://dbpedia.org/resource/David_Hawley, http://dbpedia.org/resource/Henry_Burbeck, http://dbpedia.org/resource/William_Johnstone_Hope, http://dbpedia.org/resource/Simon_Larned, http://dbpedia.org/resource/Sir_Peter_Parker,_1st_Baronet, http://dbpedia.org/resource/Adam_Mackenzie, http://dbpedia.org/resource/Christopher_Greene, http://dbpedia.org/resource/Edward_Antill_(soldier), http://dbpedia.org/resource/John_Lacey_(general), http://dbpedia.org/resource/Robert_Howe_(Continental_Army_officer), http://dbpedia.org/resource/William_Maxwell_(Continental_Army_general), http://dbpedia.org/resource/Angus_McDonald_(Virginia_militiaman), http://dbpedia.org/resource/Anselm_Tupper, http://dbpedia.org/resource/Ebenezer_Denny, http://dbpedia.org/resource/Griffin_Greene, http://dbpedia.org/resource/Israel_Shreve, http://dbpedia.org/resource/Samuel_Wear, http://dbpedia.org/resource/Henry_Edward_Fox, http://dbpedia.org/resource/Judge_Advocate_General's_Corps,_United_States_Army, http://dbpedia.org/resource/Aedanus_Burke, http://dbpedia.org/resource/John_Stuart,_Count_of_Maida, http://dbpedia.org/resource/Alexander_Murray_(1755–1821), http://dbpedia.org/resource/Pierre_Jean_Van_Stabel, http://dbpedia.org/resource/Edward_Rotheram, http://dbpedia.org/resource/John_Halsted, http://dbpedia.org/resource/Samuel_Warren_(Royal_Navy_officer), http://dbpedia.org/resource/Sir_Thomas_Pasley,_1st_Baronet, http://dbpedia.org/resource/Zachary_Mudge, http://dbpedia.org/resource/12th_Pennsylvania_Regiment, http://dbpedia.org/resource/John_Butler_(general), http://dbpedia.org/resource/John_Holloway_(Royal_Navy_officer), http://dbpedia.org/resource/William_Madison, http://dbpedia.org/resource/John_Dalling, http://dbpedia.org/resource/Philippe_Liébert, http://dbpedia.org/resource/Sir_Cornelius_Cuyler,_1st_Baronet, http://dbpedia.org/resource/Alexander_Graeme, http://dbpedia.org/resource/Dillon's_Regiment, http://dbpedia.org/resource/Jean-Baptiste_Hamelin, http://dbpedia.org/resource/16th_Massachusetts_Regiment, http://dbpedia.org/resource/Moses_Rawlings, http://dbpedia.org/resource/Edward_Stevens_(general), http://dbpedia.org/resource/20_Battery_Royal_Artillery, http://dbpedia.org/resource/John_Smith_(Virginia_representative), http://dbpedia.org/resource/Moses_Corbet, http://dbpedia.org/resource/Jonathan_Faulknor_the_elder, http://dbpedia.org/resource/James_Young_(Royal_Navy_officer,_born_1717), http://dbpedia.org/resource/Jean-François_Tartu, http://dbpedia.org/resource/Isaac_Hayne, http://dbpedia.org/resource/George_Farmer_(Royal_Navy_officer), http://dbpedia.org/resource/John_Loring_(Royal_Navy_officer,_died_1808), http://dbpedia.org/resource/Alexander_Hamilton, http://dbpedia.org/resource/Andrew_Jackson, http://dbpedia.org/resource/Arthur_St._Clair, http://dbpedia.org/resource/Augustus_Keppel,_1st_Viscount_Keppel, http://dbpedia.org/resource/Banastre_Tarleton, http://dbpedia.org/resource/Benedict_Arnold, http://dbpedia.org/resource/Charles_Cornwallis,_1st_Marquess_Cornwallis, http://dbpedia.org/resource/Charles_Hector,_comte_d'Estaing, http://dbpedia.org/resource/Charles_Lee_(general), http://dbpedia.org/resource/Daniel_Morgan, http://dbpedia.org/resource/Edward_Hughes_(Royal_Navy_officer), http://dbpedia.org/resource/Edward_Pellew,_1st_Viscount_Exmouth, http://dbpedia.org/resource/Esek_Hopkins, http://dbpedia.org/resource/Ethan_Allen, http://dbpedia.org/resource/François_Joseph_Paul_de_Grasse, http://dbpedia.org/resource/French_Army, http://dbpedia.org/resource/Friedrich_Wilhelm_von_Steuben, http://dbpedia.org/resource/George_Rogers_Clark, http://dbpedia.org/resource/Guy_Carleton,_1st_Baron_Dorchester, http://dbpedia.org/resource/Henry_Knox, http://dbpedia.org/resource/Horatio_Gates, http://dbpedia.org/resource/Hugh_Mercer, http://dbpedia.org/resource/James_Monroe, http://dbpedia.org/resource/Jean-Baptiste_Donatien_de_Vimeur,_comte_de_Rochambeau, http://dbpedia.org/resource/Jeffery_Amherst,_1st_Baron_Amherst, http://dbpedia.org/resource/John_Burgoyne, http://dbpedia.org/resource/John_Byron, http://dbpedia.org/resource/John_Paul_Jones, http://dbpedia.org/resource/Louis-René_Levassor_de_Latouche_Tréville, http://dbpedia.org/resource/Nathanael_Greene, http://dbpedia.org/resource/Richard_Howe,_1st_Earl_Howe, http://dbpedia.org/resource/Richard_Montgomery, http://dbpedia.org/resource/Thomas_Gage, http://dbpedia.org/resource/Timothy_Pickering, http://dbpedia.org/resource/United_States_Army, http://dbpedia.org/resource/William_Howe,_5th_Viscount_Howe, http://dbpedia.org/resource/Aaron_Burr, http://dbpedia.org/resource/Alexander_Cochrane, http://dbpedia.org/resource/George_Brydges_Rodney,_1st_Baron_Rodney, http://dbpedia.org/resource/John_MacBride_(Royal_Navy_officer), http://dbpedia.org/resource/John_Sevier, http://dbpedia.org/resource/Joshua_Rowley, http://dbpedia.org/resource/William_Stacy, http://dbpedia.org/resource/Charles_Grey,_1st_Earl_Grey, http://dbpedia.org/resource/Henry_Watson_Powell, http://dbpedia.org/resource/Robert_Abercromby_of_Airthrey, http://dbpedia.org/resource/William_Cathcart,_1st_Earl_Cathcart, http://dbpedia.org/resource/William_Medows, http://dbpedia.org/resource/Sir_William_Parker,_1st_Baronet,_of_Harburn, http://dbpedia.org/resource/Alexander_Hood_(Royal_Navy_officer), http://dbpedia.org/resource/Claude-Augustin_Tercier, http://dbpedia.org/resource/David_Fanning_(loyalist), http://dbpedia.org/resource/Ebenezer_Sproat, http://dbpedia.org/resource/John_Macleod_(British_Army_officer), http://dbpedia.org/resource/John_Willett_Payne, http://dbpedia.org/resource/Pierre_Martin_(French_Navy_officer), http://dbpedia.org/resource/Sir_Charles_Thompson,_1st_Baronet, http://dbpedia.org/resource/Thomas_Carpenter_(glassmaker), http://dbpedia.org/resource/Thomas_Graves,_1st_Baron_Graves, http://dbpedia.org/resource/Wilhelm_von_Knyphausen, http://dbpedia.org/resource/Joseph_Habersham, http://dbpedia.org/resource/4th_Maryland_Regiment, http://dbpedia.org/resource/Johann_de_Kalb, http://dbpedia.org/resource/James_Inglis_Hamilton, http://dbpedia.org/resource/12th_Continental_Regiment, http://dbpedia.org/resource/64th_(2nd_Staffordshire)_Regiment_of_Foot, http://dbpedia.org/resource/Alexander_Leslie_(British_Army_officer), http://dbpedia.org/resource/Asa_Pollard, http://dbpedia.org/resource/Benjamin_Brown_(politician), http://dbpedia.org/resource/David_Cobb_(Massachusetts), http://dbpedia.org/resource/David_Shepard_(surgeon), http://dbpedia.org/resource/Francis_Smith_(British_Army_officer), http://dbpedia.org/resource/Joshua_Cushman, http://dbpedia.org/resource/Michael_Rudolph, http://dbpedia.org/resource/Otho_Holland_Williams, http://dbpedia.org/resource/Richard_Taylor_(colonel), http://dbpedia.org/resource/William_Shepard, http://dbpedia.org/resource/John_Simpson_(soldier), http://dbpedia.org/resource/Nathan_Hale_(colonel), http://dbpedia.org/resource/Frederick_Hambright, http://dbpedia.org/resource/John_Butler_(pioneer), http://dbpedia.org/resource/175th_Infantry_Regiment_(United_States), http://dbpedia.org/resource/46th_(South_Devonshire)_Regiment_of_Foot, http://dbpedia.org/resource/Georg_Wilhelm_von_dem_Bussche, http://dbpedia.org/resource/Harry_Calvert, http://dbpedia.org/resource/Henry_Harvey, http://dbpedia.org/resource/Jacques_Fromentin, http://dbpedia.org/resource/John_Bazely, http://dbpedia.org/resource/John_Elliot_(Royal_Navy_officer), http://dbpedia.org/resource/John_Lockhart-Ross, http://dbpedia.org/resource/Lord_Hugh_Seymour, http://dbpedia.org/resource/Philemon_Pownoll, http://dbpedia.org/resource/Robert_Digby_(Royal_Navy_officer), http://dbpedia.org/resource/Sir_Chaloner_Ogle,_1st_Baronet, http://dbpedia.org/resource/Thomas_Dundas_(British_Army_officer), http://dbpedia.org/resource/Thomas_Hurd, http://dbpedia.org/resource/Sir_Alexander_Campbell,_1st_Baronet, http://dbpedia.org/resource/Royal_Lincolnshire_Regiment, http://dbpedia.org/resource/Thomas_Pye, http://dbpedia.org/resource/10th_Virginia_Regiment, http://dbpedia.org/resource/2nd_Virginia_Regiment, http://dbpedia.org/resource/Courtenay_Boyle, http://dbpedia.org/resource/Morgan_Connor, http://dbpedia.org/resource/Alexander_Mackenzie_Fraser, http://dbpedia.org/resource/Clark_Gayton_(Royal_Navy_officer), http://dbpedia.org/resource/George_Darby, http://dbpedia.org/resource/John_Colpoys, http://dbpedia.org/resource/Robert_Roddam, http://dbpedia.org/resource/Samuel_Goodall, http://dbpedia.org/resource/Sir_George_Osborn,_4th_Baronet, http://dbpedia.org/resource/William_Hotham,_1st_Baron_Hotham, http://dbpedia.org/resource/Virginia_Defense_Force, http://dbpedia.org/resource/2nd_Connecticut_Regiment, http://dbpedia.org/resource/James_Robertson_(British_Army_officer), http://dbpedia.org/resource/Joseph_Platt_Cooke, http://dbpedia.org/resource/Eyre_Coote_(British_Army_officer), http://dbpedia.org/resource/Thomas_Murray_(died_1816), http://dbpedia.org/resource/5th_Maryland_Regiment, http://dbpedia.org/resource/Jean_René_Moreaux, http://dbpedia.org/resource/James_Anderson_(Royal_Navy_officer), http://dbpedia.org/resource/Charles_Brisbane, http://dbpedia.org/resource/James_Malcolm_(Royal_Marines_officer), http://dbpedia.org/resource/Robert_Montague_(Royal_Navy_officer), http://dbpedia.org/resource/George_Jackson_(Virginia_politician), http://dbpedia.org/resource/Edmund_Nagle, http://dbpedia.org/resource/Charles_Hughes_(Royal_Navy_officer), http://dbpedia.org/resource/Walter_Griffith, http://dbpedia.org/resource/Thomas_Frederick_Mackenzie_Humberston, http://dbpedia.org/resource/William_Young_(Royal_Navy_officer,_born_1761), http://dbpedia.org/resource/Johann_von_Hinrichs, http://dbpedia.org/resource/Adam_Philippe,_Comte_de_Custine, http://dbpedia.org/resource/Arthur_Phillip, http://dbpedia.org/resource/August_Neidhardt_von_Gneisenau, http://dbpedia.org/resource/Coldstream_Guards, http://dbpedia.org/resource/Gilbert_du_Motier,_Marquis_de_Lafayette, http://dbpedia.org/resource/Jean-Baptiste_Jourdan, http://dbpedia.org/resource/Jean-François_de_Galaup,_comte_de_Lapérouse, http://dbpedia.org/resource/John_Marshall, http://dbpedia.org/resource/John_Stark, http://dbpedia.org/resource/Philip_Charles_Durham, http://dbpedia.org/resource/Pierre-Charles_Villeneuve, http://dbpedia.org/resource/Sir_John_Duckworth,_1st_Baronet, http://dbpedia.org/resource/William_Cornwallis, http://dbpedia.org/resource/278th_Armored_Cavalry_Regiment, http://dbpedia.org/resource/Andrew_Pickens_(congressman), http://dbpedia.org/resource/Jonathan_Eddy, http://dbpedia.org/resource/Tadeusz_Kościuszko, http://dbpedia.org/resource/Christopher_Gore, http://dbpedia.org/resource/John_Cooke_(Royal_Navy_officer), http://dbpedia.org/resource/Bedfordshire_and_Hertfordshire_Regiment, http://dbpedia.org/resource/Henry_Dearborn, http://dbpedia.org/resource/101st_Field_Artillery_Regiment, http://dbpedia.org/resource/Abraham_Drake, http://dbpedia.org/resource/Federico_Carlos_Gravina_y_Nápoli, http://dbpedia.org/resource/Honoré_Joseph_Antoine_Ganteaume, http://dbpedia.org/resource/Lewis_Nicola, http://dbpedia.org/resource/Pulteney_Malcolm, http://dbpedia.org/resource/Sir_Richard_Strachan,_6th_Baronet, http://dbpedia.org/resource/Peleg_Wadsworth, http://dbpedia.org/resource/George_Henry_Towry, http://dbpedia.org/resource/George_Martin_(Royal_Navy_officer), http://dbpedia.org/resource/George_Stewart,_8th_Earl_of_Galloway, http://dbpedia.org/resource/James_Macnamara, http://dbpedia.org/resource/Philemon_Dickinson, http://dbpedia.org/resource/Sir_George_Grey,_1st_Baronet, http://dbpedia.org/resource/Thomas_Frederick_(Royal_Navy_officer), http://dbpedia.org/resource/Thomas_Polk, http://dbpedia.org/resource/Charles_O'Hara, http://dbpedia.org/resource/William_Caldwell_(ranger), http://dbpedia.org/resource/Thomas_Twisleton,_13th_Baron_Saye_and_Sele, http://dbpedia.org/resource/Charles_John_Moore_Mansfield, http://dbpedia.org/resource/Richard_Grindall, http://dbpedia.org/resource/Henry_William_Bayntun, http://dbpedia.org/resource/Thomas_Williams_(Royal_Navy_officer), http://dbpedia.org/resource/Philippe_d'Auvergne, http://dbpedia.org/resource/Benjamin_Guerard, http://dbpedia.org/resource/Edward_Mathew, http://dbpedia.org/resource/John_Markham_(Royal_Navy_officer), http://dbpedia.org/resource/Andrew_Moore_(politician), http://dbpedia.org/resource/Hugh_Downman, http://dbpedia.org/resource/Elihu_Adams, http://dbpedia.org/resource/John_Parker_(captain), http://dbpedia.org/resource/Johann_Ewald, http://dbpedia.org/resource/Robert_Murray_(Royal_Navy_officer), http://dbpedia.org/resource/John_Gray_(American_Revolutionary_War), http://dbpedia.org/resource/Peter_Hare, http://dbpedia.org/resource/Antoine_Digonet, http://dbpedia.org/resource/John_Colville,_9th_Lord_Colville_of_Culross, http://dbpedia.org/resource/Samuel_McDowell, http://dbpedia.org/resource/Robert_Carthew_Reynolds, http://dbpedia.org/resource/William_Lechmere, http://dbpedia.org/resource/Charles_Aimé_de_Royrand, http://dbpedia.org/resource/John_Blankett, http://dbpedia.org/resource/John_Hamilton_(American_Revolution), http://dbpedia.org/resource/Louis-Jean-Nicolas_Lejoille, http://dbpedia.org/resource/Manley_Dixon, http://dbpedia.org/resource/John_Pratt_(soldier), http://dbpedia.org/resource/13th_Continental_Regiment, http://dbpedia.org/resource/Aiskew_Hollis, http://dbpedia.org/resource/Jean_Jacques_Étienne_Lucas, http://dbpedia.org/resource/John_Hill_(Royal_Navy_officer), http://dbpedia.org/resource/Joseph_Spear, http://dbpedia.org/resource/William_Charles_Fahie, http://dbpedia.org/resource/John_Laforey, http://dbpedia.org/resource/Alexander_Shaw_(British_Army_officer), http://dbpedia.org/resource/Herbert_Sawyer_(Royal_Navy_officer,_died_1798), http://dbpedia.org/resource/John_Mills_(soldier), http://dbpedia.org/resource/George_Murray_(Royal_Navy_officer,_born_1759), http://dbpedia.org/resource/Frederick_Haldimand, http://dbpedia.org/resource/James_Gambier,_1st_Baron_Gambier, http://dbpedia.org/resource/John_Barry_(naval_officer), http://dbpedia.org/resource/John_Hunter_(Royal_Navy_officer), http://dbpedia.org/resource/Josiah_Harmar, http://dbpedia.org/resource/Molly_Pitcher, http://dbpedia.org/resource/Richard_Goodwin_Keats, http://dbpedia.org/resource/Samuel_Hood,_1st_Viscount_Hood, http://dbpedia.org/resource/William_Carnegie,_7th_Earl_of_Northesk, http://dbpedia.org/resource/Christopher_Cole_(Royal_Navy_officer), http://dbpedia.org/resource/Francisco_de_Miranda, http://dbpedia.org/resource/James_Henry_Craig, http://dbpedia.org/resource/Caesar_Rodney, http://dbpedia.org/resource/Francis_Rawdon-Hastings,_1st_Marquess_of_Hastings, http://dbpedia.org/resource/Gerard_Lake,_1st_Viscount_Lake, http://dbpedia.org/resource/Henry_Lee_III, http://dbpedia.org/resource/Hessian_(soldier), http://dbpedia.org/resource/Hugh_Percy,_2nd_Duke_of_Northumberland, http://dbpedia.org/resource/John_Breckinridge_(U.S._Attorney_General), http://dbpedia.org/resource/Sir_John_Doyle,_1st_Baronet, http://dbpedia.org/resource/William_Harcourt,_3rd_Earl_Harcourt, http://dbpedia.org/resource/Thomas_Posey, http://dbpedia.org/resource/James_Garrard, http://dbpedia.org/resource/Thomas_Mathews_(politician), http://dbpedia.org/resource/Christopher_Greenup, http://dbpedia.org/resource/William_Smith_(South_Carolina_representative), http://dbpedia.org/resource/William_Stark_(loyalist), http://dbpedia.org/resource/Ballard_Smith, http://dbpedia.org/resource/Isaac_Bloom, http://dbpedia.org/resource/Isaac_Coles, http://dbpedia.org/resource/James_Breckinridge, http://dbpedia.org/resource/John_B._Earle, http://dbpedia.org/resource/John_Blake,_Jr._(politician), http://dbpedia.org/resource/John_Johns_Trigg, http://dbpedia.org/resource/Joseph_Brevard, http://dbpedia.org/resource/Joseph_Dickson, http://dbpedia.org/resource/Joseph_Neville, http://dbpedia.org/resource/Josiah_Parker, http://dbpedia.org/resource/Lemuel_Benton, http://dbpedia.org/resource/Levi_Casey_(politician), http://dbpedia.org/resource/Matthew_Clay, http://dbpedia.org/resource/Nathaniel_Appleton_Haven, http://dbpedia.org/resource/Peter_Carleton, http://dbpedia.org/resource/Samuel_Goode, http://dbpedia.org/resource/Thomas_Forrest, http://dbpedia.org/resource/William_Butler_(1759–1821), http://dbpedia.org/resource/George_Madison, http://dbpedia.org/resource/31st_(Huntingdonshire)_Regiment_of_Foot, http://dbpedia.org/resource/Nicholas_Biddle_(naval_officer), http://dbpedia.org/resource/James_Abercrombie_(British_Army_officer,_born_1732), http://dbpedia.org/resource/James_Agnew_(British_Army_officer), http://dbpedia.org/resource/John_Ross_(1744–1809), http://dbpedia.org/resource/11th_Massachusetts_Regiment, http://dbpedia.org/resource/12th_Massachusetts_Regiment, http://dbpedia.org/resource/James_Hillyar, http://dbpedia.org/resource/Sir_George_Nugent,_1st_Baronet, http://dbpedia.org/resource/Richard_Dacres_(Royal_Navy_officer), http://dbpedia.org/resource/5th_Hussar_Regiment_(France), http://dbpedia.org/resource/Edward_O'Bryen, http://dbpedia.org/resource/Clement_Biddle, http://dbpedia.org/resource/John_Ferrier, http://dbpedia.org/resource/Hyde_Parker_(Royal_Navy_officer,_born_1739), http://dbpedia.org/resource/Jean_Gaspard_de_Vence, http://dbpedia.org/resource/Isaac_Schomberg, http://dbpedia.org/resource/Robert_Lawson_(British_Army_officer), http://dbpedia.org/resource/10th_Continental_Regiment, http://dbpedia.org/resource/4th_New_Jersey_Regiment, http://dbpedia.org/resource/Anne_Bailey, http://dbpedia.org/resource/Nathaniel_Saltonstall_(American_Revolution), http://dbpedia.org/resource/2nd_Continental_Artillery_Regiment, http://dbpedia.org/resource/201st_Field_Artillery_Regiment, http://dbpedia.org/resource/Clapp_Raymond, http://dbpedia.org/resource/Eliphalet_Lockwood, http://dbpedia.org/resource/Job_Bartram, http://dbpedia.org/resource/Matthew_Mead_(politician), http://dbpedia.org/resource/Samuel_Cook_Silliman, http://dbpedia.org/resource/Samuel_Rimington, http://dbpedia.org/resource/Pierre_Douville, http://dbpedia.org/resource/111th_Infantry_Regiment_(United_States), http://dbpedia.org/resource/Andrew_Deveaux, http://dbpedia.org/resource/John_Montresor, http://dbpedia.org/resource/Donatien-Marie-Joseph_de_Vimeur,_vicomte_de_Rochambeau, http://dbpedia.org/resource/Robert_Rogers_(soldier), http://dbpedia.org/resource/Benjamin_Tupper, http://dbpedia.org/resource/Charles_Armand_Tuffin,_marquis_de_la_Rouerie, http://dbpedia.org/resource/Charles_Middleton,_1st_Baron_Barham, http://dbpedia.org/resource/George_Hancock_(Virginia), http://dbpedia.org/resource/John_Armstrong,_Jr., http://dbpedia.org/resource/John_Page_(Virginia_politician), http://dbpedia.org/resource/John_Tyler,_Sr., http://dbpedia.org/resource/Joseph_Warren, http://dbpedia.org/resource/Louis_Alexandre_Andrault_de_Langeron, http://dbpedia.org/resource/Paul_Hamilton_(politician), http://dbpedia.org/resource/Robert_Calder, http://dbpedia.org/resource/Robert_Smith_(Cabinet_member), http://dbpedia.org/resource/Samuel_Graves, http://dbpedia.org/resource/Samuel_Smith_(Maryland), http://dbpedia.org/resource/Sidney_Smith_(Royal_Navy_officer), http://dbpedia.org/resource/Sir_Henry_Johnson,_1st_Baronet, http://dbpedia.org/resource/Thomas_Macnamara_Russell, http://dbpedia.org/resource/William_Eustis, http://dbpedia.org/resource/William_Jones_(statesman), http://dbpedia.org/resource/William_Leslie_(British_Army_officer), http://dbpedia.org/resource/Artemas_Ward, http://dbpedia.org/resource/Thomas_Sumter, http://dbpedia.org/resource/James_Luttrell, http://dbpedia.org/resource/James_Mitchell_Varnum, http://dbpedia.org/resource/Alexander_Hood,_1st_Viscount_Bridport, http://dbpedia.org/resource/Anthony_James_Pye_Molloy, http://dbpedia.org/resource/Henry_Clinton_(British_Army_officer,_born_1730), http://dbpedia.org/resource/Henry_Mowat, http://dbpedia.org/resource/Hugh_Pigot_(Royal_Navy_officer,_born_1722), http://dbpedia.org/resource/Jean-Baptiste-François_Bompart, http://dbpedia.org/resource/Joshua_Clayton, http://dbpedia.org/resource/Lemuel_Cook, http://dbpedia.org/resource/Luis_de_Córdova_y_Córdova, http://dbpedia.org/resource/Sir_Charles_Saxton,_1st_Baronet, http://dbpedia.org/resource/Smith_Child_(Royal_Navy_officer), http://dbpedia.org/resource/Thomas_Graves_(Royal_Navy_officer), http://dbpedia.org/resource/William_Bayne_(Royal_Navy_officer), http://dbpedia.org/resource/William_Phillips_(British_Army_officer), http://dbpedia.org/resource/John_Adair, http://dbpedia.org/resource/Joseph_Bradley_Varnum, http://dbpedia.org/resource/Daniel_F._Bakeman, http://dbpedia.org/resource/Maryland_Loyalists_Battalion, http://dbpedia.org/resource/Daniel_Woodriff, http://dbpedia.org/resource/Thomas_Allin, http://dbpedia.org/resource/Tench_Tilghman, http://dbpedia.org/resource/Jeremiah_Olney, http://dbpedia.org/resource/Alexander_Martin, http://dbpedia.org/resource/John_Lawford, http://dbpedia.org/resource/John_Paul_(pioneer), http://dbpedia.org/resource/Hildebrand_Oakes, http://dbpedia.org/resource/Alexander_Gillon, http://dbpedia.org/resource/Abram_Trigg, http://dbpedia.org/resource/Charles_Harrison_(general), http://dbpedia.org/resource/Charles_Mawhood, http://dbpedia.org/resource/Daniel_Hitchcock, http://dbpedia.org/resource/David_Forman_(general), http://dbpedia.org/resource/Simeon_Thayer, http://dbpedia.org/resource/Sir_William_Erskine,_1st_Baronet, http://dbpedia.org/resource/Thomas_Proctor_(general), http://dbpedia.org/resource/William_Fawcett_(British_Army_officer), http://dbpedia.org/resource/Phillips_Cosby, http://dbpedia.org/resource/Sir_Robert_Kingsmill,_1st_Baronet, http://dbpedia.org/resource/Robert_Duff_(Royal_Navy_officer), http://dbpedia.org/resource/William_Sutherland_(British_Army_officer), http://dbpedia.org/resource/30_Battery_(Rogers's_Company)_Royal_Artillery, http://dbpedia.org/resource/Joseph_Bingham_(Royal_Navy_officer), http://dbpedia.org/resource/Samuel_James_Ballard, http://dbpedia.org/resource/Richard_Byron_(Royal_Navy_officer), http://dbpedia.org/resource/Military_career_of_Benedict_Arnold,_1781, http://dbpedia.org/resource/Eliab_Harvey, http://dbpedia.org/resource/Forbes_Champagné, http://dbpedia.org/resource/Henry_Curzon, http://dbpedia.org/resource/Martin_Waghorn, http://dbpedia.org/resource/Joseph_Brome_(British_Army_officer,_died_1796), http://dbpedia.org/resource/Joseph_Walton_(British_Army_officer), http://dbpedia.org/resource/Richard_Humpton, http://dbpedia.org/resource/Alexander_Oliver, http://dbpedia.org/resource/Solomon_Ferris, http://dbpedia.org/resource/James_Stuart_(British_Army_officer,_born_1741), http://dbpedia.org/resource/Louis_Lhéritier, http://dbpedia.org/resource/Marquis_Calmes, http://dbpedia.org/resource/Shuldham_Peard, http://dbpedia.org/resource/René_François_Lecomte, http://dbpedia.org/resource/Military_career_of_Benedict_Arnold,_1777–79, http://dbpedia.org/resource/George_Hewett, http://dbpedia.org/resource/52nd_(Oxfordshire)_Regiment_of_Foot, http://dbpedia.org/resource/Guillaume-Mathieu_Dumas, http://dbpedia.org/resource/Horatio_Nelson,_1st_Viscount_Nelson, http://dbpedia.org/resource/John_Jervis,_1st_Earl_of_St_Vincent, http://dbpedia.org/resource/John_Moore_(British_Army_officer), http://dbpedia.org/resource/Louis-Alexandre_Berthier, http://dbpedia.org/resource/Sir_Thomas_Troubridge,_1st_Baronet, http://dbpedia.org/resource/Thomas_Louis, http://dbpedia.org/resource/Benjamin_Lincoln, http://dbpedia.org/resource/Casimir_Pulaski, http://dbpedia.org/resource/Francis_William_Drake, http://dbpedia.org/resource/French_Royal_Army_(1652–1830), http://dbpedia.org/resource/Graham_Moore_(admiral), http://dbpedia.org/resource/John_Pitcairn, http://dbpedia.org/resource/John_Small_(British_Army_officer), http://dbpedia.org/resource/Patrick_Ferguson, http://dbpedia.org/resource/Sir_Isaac_Coffin,_1st_Baronet, http://dbpedia.org/resource/42nd_Regiment_of_Foot, http://dbpedia.org/resource/Edward_Rutledge, http://dbpedia.org/resource/James_King_(Royal_Navy_officer), http://dbpedia.org/resource/John_Laurens, http://dbpedia.org/resource/Isaac_Shelby, http://dbpedia.org/resource/Francis_Nash, http://dbpedia.org/resource/Joseph_Plumb_Martin, http://dbpedia.org/resource/Pierre_César_Charles_de_Sercey, http://dbpedia.org/resource/Andrew_Wodrow, http://dbpedia.org/resource/George_Mathews_(Georgia), http://dbpedia.org/resource/Isaac_Parsons_(Virginia_politician), http://dbpedia.org/resource/Moses_Hazen, http://dbpedia.org/resource/Humphrey_Marshall_(politician), http://dbpedia.org/resource/Jean-Anne_Christy_de_la_Pallière, http://dbpedia.org/resource/Charles_Tyler, http://dbpedia.org/resource/Henry_Inman_(Royal_Navy_officer), http://dbpedia.org/resource/James_Walker_(Royal_Navy_officer), http://dbpedia.org/resource/John_Conn, http://dbpedia.org/resource/Samuel_Sutton, http://dbpedia.org/resource/William_Domett, http://dbpedia.org/resource/Alured_Clarke, http://dbpedia.org/resource/George_Napier, http://dbpedia.org/resource/John_Campbell,_of_Strachur, http://dbpedia.org/resource/Alexander_Stewart_(British_Army_officer), http://dbpedia.org/resource/Sir_Thomas_Hislop,_1st_Baronet, http://dbpedia.org/resource/James_Markham_Marshall, http://dbpedia.org/resource/James_Coutts_Crawford, http://dbpedia.org/resource/35th_(Royal_Sussex)_Regiment_of_Foot, http://dbpedia.org/resource/James_Dunlop_of_Dunlop, http://dbpedia.org/resource/James_Rogers_(soldier), http://dbpedia.org/resource/Robert_White_(judge), http://dbpedia.org/resource/Jean-Marthe-Adrien_l'Hermite, http://dbpedia.org/resource/Hugh_Cloberry_Christian, http://dbpedia.org/resource/Robert_Mann_(Royal_Navy_officer), http://dbpedia.org/resource/Robert_Moorsom, http://dbpedia.org/resource/Bartholomew_Rowley, http://dbpedia.org/resource/Thomas_Moore_(South_Carolina_congressman), http://dbpedia.org/resource/Ranald_MacKinnon, http://dbpedia.org/resource/Adams'_Rangers, http://dbpedia.org/resource/Richard_Lee_(Royal_Navy_officer), http://dbpedia.org/resource/John_Leveson-Gower_(Royal_Navy_officer), http://dbpedia.org/resource/George_Eyre, http://dbpedia.org/resource/Richard_England_(died_1812), http://dbpedia.org/resource/William_Wynyard_(British_Army_officer), http://dbpedia.org/resource/William_Taylor_(Royal_Navy_officer), http://dbpedia.org/resource/Robert_Mends, http://dbpedia.org/resource/Samuel_Hildreth_(American_Revolution), http://dbpedia.org/resource/10th_Pennsylvania_Regiment, http://dbpedia.org/resource/Philippe_Hubert_Preudhomme_de_Borre, http://dbpedia.org/resource/Jean-Baptiste_Meynier, http://dbpedia.org/resource/Sharp_Delany, http://dbpedia.org/resource/Richard_Wickes, http://dbpedia.org/resource/Étienne_Charlet, http://dbpedia.org/resource/Military_career_of_Benedict_Arnold,_1775–76, http://dbpedia.org/resource/Charles_Inglis_(Royal_Navy_officer,_died_1791), http://dbpedia.org/resource/Joseph_Bullen, http://dbpedia.org/resource/Anthony_Farrington, http://dbpedia.org/resource/Alexander_Ball, http://dbpedia.org/resource/John_André, http://dbpedia.org/resource/John_Brooks_(governor), http://dbpedia.org/resource/Edmund_Fanning_(colonial_administrator), http://dbpedia.org/resource/Richard_Dale, http://dbpedia.org/resource/Sampson_Mathews, http://dbpedia.org/resource/Sir_James_Pulteney,_7th_Baronet, http://dbpedia.org/resource/James_Colnett, http://dbpedia.org/resource/104th_Infantry_Regiment_(United_States), http://dbpedia.org/resource/Jethro_Sumner, http://dbpedia.org/resource/Richard_Winn, http://dbpedia.org/resource/Robert_Barnwell, http://dbpedia.org/resource/Thomas_Hartley, http://dbpedia.org/resource/William_Irvine_(general), http://dbpedia.org/resource/2nd_Rhode_Island_Regiment, http://dbpedia.org/resource/James_Lingan, http://dbpedia.org/resource/Phillip_Hamman, http://dbpedia.org/resource/Sir_Henry_Edwyn_Stanhope,_1st_Baronet, http://dbpedia.org/resource/Charles-Henri-Louis_d'Arsac_de_Ternay, http://dbpedia.org/resource/Curt_von_Stedingk, http://dbpedia.org/resource/Gunning_Bedford,_Sr., http://dbpedia.org/resource/James_Moore_(Continental_Army_officer), http://dbpedia.org/resource/David_Manson_(militiaman), http://dbpedia.org/resource/James_Hawkins-Whitshed, http://dbpedia.org/resource/Sir_Charles_Pole,_1st_Baronet, http://dbpedia.org/resource/William_Peere_Williams-Freeman, http://dbpedia.org/resource/James_Trevenen, http://dbpedia.org/resource/Lambert_Wickes, http://dbpedia.org/resource/James_Grant_(British_Army_officer,_born_1720), http://dbpedia.org/resource/William_Hargood, http://dbpedia.org/resource/Veteran_Corps_of_Artillery_of_the_State_of_New_York, http://dbpedia.org/resource/George_Beckwith_(British_Army_officer), http://dbpedia.org/resource/Walter_Stewart_(general), http://dbpedia.org/resource/Lawrence_Halsted, http://dbpedia.org/resource/Hugh_Gordon_(British_Army_officer), http://dbpedia.org/resource/James_Hogun, http://dbpedia.org/resource/John_Reily, http://dbpedia.org/resource/Charles_René_Magon_de_Médine, http://dbpedia.org/resource/Pierre-Nicolas_Rolland, http://dbpedia.org/resource/Sir_Edward_Buller,_1st_Baronet, http://dbpedia.org/resource/Robert_Plampin, http://dbpedia.org/resource/Charles_Phipps_(Royal_Navy_officer), http://dbpedia.org/resource/150th_Cavalry_Regiment, http://dbpedia.org/resource/192nd_Military_Police_Battalion, http://dbpedia.org/resource/Robert_Faulknor_the_younger, http://dbpedia.org/resource/Eleazer_Oswald, http://dbpedia.org/resource/Jose_de_Mazarredo_y_Salazar, http://dbpedia.org/resource/Mordecai_Buckner, http://dbpedia.org/resource/Littleberry_Mosby, http://dbpedia.org/resource/Manasseh_Cutler_(representative), http://dbpedia.org/resource/Thomas_Eaton_(general), http://dbpedia.org/resource/Sir_Charles_Green,_1st_Baronet, http://dbpedia.org/resource/André_Poncet, http://dbpedia.org/resource/Walter_Bathurst, http://dbpedia.org/resource/Cuthbert_Collingwood,_1st_Baron_Collingwood, http://dbpedia.org/resource/Jacques_Bernard_d'Anselme, http://dbpedia.org/resource/Jean_Thurel, http://dbpedia.org/resource/Simon_Fraser_of_Balnain, http://dbpedia.org/resource/Touraine_Regiment, http://dbpedia.org/resource/William_Moultrie, http://dbpedia.org/resource/116th_Infantry_Brigade_Combat_Team_(United_States), http://dbpedia.org/resource/Chaplain_Corps_(United_States_Army), http://dbpedia.org/resource/Pierce_Butler, http://dbpedia.org/resource/Theodore_Sedgwick, http://dbpedia.org/resource/Thomas_Pinckney, http://dbpedia.org/resource/Thomas_Tudor_Tucker, http://dbpedia.org/resource/Francis_Barber_(Colonel), http://dbpedia.org/resource/James_Jackson_(Georgia_politician), http://dbpedia.org/resource/Thomas_Humphrey_Cushing, http://dbpedia.org/resource/William_Alexander,_Lord_Stirling, http://dbpedia.org/resource/Thomas_Davies_(British_Army_officer), http://dbpedia.org/resource/115th_Infantry_Regiment_(United_States), http://dbpedia.org/resource/11th_Virginia_Regiment, http://dbpedia.org/resource/43rd_(Monmouthshire)_Regiment_of_Foot, http://dbpedia.org/resource/Griffith_Rutherford, http://dbpedia.org/resource/Thomas_Person, http://dbpedia.org/resource/William_Darke, http://dbpedia.org/resource/Joseph_Bartholomew_(Major_General), http://dbpedia.org/resource/George_Johnstone_(Royal_Navy_officer), http://dbpedia.org/resource/Johan_Zoutman, http://dbpedia.org/resource/Pierre-Louis_Binet_de_Marcognet, http://dbpedia.org/resource/Pierre_Belon_Lapisse, http://dbpedia.org/resource/Joseph_Ingraham, http://dbpedia.org/resource/Charles_Edward_Jennings_de_Kilmaine, http://dbpedia.org/resource/Clément_Gosselin, http://dbpedia.org/resource/Louis-Philippe_de_Vaudreuil, http://dbpedia.org/resource/Carl_Olof_Cronstedt, http://dbpedia.org/resource/Ross_Donnelly, http://dbpedia.org/resource/Benjamin_Caldwell, http://dbpedia.org/resource/Charles_Dashwood_(Royal_Navy_officer), http://dbpedia.org/resource/Edward_Foote, http://dbpedia.org/resource/Edward_Thornbrough, http://dbpedia.org/resource/George_Montagu_(Royal_Navy_officer), http://dbpedia.org/resource/John_Hutt_(Royal_Navy_officer), http://dbpedia.org/resource/John_Monkton, http://dbpedia.org/resource/Louis_Thomas_Villaret_de_Joyeuse, http://dbpedia.org/resource/Peter_Rainier,_junior, http://dbpedia.org/resource/Robert_Barlow_(Royal_Navy_officer), http://dbpedia.org/resource/Roger_Curtis, http://dbpedia.org/resource/Sir_Charles_Cotton,_5th_Baronet, http://dbpedia.org/resource/Thomas_Mackenzie_(Royal_Navy_officer), http://dbpedia.org/resource/Thomas_Pakenham_(Royal_Navy_officer), http://dbpedia.org/resource/William_Bentinck_(Royal_Navy_officer), http://dbpedia.org/resource/Charles_Edmund_Nugent, http://dbpedia.org/resource/David_Henley, http://dbpedia.org/resource/Jean-Jacques_Ambert, http://dbpedia.org/resource/Sir_Richard_Hughes,_2nd_Baronet, http://dbpedia.org/resource/Charles_Buckner, http://dbpedia.org/resource/Jean_Isaac_Chadeau_de_la_Clocheterie, http://dbpedia.org/resource/John_Knight_(Royal_Navy_officer), http://dbpedia.org/resource/Robert_Linzee, http://dbpedia.org/resource/Sir_Francis_Samuel_Drake,_1st_Baronet, http://dbpedia.org/resource/Sir_John_Orde,_1st_Baronet, http://dbpedia.org/resource/Louis_André_Bon, http://dbpedia.org/resource/George_Cranfield_Berkeley, http://dbpedia.org/resource/George_Muter, http://dbpedia.org/resource/Jean-Antoine_Panet, http://dbpedia.org/resource/Arthur_Kaye_Legge, http://dbpedia.org/resource/John_Bligh_(Royal_Navy_officer), http://dbpedia.org/resource/William_Mitchell_(Royal_Navy_officer), http://dbpedia.org/resource/3rd_Continental_Artillery_Regiment, http://dbpedia.org/resource/William_Inglis_(British_Army_officer), http://dbpedia.org/resource/James_Alms, http://dbpedia.org/resource/James_Richards_(politician), http://dbpedia.org/resource/Daniel_Jones_(British_Army_officer), http://dbpedia.org/resource/James_Coates_(British_Army_officer), http://dbpedia.org/resource/Harry_Burrard-Neale, http://dbpedia.org/resource/Hezekiah_Rogers, http://dbpedia.org/resource/2nd_Independent_Maryland_Company_–_Somerset_County, http://dbpedia.org/resource/François-Xavier_Octavie_Fontaine, http://dbpedia.org/resource/George_Vandeput, http://dbpedia.org/resource/Nathaniel_Freeman_(physician), http://dbpedia.org/resource/Robert_Dudley_Oliver, http://dbpedia.org/resource/John_Harvey_(Royal_Navy_officer,_born_1740), http://dbpedia.org/resource/11th_Regiment_of_Connecticut_Militia, http://dbpedia.org/resource/William_Bedford_(Royal_Navy_officer), http://dbpedia.org/resource/1st_Pennsylvania_Regiment, http://dbpedia.org/resource/4th_Pennsylvania_Regiment, http://dbpedia.org/resource/5th_Connecticut_Regiment, http://dbpedia.org/resource/12th_Virginia_Regiment, http://dbpedia.org/resource/15th_Virginia_Regiment, http://dbpedia.org/resource/2nd_North_Carolina_Regiment, http://dbpedia.org/resource/4th_North_Carolina_Regiment, http://dbpedia.org/resource/4th_Virginia_Regiment, http://dbpedia.org/resource/5th_North_Carolina_Regiment, http://dbpedia.org/resource/5th_Virginia_Regiment, http://dbpedia.org/resource/13th_Virginia_Regiment, http://dbpedia.org/resource/George_Washington_in_the_American_Revolution, http://dbpedia.org/resource/Richard_Butler_(general), http://dbpedia.org/resource/2nd_Maryland_Regiment, http://dbpedia.org/resource/3rd_Maryland_Regiment, http://dbpedia.org/resource/6th_Maryland_Regiment, http://dbpedia.org/resource/1st_New_York_Regiment, http://dbpedia.org/resource/2nd_Massachusetts_Regiment, http://dbpedia.org/resource/2nd_New_Hampshire_Regiment, http://dbpedia.org/resource/2nd_New_York_Regiment, http://dbpedia.org/resource/4th_New_York_Regiment, http://dbpedia.org/resource/2nd_New_Jersey_Regiment, http://dbpedia.org/resource/Jean-Joseph_Sourbader_de_Gimat, http://dbpedia.org/resource/John_Mott_(captain), http://dbpedia.org/resource/13th_Pennsylvania_Regiment, http://dbpedia.org/resource/14th_Virginia_Regiment, http://dbpedia.org/resource/3rd_Virginia_Regiment, http://dbpedia.org/resource/6th_Virginia_Regiment, http://dbpedia.org/resource/Thomas-Antoine_de_Mauduit_du_Plessis, http://dbpedia.org/resource/1st_Virginia_Regiment, http://dbpedia.org/resource/7th_Connecticut_Regiment, http://dbpedia.org/resource/4th_Massachusetts_Regiment, http://dbpedia.org/resource/François_de_Fleury, http://dbpedia.org/resource/8th_Connecticut_Regiment, http://dbpedia.org/resource/3rd_Pennsylvania_Regiment, http://dbpedia.org/resource/Oliver_Cromwell_(American_soldier), http://dbpedia.org/resource/10th_Massachusetts_Regiment, http://dbpedia.org/resource/14th_Massachusetts_Regiment, http://dbpedia.org/resource/7th_Pennsylvania_Regiment, http://dbpedia.org/resource/1st_Continental_Artillery_Regiment, http://dbpedia.org/resource/Gist's_Additional_Continental_Regiment, http://dbpedia.org/resource/Henley's_Additional_Continental_Regiment, http://dbpedia.org/resource/Patton's_Additional_Continental_Regiment, http://dbpedia.org/resource/Oliver_Spencer, http://dbpedia.org/resource/Forman's_Additional_Continental_Regiment, http://dbpedia.org/resource/Spencer's_Additional_Continental_Regiment, http://dbpedia.org/resource/Thruston's_Additional_Continental_Regiment, http://dbpedia.org/resource/6th_Pennsylvania_Regiment, http://dbpedia.org/resource/11th_Pennsylvania_Regiment, http://dbpedia.org/resource/Thomas_Clark_(North_Carolina), http://dbpedia.org/resource/8th_Virginia_Regiment, http://dbpedia.org/resource/1st_New_Jersey_Regiment, http://dbpedia.org/resource/2nd_Pennsylvania_Regiment, http://dbpedia.org/resource/Richard_Parker_(colonel), http://dbpedia.org/resource/13th_Massachusetts_Regiment, http://dbpedia.org/resource/1st_Maryland_Regiment, http://dbpedia.org/resource/7th_Virginia_Regiment, http://dbpedia.org/resource/Maryland_and_Virginia_Rifle_Regiment, http://dbpedia.org/resource/4th_Continental_Artillery_Regiment, http://dbpedia.org/resource/5th_Pennsylvania_Regiment, http://dbpedia.org/resource/7th_Maryland_Regiment, http://dbpedia.org/resource/6th_North_Carolina_Regiment, http://dbpedia.org/resource/1st_North_Carolina_Regiment, http://dbpedia.org/resource/9th_Pennsylvania_Regiment, http://dbpedia.org/resource/German_Battalion, http://dbpedia.org/resource/Grayson's_Additional_Continental_Regiment, http://dbpedia.org/resource/3rd_New_Jersey_Regiment, http://dbpedia.org/resource/1st_Connecticut_Regiment, http://dbpedia.org/resource/4th_Connecticut_Regiment, http://dbpedia.org/resource/Lee's_Additional_Continental_Regiment, http://dbpedia.org/resource/Van_Alstyne's_Regiment_of_Militia, http://dbpedia.org/resource/1st_Canadian_Regiment, http://dbpedia.org/resource/3rd_New_Hampshire_Regiment, http://dbpedia.org/resource/8th_Massachusetts_Regiment, http://dbpedia.org/resource/Lemuel_Roberts, http://dbpedia.org/resource/7th_Massachusetts_Regiment, http://dbpedia.org/resource/Peter_Olcott, http://dbpedia.org/resource/John_Hardin, http://dbpedia.org/resource/8th_Pennsylvania_Regiment, http://dbpedia.org/resource/Ashley's_Regiment_of_Militia, http://dbpedia.org/resource/6th_Massachusetts_Regiment, http://dbpedia.org/resource/3rd_Massachusetts_Regiment, http://dbpedia.org/resource/5th_Massachusetts_Regiment, http://dbpedia.org/resource/James_Livingston_(American_Revolution), http://dbpedia.org/resource/Drake's_Regiment_of_Militia, http://dbpedia.org/resource/Evans'_Regiment_of_Militia, http://dbpedia.org/resource/Bellow's_Regiment_of_Militia, http://dbpedia.org/resource/Moore's_Regiment_of_Militia, http://dbpedia.org/resource/Cogswell's_Regiment_of_Militia, http://dbpedia.org/resource/Gage's_Regiment_of_Militia, http://dbpedia.org/resource/May's_Regiment_of_Militia, http://dbpedia.org/resource/Moulton's_Regiment_of_Militia, http://dbpedia.org/resource/Langdon's_Company_of_Light_Horse_Volunteers, http://dbpedia.org/resource/Welch's_Regiment_of_Militia, http://dbpedia.org/resource/Chase's_Regiment_of_Militia, http://dbpedia.org/resource/Joseph_Desha, http://dbpedia.org/resource/1st_Infantry_Regiment_(United_States), http://dbpedia.org/resource/Reasin_Beall, http://dbpedia.org/resource/John_Johnston_(Indian_agent), http://dbpedia.org/resource/James_Stephenson_(congressman), http://dbpedia.org/resource/Hugh_Brady] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 549, P: 0.519, R: 0.571, macro F1: 0.525, macro F1*: 0.544 -[INFO] Current time: 1962 ms, average: 2715.14 ms/ques, entity detection: 418.35 ms/ques (linking tool: 326.50 ms/ques), relation detection: 891.49 ms/ques (similarity: 701.06 ms/ques), EDG generation: 51.00 ms/ques +[INFO] Cumulative metrics, sample: 549, P: 0.519, R: 0.571, macro F1: 0.544 +[INFO] Current question consumed time: 1611 ms, average: 2480.70 ms/ques, entity detection: 377.07 ms/ques (linking tool: 296.94 ms/ques), relation detection: 832.53 ms/ques (similarity: 688.45 ms/ques), EDG generation: 50.83 ms/ques [INFO] Entity Linking P: 0.7486338797814208, R: 0.7021857923497268, micro F1: 0.7170613236187013, macro F1: 0.7246663167752359 [INFO] Relation Linking P: 0.6000303582270795, R: 0.5491803278688525, micro F1: 0.5629022465088039, macro F1: 0.5734803423763237 [INFO] Type Linking P: 0.5409836065573771, R: 0.5409836065573771, micro F1: 0.5409836065573771, macro F1: 0.5409836065573771 @@ -7709,8 +7709,8 @@ Question 549: In which city was the film star of Singhasan Battisi born ? [INFO] Golden Answer: [Ahmedabad@en, http://dbpedia.org/resource/Uttarakhand, http://dbpedia.org/resource/India, http://dbpedia.org/resource/Dehradun, http://dbpedia.org/resource/Kolkata, http://dbpedia.org/resource/West_Bengal, http://dbpedia.org/resource/Mumbai, http://dbpedia.org/resource/Uttar_Pradesh, http://dbpedia.org/resource/Varanasi] [INFO] Predicted Answer: [http://dbpedia.org/resource/Karan_Suchak, http://dbpedia.org/resource/Aditi_Sajwan, http://dbpedia.org/resource/Sayantani_Ghosh, http://dbpedia.org/resource/Navina_Bole, http://dbpedia.org/resource/Siddharth_Arora, Sayantani Ghosh@en, Aditi Sajwan@en, Navina Bole@en, Siddharth Arora@en, Cheshta Mehta@en] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 550, P: 0.518, R: 0.570, macro F1: 0.524, macro F1*: 0.543 -[INFO] Current time: 696 ms, average: 2711.47 ms/ques, entity detection: 417.73 ms/ques (linking tool: 326.00 ms/ques), relation detection: 890.30 ms/ques (similarity: 700.30 ms/ques), EDG generation: 50.96 ms/ques +[INFO] Cumulative metrics, sample: 550, P: 0.518, R: 0.570, macro F1: 0.543 +[INFO] Current question consumed time: 630 ms, average: 2477.33 ms/ques, entity detection: 376.51 ms/ques (linking tool: 296.48 ms/ques), relation detection: 831.43 ms/ques (similarity: 687.72 ms/ques), EDG generation: 50.77 ms/ques [INFO] Entity Linking P: 0.7490909090909091, R: 0.7027272727272728, micro F1: 0.7175757575757582, macro F1: 0.7251687823760461 [INFO] Relation Linking P: 0.6007575757575757, R: 0.5490909090909091, micro F1: 0.5630909090909092, macro F1: 0.5737634614693515 [INFO] Type Linking P: 0.5418181818181819, R: 0.5418181818181819, micro F1: 0.5418181818181819, macro F1: 0.5418181818181819 @@ -7723,8 +7723,8 @@ Question 550: What are the reason served by Toll Global Express which is also th [INFO] Golden Answer: [http://dbpedia.org/resource/Asia, http://dbpedia.org/resource/Australia] [INFO] Predicted Answer: [http://dbpedia.org/resource/Asia, http://dbpedia.org/resource/Australia] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 551, P: 0.519, R: 0.571, macro F1: 0.525, macro F1*: 0.544 -[INFO] Current time: 2053 ms, average: 2710.28 ms/ques, entity detection: 417.58 ms/ques (linking tool: 325.85 ms/ques), relation detection: 890.05 ms/ques (similarity: 699.81 ms/ques), EDG generation: 50.95 ms/ques +[INFO] Cumulative metrics, sample: 551, P: 0.519, R: 0.571, macro F1: 0.544 +[INFO] Current question consumed time: 1808 ms, average: 2476.12 ms/ques, entity detection: 376.33 ms/ques (linking tool: 296.33 ms/ques), relation detection: 831.17 ms/ques (similarity: 687.27 ms/ques), EDG generation: 50.77 ms/ques [INFO] Entity Linking P: 0.7495462794918331, R: 0.7032667876588021, micro F1: 0.7180883242589238, macro F1: 0.7256694148734173 [INFO] Relation Linking P: 0.6005747126436781, R: 0.5490018148820327, micro F1: 0.5629764065335754, macro F1: 0.5736314187334697 [INFO] Type Linking P: 0.542649727767695, R: 0.542649727767695, micro F1: 0.542649727767695, macro F1: 0.542649727767695 @@ -7737,8 +7737,8 @@ Question 551: List the territory of Tonkin Campaign ? [INFO] Golden Answer: [http://dbpedia.org/resource/Annam_(French_protectorate), http://dbpedia.org/resource/Tonkin] [INFO] Predicted Answer: [http://dbpedia.org/resource/Annam_(French_protectorate), http://dbpedia.org/resource/Tonkin, French protectorate over Tonkin and Annam@en] [INFO] P: 0.6666666666666666, R: 1.0, F: 0.8 -[INFO] Cumulative metrics, sample: 552, P: 0.520, R: 0.572, macro F1: 0.526, macro F1*: 0.544 -[INFO] Current time: 710 ms, average: 2706.65 ms/ques, entity detection: 417.24 ms/ques (linking tool: 325.55 ms/ques), relation detection: 888.68 ms/ques (similarity: 698.70 ms/ques), EDG generation: 50.87 ms/ques +[INFO] Cumulative metrics, sample: 552, P: 0.520, R: 0.572, macro F1: 0.544 +[INFO] Current question consumed time: 622 ms, average: 2472.76 ms/ques, entity detection: 376.00 ms/ques (linking tool: 296.03 ms/ques), relation detection: 829.90 ms/ques (similarity: 686.17 ms/ques), EDG generation: 50.71 ms/ques [INFO] Entity Linking P: 0.75, R: 0.7038043478260869, micro F1: 0.7185990338164258, macro F1: 0.7261682242990654 [INFO] Relation Linking P: 0.6012983091787439, R: 0.5498188405797102, micro F1: 0.563768115942029, macro F1: 0.5744074602043241 [INFO] Type Linking P: 0.5434782608695652, R: 0.5434782608695652, micro F1: 0.5434782608695652, macro F1: 0.5434782608695652 @@ -7751,8 +7751,8 @@ Question 552: For which teams did Ryan Powell play for? [INFO] Golden Answer: [http://dbpedia.org/resource/Syracuse_University] [INFO] Predicted Answer: [http://dbpedia.org/resource/Syracuse_University] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 553, P: 0.520, R: 0.572, macro F1: 0.527, macro F1*: 0.545 -[INFO] Current time: 1262 ms, average: 2704.04 ms/ques, entity detection: 416.72 ms/ques (linking tool: 325.13 ms/ques), relation detection: 887.92 ms/ques (similarity: 698.37 ms/ques), EDG generation: 50.80 ms/ques +[INFO] Cumulative metrics, sample: 553, P: 0.520, R: 0.572, macro F1: 0.545 +[INFO] Current question consumed time: 1131 ms, average: 2470.33 ms/ques, entity detection: 375.52 ms/ques (linking tool: 295.64 ms/ques), relation detection: 829.17 ms/ques (similarity: 685.87 ms/ques), EDG generation: 50.63 ms/ques [INFO] Entity Linking P: 0.7504520795660036, R: 0.7043399638336347, micro F1: 0.7191078963230868, macro F1: 0.7266652206114564 [INFO] Relation Linking P: 0.6020192887281495, R: 0.5506329113924051, micro F1: 0.5645569620253166, macro F1: 0.5751806722480477 [INFO] Type Linking P: 0.5443037974683544, R: 0.5443037974683544, micro F1: 0.5443037974683544, macro F1: 0.5443037974683544 @@ -7765,8 +7765,8 @@ Question 553: What are the awards won by the person who wrote the screen play fo [INFO] Golden Answer: [http://dbpedia.org/resource/César_Award_for_Best_Director, http://dbpedia.org/resource/Academy_Award_for_Best_Director, http://dbpedia.org/resource/Academy_Awards, http://dbpedia.org/resource/César_Award, BAFTA for Best Director@en, and other see ''[[#Awards@en] [INFO] Predicted Answer: [http://dbpedia.org/resource/Goa_Medical_College, http://dbpedia.org/resource/University_of_Porto] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 554, P: 0.520, R: 0.571, macro F1: 0.526, macro F1*: 0.544 -[INFO] Current time: 2603 ms, average: 2703.86 ms/ques, entity detection: 416.32 ms/ques (linking tool: 324.79 ms/ques), relation detection: 887.79 ms/ques (similarity: 698.30 ms/ques), EDG generation: 50.91 ms/ques +[INFO] Cumulative metrics, sample: 554, P: 0.520, R: 0.571, macro F1: 0.544 +[INFO] Current question consumed time: 2256 ms, average: 2469.95 ms/ques, entity detection: 375.11 ms/ques (linking tool: 295.31 ms/ques), relation detection: 829.00 ms/ques (similarity: 685.81 ms/ques), EDG generation: 50.79 ms/ques [INFO] Entity Linking P: 0.7490974729241877, R: 0.7030685920577617, micro F1: 0.7178098676293628, macro F1: 0.7253535505381506 [INFO] Relation Linking P: 0.6009326113116726, R: 0.5496389891696751, micro F1: 0.5635379061371842, macro F1: 0.5741424399876721 [INFO] Type Linking P: 0.5433212996389891, R: 0.5433212996389891, micro F1: 0.5433212996389891, macro F1: 0.5433212996389891 @@ -7779,8 +7779,8 @@ Question 554: Total number of places where people drafted in the detroit pistons [INFO] Golden Answer: [8] [INFO] Predicted Answer: [3] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 555, P: 0.519, R: 0.570, macro F1: 0.525, macro F1*: 0.543 -[INFO] Current time: 2448 ms, average: 2703.40 ms/ques, entity detection: 416.18 ms/ques (linking tool: 324.59 ms/ques), relation detection: 887.47 ms/ques (similarity: 698.30 ms/ques), EDG generation: 50.88 ms/ques +[INFO] Cumulative metrics, sample: 555, P: 0.519, R: 0.570, macro F1: 0.543 +[INFO] Current question consumed time: 2221 ms, average: 2469.50 ms/ques, entity detection: 374.94 ms/ques (linking tool: 295.11 ms/ques), relation detection: 828.70 ms/ques (similarity: 685.85 ms/ques), EDG generation: 50.77 ms/ques [INFO] Entity Linking P: 0.7495495495495496, R: 0.7036036036036036, micro F1: 0.7183183183183189, macro F1: 0.7258502147528806 [INFO] Relation Linking P: 0.5998498498498498, R: 0.5486486486486486, micro F1: 0.5625225225225226, macro F1: 0.5731079491048113 [INFO] Type Linking P: 0.5423423423423424, R: 0.5423423423423424, micro F1: 0.5423423423423424, macro F1: 0.5423423423423424 @@ -7793,8 +7793,8 @@ Question 555: What band made The trial and One of these days song? [INFO] Golden Answer: [http://dbpedia.org/resource/Pink_Floyd] [INFO] Predicted Answer: [Kriz G. Gazmen@en] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 556, P: 0.518, R: 0.569, macro F1: 0.524, macro F1*: 0.542 -[INFO] Current time: 1099 ms, average: 2700.51 ms/ques, entity detection: 415.62 ms/ques (linking tool: 324.08 ms/ques), relation detection: 886.56 ms/ques (similarity: 697.98 ms/ques), EDG generation: 50.85 ms/ques +[INFO] Cumulative metrics, sample: 556, P: 0.518, R: 0.569, macro F1: 0.542 +[INFO] Current question consumed time: 1006 ms, average: 2466.87 ms/ques, entity detection: 374.43 ms/ques (linking tool: 294.65 ms/ques), relation detection: 827.85 ms/ques (similarity: 685.52 ms/ques), EDG generation: 50.76 ms/ques [INFO] Entity Linking P: 0.7482014388489209, R: 0.7023381294964028, micro F1: 0.7170263788968831, macro F1: 0.7245447287551237 [INFO] Relation Linking P: 0.5987709832134291, R: 0.5476618705035972, micro F1: 0.5615107913669065, macro F1: 0.5720771794121768 [INFO] Type Linking P: 0.5431654676258992, R: 0.5431654676258992, micro F1: 0.5431654676258992, macro F1: 0.5431654676258992 @@ -7807,8 +7807,8 @@ Question 556: Name the club of Julie Hastings ? [INFO] Golden Answer: [http://dbpedia.org/resource/Ontario, http://dbpedia.org/resource/Bayview_Golf_&_Country_Club] [INFO] Predicted Answer: [http://dbpedia.org/resource/Ontario, http://dbpedia.org/resource/Bayview_Golf_&_Country_Club] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 557, P: 0.519, R: 0.570, macro F1: 0.525, macro F1*: 0.543 -[INFO] Current time: 788 ms, average: 2697.08 ms/ques, entity detection: 415.27 ms/ques (linking tool: 323.78 ms/ques), relation detection: 885.33 ms/ques (similarity: 696.94 ms/ques), EDG generation: 50.78 ms/ques +[INFO] Cumulative metrics, sample: 557, P: 0.519, R: 0.570, macro F1: 0.543 +[INFO] Current question consumed time: 650 ms, average: 2463.61 ms/ques, entity detection: 374.07 ms/ques (linking tool: 294.34 ms/ques), relation detection: 826.69 ms/ques (similarity: 684.50 ms/ques), EDG generation: 50.69 ms/ques [INFO] Entity Linking P: 0.748653500897666, R: 0.7028725314183124, micro F1: 0.7175344105326158, macro F1: 0.7250410528174057 [INFO] Relation Linking P: 0.5994913225613404, R: 0.5484739676840216, micro F1: 0.56229802513465, macro F1: 0.5728490000025737 [INFO] Type Linking P: 0.5439856373429084, R: 0.5439856373429084, micro F1: 0.5439856373429084, macro F1: 0.5439856373429084 @@ -7821,8 +7821,8 @@ Question 557: From where does the river Upper Neretva start flowing? [INFO] Golden Answer: [http://dbpedia.org/resource/Adriatic_Sea] [INFO] Predicted Answer: [http://dbpedia.org/resource/Adriatic_Sea] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 558, P: 0.519, R: 0.571, macro F1: 0.525, macro F1*: 0.544 -[INFO] Current time: 1164 ms, average: 2694.33 ms/ques, entity detection: 414.86 ms/ques (linking tool: 323.42 ms/ques), relation detection: 884.53 ms/ques (similarity: 696.19 ms/ques), EDG generation: 50.71 ms/ques +[INFO] Cumulative metrics, sample: 558, P: 0.519, R: 0.571, macro F1: 0.544 +[INFO] Current question consumed time: 1048 ms, average: 2461.07 ms/ques, entity detection: 373.68 ms/ques (linking tool: 294.00 ms/ques), relation detection: 825.97 ms/ques (similarity: 683.77 ms/ques), EDG generation: 50.62 ms/ques [INFO] Entity Linking P: 0.7491039426523297, R: 0.703405017921147, micro F1: 0.7180406212664283, macro F1: 0.7255355891203934 [INFO] Relation Linking P: 0.6002090800477896, R: 0.5492831541218638, micro F1: 0.5630824372759857, macro F1: 0.5736180320685385 [INFO] Type Linking P: 0.5448028673835126, R: 0.5448028673835126, micro F1: 0.5448028673835126, macro F1: 0.5448028673835126 @@ -7835,8 +7835,8 @@ Question 558: Name the university located in Indiana and has affiliations with K [INFO] Golden Answer: [http://dbpedia.org/resource/Kaplan_University] [INFO] Predicted Answer: [http://dbpedia.org/resource/WTIU, http://dbpedia.org/resource/Emens_Auditorium,_Ball_State_University, http://dbpedia.org/resource/Owensville_High_School_(Indiana), http://dbpedia.org/resource/WVUT, http://dbpedia.org/resource/Indiana_Geological_Survey, http://dbpedia.org/resource/U.S._Philatelic_Classics_Society, http://dbpedia.org/resource/WIPB, http://dbpedia.org/resource/Indiana_State_University_Recreational_Center, http://dbpedia.org/resource/Indianapolis_Art_Center, http://dbpedia.org/resource/O._C._Lewis_Gymnasium, http://dbpedia.org/resource/Bracken_House, http://dbpedia.org/resource/Morgan–Monroe_Observatory, http://dbpedia.org/resource/Herron_School_of_Art_and_Design, http://dbpedia.org/resource/Indianapolis_Zoo, http://dbpedia.org/resource/National_College_(Virginia), http://dbpedia.org/resource/Mennonite_Historical_Library, http://dbpedia.org/resource/Indiana_University_Piano_Academy] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 559, P: 0.518, R: 0.570, macro F1: 0.524, macro F1*: 0.543 -[INFO] Current time: 8498 ms, average: 2704.71 ms/ques, entity detection: 415.65 ms/ques (linking tool: 324.16 ms/ques), relation detection: 888.67 ms/ques (similarity: 701.16 ms/ques), EDG generation: 50.67 ms/ques +[INFO] Cumulative metrics, sample: 559, P: 0.518, R: 0.570, macro F1: 0.543 +[INFO] Current question consumed time: 7772 ms, average: 2470.57 ms/ques, entity detection: 374.45 ms/ques (linking tool: 294.71 ms/ques), relation detection: 829.85 ms/ques (similarity: 688.72 ms/ques), EDG generation: 50.58 ms/ques [INFO] Entity Linking P: 0.7495527728085868, R: 0.7030411449016101, micro F1: 0.7179487179487186, macro F1: 0.7255523145659473 [INFO] Relation Linking P: 0.5991353607632677, R: 0.5483005366726297, micro F1: 0.5620751341681575, macro F1: 0.5725918817428345 [INFO] Type Linking P: 0.5438282647584973, R: 0.5438282647584973, micro F1: 0.5438282647584973, macro F1: 0.5438282647584973 @@ -7849,8 +7849,8 @@ Question 559: List the newspapers whose offices are situated in 1211 Avenue of t [INFO] Golden Answer: [http://dbpedia.org/resource/The_Wall_Street_Journal, http://dbpedia.org/resource/New_York_Post] [INFO] Predicted Answer: [http://dbpedia.org/resource/The_Wall_Street_Journal, http://dbpedia.org/resource/New_York_Post] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 560, P: 0.519, R: 0.571, macro F1: 0.525, macro F1*: 0.544 -[INFO] Current time: 1243 ms, average: 2702.10 ms/ques, entity detection: 415.35 ms/ques (linking tool: 323.92 ms/ques), relation detection: 887.58 ms/ques (similarity: 700.43 ms/ques), EDG generation: 50.61 ms/ques +[INFO] Cumulative metrics, sample: 560, P: 0.519, R: 0.571, macro F1: 0.544 +[INFO] Current question consumed time: 1098 ms, average: 2468.12 ms/ques, entity detection: 374.16 ms/ques (linking tool: 294.48 ms/ques), relation detection: 828.82 ms/ques (similarity: 688.00 ms/ques), EDG generation: 50.51 ms/ques [INFO] Entity Linking P: 0.75, R: 0.7035714285714286, micro F1: 0.7184523809523816, macro F1: 0.726044226044226 [INFO] Relation Linking P: 0.5998511904761904, R: 0.5491071428571429, micro F1: 0.562857142857143, macro F1: 0.5733586045478936 [INFO] Type Linking P: 0.5428571428571428, R: 0.5428571428571428, micro F1: 0.5428571428571428, macro F1: 0.5428571428571428 @@ -7863,8 +7863,8 @@ Question 560: Was Morrissey a member of the Smiths? [INFO] Golden Answer: [true] [INFO] Predicted Answer: [true] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 561, P: 0.520, R: 0.571, macro F1: 0.526, macro F1*: 0.545 -[INFO] Current time: 1143 ms, average: 2699.32 ms/ques, entity detection: 415.16 ms/ques (linking tool: 323.77 ms/ques), relation detection: 886.84 ms/ques (similarity: 699.73 ms/ques), EDG generation: 50.54 ms/ques +[INFO] Cumulative metrics, sample: 561, P: 0.520, R: 0.571, macro F1: 0.545 +[INFO] Current question consumed time: 1036 ms, average: 2465.57 ms/ques, entity detection: 373.96 ms/ques (linking tool: 294.31 ms/ques), relation detection: 828.18 ms/ques (similarity: 687.31 ms/ques), EDG generation: 50.45 ms/ques [INFO] Entity Linking P: 0.750445632798574, R: 0.7040998217468806, micro F1: 0.7189542483660137, macro F1: 0.7265343748907763 [INFO] Relation Linking P: 0.600564468211527, R: 0.5499108734402852, micro F1: 0.5636363636363637, macro F1: 0.5741225723225495 [INFO] Type Linking P: 0.5436720142602496, R: 0.5436720142602496, micro F1: 0.5436720142602496, macro F1: 0.5436720142602496 @@ -7877,8 +7877,8 @@ Question 561: Did Phillippe Cabot go to Columbia for studying? [INFO] Golden Answer: [true] [INFO] Predicted Answer: [true] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 562, P: 0.521, R: 0.572, macro F1: 0.527, macro F1*: 0.545 -[INFO] Current time: 4092 ms, average: 2701.80 ms/ques, entity detection: 415.46 ms/ques (linking tool: 323.91 ms/ques), relation detection: 889.46 ms/ques (similarity: 700.76 ms/ques), EDG generation: 50.48 ms/ques +[INFO] Cumulative metrics, sample: 562, P: 0.521, R: 0.572, macro F1: 0.545 +[INFO] Current question consumed time: 3651 ms, average: 2467.67 ms/ques, entity detection: 374.15 ms/ques (linking tool: 294.40 ms/ques), relation detection: 830.58 ms/ques (similarity: 688.37 ms/ques), EDG generation: 50.39 ms/ques [INFO] Entity Linking P: 0.7508896797153025, R: 0.7046263345195729, micro F1: 0.7194543297746151, macro F1: 0.7270227705800972 [INFO] Relation Linking P: 0.6012752075919335, R: 0.550711743772242, micro F1: 0.5644128113879004, macro F1: 0.5748837999733405 [INFO] Type Linking P: 0.5444839857651246, R: 0.5444839857651246, micro F1: 0.5444839857651246, macro F1: 0.5444839857651246 @@ -7891,8 +7891,8 @@ Question 562: What river originates in Kingston Ontario? [INFO] Golden Answer: [http://dbpedia.org/resource/Saint_Lawrence_River] [INFO] Predicted Answer: [http://dbpedia.org/resource/Headstones_(band), http://dbpedia.org/resource/The_Abrams_Brothers, http://dbpedia.org/resource/The_Mahones, http://dbpedia.org/resource/Devon_Sproule, http://dbpedia.org/resource/The_Tragically_Hip, http://dbpedia.org/resource/PS_I_Love_You_(band), http://dbpedia.org/resource/Weeping_Tile_(band), http://dbpedia.org/resource/Tea_for_the_Voyage, http://dbpedia.org/resource/I_Hate_Sally, http://dbpedia.org/resource/Hugh_Dillon, http://dbpedia.org/resource/The_Arrogant_Worms, http://dbpedia.org/resource/Luther_Wright_and_the_Wrongs, http://dbpedia.org/resource/The_Inbreds] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 563, P: 0.520, R: 0.571, macro F1: 0.526, macro F1*: 0.544 -[INFO] Current time: 5689 ms, average: 2707.11 ms/ques, entity detection: 415.16 ms/ques (linking tool: 323.66 ms/ques), relation detection: 892.45 ms/ques (similarity: 704.80 ms/ques), EDG generation: 50.40 ms/ques +[INFO] Cumulative metrics, sample: 563, P: 0.520, R: 0.571, macro F1: 0.544 +[INFO] Current question consumed time: 5169 ms, average: 2472.47 ms/ques, entity detection: 373.84 ms/ques (linking tool: 294.13 ms/ques), relation detection: 833.30 ms/ques (similarity: 692.48 ms/ques), EDG generation: 50.32 ms/ques [INFO] Entity Linking P: 0.7513321492007105, R: 0.7051509769094139, micro F1: 0.7199526346950865, macro F1: 0.7275094225187367 [INFO] Relation Linking P: 0.6002072232089993, R: 0.5497335701598579, micro F1: 0.5634103019538189, macro F1: 0.5738626919804926 [INFO] Type Linking P: 0.5435168738898757, R: 0.5435168738898757, micro F1: 0.5435168738898757, macro F1: 0.5435168738898757 @@ -7905,8 +7905,8 @@ Question 563: By what style was Providence Chapel, Charlwood made? [INFO] Golden Answer: [http://dbpedia.org/resource/New_England, http://dbpedia.org/resource/Vernacular_architecture] [INFO] Predicted Answer: [http://dbpedia.org/resource/New_England, http://dbpedia.org/resource/Vernacular_architecture] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 564, P: 0.521, R: 0.572, macro F1: 0.527, macro F1*: 0.545 -[INFO] Current time: 1608 ms, average: 2705.16 ms/ques, entity detection: 414.80 ms/ques (linking tool: 323.38 ms/ques), relation detection: 891.62 ms/ques (similarity: 704.29 ms/ques), EDG generation: 50.34 ms/ques +[INFO] Cumulative metrics, sample: 564, P: 0.521, R: 0.572, macro F1: 0.545 +[INFO] Current question consumed time: 1403 ms, average: 2470.58 ms/ques, entity detection: 373.47 ms/ques (linking tool: 293.84 ms/ques), relation detection: 832.50 ms/ques (similarity: 691.96 ms/ques), EDG generation: 50.25 ms/ques [INFO] Entity Linking P: 0.75177304964539, R: 0.7056737588652482, micro F1: 0.7204491725768328, macro F1: 0.7279943400459008 [INFO] Relation Linking P: 0.6009160756501182, R: 0.550531914893617, micro F1: 0.5641843971631206, macro F1: 0.5746216599184756 [INFO] Type Linking P: 0.5443262411347518, R: 0.5443262411347518, micro F1: 0.5443262411347518, macro F1: 0.5443262411347518 @@ -7919,8 +7919,8 @@ Question 564: For how many PMs did the politician work, whose successor was King [INFO] Golden Answer: [7] [INFO] Predicted Answer: [] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 565, P: 0.520, R: 0.571, macro F1: 0.526, macro F1*: 0.544 -[INFO] Current time: 3494 ms, average: 2706.55 ms/ques, entity detection: 414.77 ms/ques (linking tool: 323.22 ms/ques), relation detection: 892.33 ms/ques (similarity: 704.52 ms/ques), EDG generation: 50.30 ms/ques +[INFO] Cumulative metrics, sample: 565, P: 0.520, R: 0.571, macro F1: 0.544 +[INFO] Current question consumed time: 2917 ms, average: 2471.37 ms/ques, entity detection: 373.33 ms/ques (linking tool: 293.64 ms/ques), relation detection: 833.05 ms/ques (similarity: 692.22 ms/ques), EDG generation: 50.22 ms/ques [INFO] Entity Linking P: 0.7504424778761062, R: 0.7044247787610619, micro F1: 0.7191740412979358, macro F1: 0.7267058544882975 [INFO] Relation Linking P: 0.5998525073746312, R: 0.5495575221238939, micro F1: 0.5631858407079647, macro F1: 0.5736046304318944 [INFO] Type Linking P: 0.5433628318584071, R: 0.5433628318584071, micro F1: 0.5433628318584071, macro F1: 0.5433628318584071 @@ -7933,8 +7933,8 @@ Question 565: list some companies founded by Jim Harris. [INFO] Golden Answer: [http://dbpedia.org/resource/Compaq] [INFO] Predicted Answer: [American writer@en] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 566, P: 0.519, R: 0.570, macro F1: 0.525, macro F1*: 0.543 -[INFO] Current time: 624 ms, average: 2702.87 ms/ques, entity detection: 414.37 ms/ques (linking tool: 322.85 ms/ques), relation detection: 890.89 ms/ques (similarity: 703.37 ms/ques), EDG generation: 50.23 ms/ques +[INFO] Cumulative metrics, sample: 566, P: 0.519, R: 0.570, macro F1: 0.543 +[INFO] Current question consumed time: 528 ms, average: 2467.93 ms/ques, entity detection: 372.95 ms/ques (linking tool: 293.29 ms/ques), relation detection: 831.71 ms/ques (similarity: 691.08 ms/ques), EDG generation: 50.14 ms/ques [INFO] Entity Linking P: 0.7491166077738516, R: 0.7031802120141343, micro F1: 0.7179034157832751, macro F1: 0.7254219218831943 [INFO] Relation Linking P: 0.5987926972909304, R: 0.5485865724381626, micro F1: 0.5621908127208481, macro F1: 0.5725911946890818 [INFO] Type Linking P: 0.5441696113074205, R: 0.5441696113074205, micro F1: 0.5441696113074205, macro F1: 0.5441696113074205 @@ -7947,8 +7947,8 @@ Question 566: Give me the total number of employer of the engineers whose one of [INFO] Golden Answer: [9] [INFO] Predicted Answer: [9, 11, 2] [INFO] P: 0.3333333333333333, R: 1.0, F: 0.5 -[INFO] Cumulative metrics, sample: 567, P: 0.519, R: 0.571, macro F1: 0.525, macro F1*: 0.543 -[INFO] Current time: 2127 ms, average: 2701.86 ms/ques, entity detection: 414.07 ms/ques (linking tool: 322.49 ms/ques), relation detection: 890.25 ms/ques (similarity: 702.81 ms/ques), EDG generation: 50.26 ms/ques +[INFO] Cumulative metrics, sample: 567, P: 0.519, R: 0.571, macro F1: 0.543 +[INFO] Current question consumed time: 1716 ms, average: 2466.61 ms/ques, entity detection: 372.59 ms/ques (linking tool: 292.93 ms/ques), relation detection: 831.04 ms/ques (similarity: 690.55 ms/ques), EDG generation: 50.17 ms/ques [INFO] Entity Linking P: 0.7495590828924162, R: 0.7037037037037037, micro F1: 0.7184009406231635, macro F1: 0.7259079467817332 [INFO] Relation Linking P: 0.5995002939447384, R: 0.5493827160493827, micro F1: 0.562962962962963, macro F1: 0.5733483686236234 [INFO] Type Linking P: 0.5432098765432098, R: 0.5432098765432098, micro F1: 0.5432098765432098, macro F1: 0.5432098765432098 @@ -7961,8 +7961,8 @@ Question 567: How many developers have games where I can hear Lightning McQueen? [INFO] Golden Answer: [14] [INFO] Predicted Answer: [9, 2] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 568, P: 0.518, R: 0.570, macro F1: 0.524, macro F1*: 0.543 -[INFO] Current time: 1617 ms, average: 2699.95 ms/ques, entity detection: 413.99 ms/ques (linking tool: 322.33 ms/ques), relation detection: 889.30 ms/ques (similarity: 702.11 ms/ques), EDG generation: 50.20 ms/ques +[INFO] Cumulative metrics, sample: 568, P: 0.518, R: 0.570, macro F1: 0.543 +[INFO] Current question consumed time: 1298 ms, average: 2464.55 ms/ques, entity detection: 372.43 ms/ques (linking tool: 292.71 ms/ques), relation detection: 830.12 ms/ques (similarity: 689.86 ms/ques), EDG generation: 50.11 ms/ques [INFO] Entity Linking P: 0.75, R: 0.704225352112676, micro F1: 0.7188967136150242, macro F1: 0.7263922518159805 [INFO] Relation Linking P: 0.6002053990610328, R: 0.5492957746478874, micro F1: 0.5631455399061034, macro F1: 0.5736232326954709 [INFO] Type Linking P: 0.5440140845070423, R: 0.5440140845070423, micro F1: 0.5440140845070423, macro F1: 0.5440140845070423 @@ -7975,8 +7975,8 @@ Question 568: Which universit's affiliation is with Kalpan, Inc. and has campus [INFO] Golden Answer: [http://dbpedia.org/resource/Kaplan_University] [INFO] Predicted Answer: [http://dbpedia.org/resource/Kaplan_University] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 569, P: 0.519, R: 0.570, macro F1: 0.525, macro F1*: 0.543 -[INFO] Current time: 6630 ms, average: 2706.86 ms/ques, entity detection: 414.87 ms/ques (linking tool: 323.06 ms/ques), relation detection: 892.91 ms/ques (similarity: 703.65 ms/ques), EDG generation: 50.23 ms/ques +[INFO] Cumulative metrics, sample: 569, P: 0.519, R: 0.570, macro F1: 0.543 +[INFO] Current question consumed time: 5607 ms, average: 2470.07 ms/ques, entity detection: 373.16 ms/ques (linking tool: 293.35 ms/ques), relation detection: 833.31 ms/ques (similarity: 691.42 ms/ques), EDG generation: 50.14 ms/ques [INFO] Entity Linking P: 0.7504393673110721, R: 0.7038664323374341, micro F1: 0.7188049209138847, macro F1: 0.7264071700497506 [INFO] Relation Linking P: 0.6009080257762155, R: 0.5492091388400703, micro F1: 0.5633274751025192, macro F1: 0.5738966246430175 [INFO] Type Linking P: 0.5430579964850615, R: 0.5430579964850615, micro F1: 0.5430579964850615, macro F1: 0.5430579964850615 @@ -7989,8 +7989,8 @@ Question 569: How many record labels publish works of Jazz fusion? [INFO] Golden Answer: [778] [INFO] Predicted Answer: [1] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 570, P: 0.518, R: 0.569, macro F1: 0.524, macro F1*: 0.542 -[INFO] Current time: 2211 ms, average: 2705.99 ms/ques, entity detection: 414.93 ms/ques (linking tool: 323.08 ms/ques), relation detection: 892.43 ms/ques (similarity: 703.37 ms/ques), EDG generation: 50.17 ms/ques +[INFO] Cumulative metrics, sample: 570, P: 0.518, R: 0.569, macro F1: 0.542 +[INFO] Current question consumed time: 1881 ms, average: 2469.04 ms/ques, entity detection: 373.09 ms/ques (linking tool: 293.28 ms/ques), relation detection: 832.78 ms/ques (similarity: 691.16 ms/ques), EDG generation: 50.08 ms/ques [INFO] Entity Linking P: 0.7491228070175439, R: 0.7026315789473684, micro F1: 0.7175438596491235, macro F1: 0.7251327715058037 [INFO] Relation Linking P: 0.6016081871345028, R: 0.5491228070175439, micro F1: 0.5635087719298247, macro F1: 0.5741685556796323 [INFO] Type Linking P: 0.5421052631578948, R: 0.5421052631578948, micro F1: 0.5421052631578948, macro F1: 0.5421052631578948 @@ -8003,8 +8003,8 @@ Question 570: What are the regions in which the distributor of Secrets and Lies [INFO] Golden Answer: [http://dbpedia.org/resource/North_America] [INFO] Predicted Answer: [http://dbpedia.org/resource/United_Artists] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 571, P: 0.517, R: 0.568, macro F1: 0.523, macro F1*: 0.541 -[INFO] Current time: 2088 ms, average: 2704.90 ms/ques, entity detection: 414.56 ms/ques (linking tool: 322.70 ms/ques), relation detection: 892.17 ms/ques (similarity: 703.59 ms/ques), EDG generation: 50.17 ms/ques +[INFO] Cumulative metrics, sample: 571, P: 0.517, R: 0.568, macro F1: 0.541 +[INFO] Current question consumed time: 1804 ms, average: 2467.87 ms/ques, entity detection: 372.74 ms/ques (linking tool: 292.93 ms/ques), relation detection: 832.46 ms/ques (similarity: 691.39 ms/ques), EDG generation: 50.08 ms/ques [INFO] Entity Linking P: 0.7478108581436077, R: 0.7014010507880911, micro F1: 0.7162872154115594, macro F1: 0.7238628367045677 [INFO] Relation Linking P: 0.6023058960887332, R: 0.5490367775831874, micro F1: 0.5636894337419733, macro F1: 0.5744390369085599 [INFO] Type Linking P: 0.542907180385289, R: 0.542907180385289, micro F1: 0.542907180385289, macro F1: 0.542907180385289 @@ -8017,8 +8017,8 @@ Question 571: List all the writers of the comics which has Robert Bernstein as o [INFO] Golden Answer: [http://dbpedia.org/resource/Robert_Bernstein_(comics), http://dbpedia.org/resource/Stan_Lee, http://dbpedia.org/resource/Larry_Lieber] [INFO] Predicted Answer: [http://dbpedia.org/resource/Robert_Bernstein_(comics), http://dbpedia.org/resource/Stan_Lee, http://dbpedia.org/resource/Larry_Lieber, http://dbpedia.org/resource/Al_Feldstein] [INFO] P: 0.75, R: 1.0, F: 0.8571428571428571 -[INFO] Cumulative metrics, sample: 572, P: 0.517, R: 0.569, macro F1: 0.524, macro F1*: 0.542 -[INFO] Current time: 1264 ms, average: 2702.38 ms/ques, entity detection: 414.15 ms/ques (linking tool: 322.32 ms/ques), relation detection: 891.35 ms/ques (similarity: 702.93 ms/ques), EDG generation: 50.17 ms/ques +[INFO] Cumulative metrics, sample: 572, P: 0.517, R: 0.569, macro F1: 0.542 +[INFO] Current question consumed time: 1084 ms, average: 2465.45 ms/ques, entity detection: 372.33 ms/ques (linking tool: 292.57 ms/ques), relation detection: 831.66 ms/ques (similarity: 690.75 ms/ques), EDG generation: 50.09 ms/ques [INFO] Entity Linking P: 0.7482517482517482, R: 0.7019230769230769, micro F1: 0.7167832167832174, macro F1: 0.7243473825752307 [INFO] Relation Linking P: 0.6012529137529137, R: 0.5480769230769231, micro F1: 0.562703962703963, macro F1: 0.5734347728580204 [INFO] Type Linking P: 0.541958041958042, R: 0.541958041958042, micro F1: 0.541958041958042, macro F1: 0.541958041958042 @@ -8031,8 +8031,8 @@ Question 572: What are the settlements whose neighboring municipalities is Lancy [INFO] Golden Answer: [http://dbpedia.org/resource/Carouge, http://dbpedia.org/resource/Vernier,_Switzerland, http://dbpedia.org/resource/Plan-les-Ouates, http://dbpedia.org/resource/Geneva, http://dbpedia.org/resource/Onex,_Switzerland] [INFO] Predicted Answer: [http://dbpedia.org/resource/Carouge, http://dbpedia.org/resource/Vernier,_Switzerland, http://dbpedia.org/resource/Plan-les-Ouates, http://dbpedia.org/resource/Geneva, http://dbpedia.org/resource/Onex,_Switzerland] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 573, P: 0.518, R: 0.570, macro F1: 0.524, macro F1*: 0.543 -[INFO] Current time: 1361 ms, average: 2700.04 ms/ques, entity detection: 413.72 ms/ques (linking tool: 321.92 ms/ques), relation detection: 890.75 ms/ques (similarity: 702.72 ms/ques), EDG generation: 50.11 ms/ques +[INFO] Cumulative metrics, sample: 573, P: 0.518, R: 0.570, macro F1: 0.543 +[INFO] Current question consumed time: 1175 ms, average: 2463.20 ms/ques, entity detection: 371.90 ms/ques (linking tool: 292.19 ms/ques), relation detection: 831.05 ms/ques (similarity: 690.55 ms/ques), EDG generation: 50.04 ms/ques [INFO] Entity Linking P: 0.7486910994764397, R: 0.7024432809773123, micro F1: 0.7172774869109955, macro F1: 0.7248302285971544 [INFO] Relation Linking P: 0.6002036067481094, R: 0.5471204188481675, micro F1: 0.5617219313554395, macro F1: 0.572434014092125 [INFO] Type Linking P: 0.5410122164048866, R: 0.5410122164048866, micro F1: 0.5410122164048866, macro F1: 0.5410122164048866 @@ -8045,8 +8045,8 @@ Question 573: Count the total number of hometown of the gymnasts whose one of t [INFO] Golden Answer: [65] [INFO] Predicted Answer: [16, 69, 65, 12] [INFO] P: 0.25, R: 1.0, F: 0.4 -[INFO] Cumulative metrics, sample: 574, P: 0.518, R: 0.571, macro F1: 0.524, macro F1*: 0.543 -[INFO] Current time: 16042 ms, average: 2723.29 ms/ques, entity detection: 420.75 ms/ques (linking tool: 328.89 ms/ques), relation detection: 898.03 ms/ques (similarity: 708.29 ms/ques), EDG generation: 50.11 ms/ques +[INFO] Cumulative metrics, sample: 574, P: 0.518, R: 0.571, macro F1: 0.543 +[INFO] Current question consumed time: 13991 ms, average: 2483.29 ms/ques, entity detection: 378.04 ms/ques (linking tool: 298.32 ms/ques), relation detection: 837.17 ms/ques (similarity: 696.07 ms/ques), EDG generation: 50.04 ms/ques [INFO] Entity Linking P: 0.7491289198606271, R: 0.7029616724738676, micro F1: 0.7177700348432062, macro F1: 0.7253113837162881 [INFO] Relation Linking P: 0.6009001161440185, R: 0.5479094076655052, micro F1: 0.5624854819976773, macro F1: 0.5731826205806966 [INFO] Type Linking P: 0.5418118466898955, R: 0.5418118466898955, micro F1: 0.5418118466898955, macro F1: 0.5418118466898955 @@ -8059,8 +8059,8 @@ Question 574: In which states does the river flow whose left tributary is Toms C [INFO] Golden Answer: [http://dbpedia.org/resource/Maryland, http://dbpedia.org/resource/Pennsylvania] [INFO] Predicted Answer: [http://dbpedia.org/resource/Maryland, http://dbpedia.org/resource/Pennsylvania] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 575, P: 0.519, R: 0.571, macro F1: 0.525, macro F1*: 0.544 -[INFO] Current time: 1296 ms, average: 2720.81 ms/ques, entity detection: 420.34 ms/ques (linking tool: 328.51 ms/ques), relation detection: 897.06 ms/ques (similarity: 707.81 ms/ques), EDG generation: 50.07 ms/ques +[INFO] Cumulative metrics, sample: 575, P: 0.519, R: 0.571, macro F1: 0.544 +[INFO] Current question consumed time: 1081 ms, average: 2480.85 ms/ques, entity detection: 377.61 ms/ques (linking tool: 297.94 ms/ques), relation detection: 836.24 ms/ques (similarity: 695.61 ms/ques), EDG generation: 50.01 ms/ques [INFO] Entity Linking P: 0.7495652173913043, R: 0.7034782608695652, micro F1: 0.7182608695652181, macro F1: 0.7257908568157572 [INFO] Relation Linking P: 0.6015942028985507, R: 0.5478260869565217, micro F1: 0.5626666666666669, macro F1: 0.5734525499827314 [INFO] Type Linking P: 0.542608695652174, R: 0.542608695652174, micro F1: 0.542608695652174, macro F1: 0.542608695652174 @@ -8073,8 +8073,8 @@ Question 575: Name the river whose source mountain is Baikal Mountains and mouth [INFO] Golden Answer: [http://dbpedia.org/resource/Lena_River] [INFO] Predicted Answer: [http://dbpedia.org/resource/Lena_River] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 576, P: 0.519, R: 0.572, macro F1: 0.526, macro F1*: 0.544 -[INFO] Current time: 1958 ms, average: 2719.48 ms/ques, entity detection: 420.60 ms/ques (linking tool: 328.57 ms/ques), relation detection: 896.22 ms/ques (similarity: 706.99 ms/ques), EDG generation: 50.17 ms/ques +[INFO] Cumulative metrics, sample: 576, P: 0.519, R: 0.572, macro F1: 0.544 +[INFO] Current question consumed time: 1570 ms, average: 2479.27 ms/ques, entity detection: 377.70 ms/ques (linking tool: 297.91 ms/ques), relation detection: 835.43 ms/ques (similarity: 694.81 ms/ques), EDG generation: 50.09 ms/ques [INFO] Entity Linking P: 0.75, R: 0.7039930555555556, micro F1: 0.7187500000000007, macro F1: 0.726268656716418 [INFO] Relation Linking P: 0.6022858796296295, R: 0.5486111111111112, micro F1: 0.5634259259259262, macro F1: 0.5741968712898945 [INFO] Type Linking P: 0.5416666666666666, R: 0.5416666666666666, micro F1: 0.5416666666666666, macro F1: 0.5416666666666666 @@ -8087,8 +8087,8 @@ Question 576: Count the PhD students whose thesis are supervised by National Med [INFO] Golden Answer: [126] [INFO] Predicted Answer: [1] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 577, P: 0.518, R: 0.571, macro F1: 0.525, macro F1*: 0.544 -[INFO] Current time: 1375 ms, average: 2717.15 ms/ques, entity detection: 420.20 ms/ques (linking tool: 328.26 ms/ques), relation detection: 895.17 ms/ques (similarity: 706.27 ms/ques), EDG generation: 50.12 ms/ques +[INFO] Cumulative metrics, sample: 577, P: 0.518, R: 0.571, macro F1: 0.544 +[INFO] Current question consumed time: 1151 ms, average: 2476.96 ms/ques, entity detection: 377.31 ms/ques (linking tool: 297.60 ms/ques), relation detection: 834.43 ms/ques (similarity: 694.11 ms/ques), EDG generation: 50.06 ms/ques [INFO] Entity Linking P: 0.7487001733102253, R: 0.7027729636048526, micro F1: 0.7175043327556333, macro F1: 0.7250099588711554 [INFO] Relation Linking P: 0.6012420566146736, R: 0.5476603119584056, micro F1: 0.5624494511842868, macro F1: 0.5732017293985776 [INFO] Type Linking P: 0.5424610051993067, R: 0.5424610051993067, micro F1: 0.5424610051993067, macro F1: 0.5424610051993067 @@ -8101,8 +8101,8 @@ Question 577: Which broadcast area of CFRX is also the palce of birth of Anthony [INFO] Golden Answer: [http://dbpedia.org/resource/North_America] [INFO] Predicted Answer: [http://dbpedia.org/resource/North_America] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 578, P: 0.519, R: 0.572, macro F1: 0.526, macro F1*: 0.544 -[INFO] Current time: 1336 ms, average: 2714.76 ms/ques, entity detection: 419.92 ms/ques (linking tool: 327.90 ms/ques), relation detection: 894.25 ms/ques (similarity: 705.44 ms/ques), EDG generation: 50.22 ms/ques +[INFO] Cumulative metrics, sample: 578, P: 0.519, R: 0.572, macro F1: 0.544 +[INFO] Current question consumed time: 1075 ms, average: 2474.54 ms/ques, entity detection: 376.99 ms/ques (linking tool: 297.24 ms/ques), relation detection: 833.53 ms/ques (similarity: 693.29 ms/ques), EDG generation: 50.16 ms/ques [INFO] Entity Linking P: 0.7491349480968859, R: 0.7032871972318339, micro F1: 0.7179930795847758, macro F1: 0.7254874482462992 [INFO] Relation Linking P: 0.6019319492502883, R: 0.5484429065743944, micro F1: 0.5632064590542102, macro F1: 0.5739438864389264 [INFO] Type Linking P: 0.5432525951557093, R: 0.5432525951557093, micro F1: 0.5432525951557093, macro F1: 0.5432525951557093 @@ -8115,8 +8115,8 @@ Question 578: Count the total number of cast member of the television shows whos [INFO] Golden Answer: [35] [INFO] Predicted Answer: [21, 23] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 579, P: 0.518, R: 0.571, macro F1: 0.525, macro F1*: 0.543 -[INFO] Current time: 2749 ms, average: 2714.82 ms/ques, entity detection: 419.55 ms/ques (linking tool: 327.52 ms/ques), relation detection: 894.38 ms/ques (similarity: 705.73 ms/ques), EDG generation: 50.19 ms/ques +[INFO] Cumulative metrics, sample: 579, P: 0.518, R: 0.571, macro F1: 0.543 +[INFO] Current question consumed time: 2329 ms, average: 2474.29 ms/ques, entity detection: 376.59 ms/ques (linking tool: 296.87 ms/ques), relation detection: 833.58 ms/ques (similarity: 693.64 ms/ques), EDG generation: 50.10 ms/ques [INFO] Entity Linking P: 0.7495682210708118, R: 0.7037996545768567, micro F1: 0.718480138169258, macro F1: 0.72596328006264 [INFO] Relation Linking P: 0.6026194588370754, R: 0.5492227979274611, micro F1: 0.5639608520437538, macro F1: 0.574683457434032 [INFO] Type Linking P: 0.542314335060449, R: 0.542314335060449, micro F1: 0.542314335060449, macro F1: 0.542314335060449 @@ -8129,8 +8129,8 @@ Question 579: List all the awards won by the artist who has won SIGGRAPH ? [INFO] Golden Answer: [http://dbpedia.org/resource/Academy_Awards, http://dbpedia.org/resource/SIGGRAPH, http://dbpedia.org/resource/Sloan_Fellowship, http://dbpedia.org/resource/Parsons_animation_Festival] [INFO] Predicted Answer: [http://dbpedia.org/resource/Academy_Awards, http://dbpedia.org/resource/SIGGRAPH, http://dbpedia.org/resource/Sloan_Fellowship, http://dbpedia.org/resource/Parsons_animation_Festival] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 580, P: 0.519, R: 0.572, macro F1: 0.526, macro F1*: 0.544 -[INFO] Current time: 963 ms, average: 2711.80 ms/ques, entity detection: 418.93 ms/ques (linking tool: 327.03 ms/ques), relation detection: 893.67 ms/ques (similarity: 705.16 ms/ques), EDG generation: 50.15 ms/ques +[INFO] Cumulative metrics, sample: 580, P: 0.519, R: 0.572, macro F1: 0.544 +[INFO] Current question consumed time: 876 ms, average: 2471.53 ms/ques, entity detection: 376.02 ms/ques (linking tool: 296.41 ms/ques), relation detection: 832.92 ms/ques (similarity: 693.01 ms/ques), EDG generation: 50.07 ms/ques [INFO] Entity Linking P: 0.75, R: 0.7043103448275863, micro F1: 0.71896551724138, macro F1: 0.7264374629519857 [INFO] Relation Linking P: 0.6033045977011494, R: 0.55, micro F1: 0.5647126436781612, macro F1: 0.5754204559611312 [INFO] Type Linking P: 0.5431034482758621, R: 0.5431034482758621, micro F1: 0.5431034482758621, macro F1: 0.5431034482758621 @@ -8143,8 +8143,8 @@ Question 580: Where was david Scherman trained ? [INFO] Golden Answer: [http://dbpedia.org/resource/Dartmouth_College] [INFO] Predicted Answer: [http://dbpedia.org/resource/Dartmouth_College] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 581, P: 0.520, R: 0.572, macro F1: 0.527, macro F1*: 0.545 -[INFO] Current time: 682 ms, average: 2708.31 ms/ques, entity detection: 418.49 ms/ques (linking tool: 326.62 ms/ques), relation detection: 892.40 ms/ques (similarity: 704.11 ms/ques), EDG generation: 50.08 ms/ques +[INFO] Cumulative metrics, sample: 581, P: 0.520, R: 0.572, macro F1: 0.545 +[INFO] Current question consumed time: 575 ms, average: 2468.27 ms/ques, entity detection: 375.62 ms/ques (linking tool: 296.03 ms/ques), relation detection: 831.73 ms/ques (similarity: 691.98 ms/ques), EDG generation: 49.99 ms/ques [INFO] Entity Linking P: 0.7504302925989673, R: 0.7048192771084337, micro F1: 0.7194492254733226, macro F1: 0.7269100054861671 [INFO] Relation Linking P: 0.6039873780837636, R: 0.5507745266781411, micro F1: 0.5654618473895585, macro F1: 0.5761548955015904 [INFO] Type Linking P: 0.5438898450946644, R: 0.5438898450946644, micro F1: 0.5438898450946644, macro F1: 0.5438898450946644 @@ -8157,8 +8157,8 @@ Question 581: Count the battles fought by Raoul Magrin-Vernerey? [INFO] Golden Answer: [4] [INFO] Predicted Answer: [5] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 582, P: 0.519, R: 0.571, macro F1: 0.526, macro F1*: 0.544 -[INFO] Current time: 899 ms, average: 2705.20 ms/ques, entity detection: 418.06 ms/ques (linking tool: 326.19 ms/ques), relation detection: 891.32 ms/ques (similarity: 703.17 ms/ques), EDG generation: 50.02 ms/ques +[INFO] Cumulative metrics, sample: 582, P: 0.519, R: 0.571, macro F1: 0.544 +[INFO] Current question consumed time: 767 ms, average: 2465.35 ms/ques, entity detection: 375.21 ms/ques (linking tool: 295.63 ms/ques), relation detection: 830.70 ms/ques (similarity: 691.08 ms/ques), EDG generation: 49.94 ms/ques [INFO] Entity Linking P: 0.7508591065292096, R: 0.7053264604810997, micro F1: 0.7199312714776639, macro F1: 0.7273809161775587 [INFO] Relation Linking P: 0.6029495990836197, R: 0.5498281786941581, micro F1: 0.5644902634593358, macro F1: 0.5751649386364674 [INFO] Type Linking P: 0.5429553264604811, R: 0.5429553264604811, micro F1: 0.5429553264604811, macro F1: 0.5429553264604811 @@ -8171,8 +8171,8 @@ Question 582: Which athlete was the flag bearer for Switzerland in 2004 and 2008 [INFO] Golden Answer: [http://dbpedia.org/resource/Roger_Federer] [INFO] Predicted Answer: [http://dbpedia.org/resource/Fabian_Schär, http://dbpedia.org/resource/Johann_Vogel, http://dbpedia.org/resource/Scott_Sutter, http://dbpedia.org/resource/Tranquillo_Barnetta, http://dbpedia.org/resource/Alain_Rochat, http://dbpedia.org/resource/Marc_Hottiger, http://dbpedia.org/resource/Nassim_Ben_Khalifa, http://dbpedia.org/resource/Christoph_Spycher, http://dbpedia.org/resource/Fabrice_Borer, http://dbpedia.org/resource/Milaim_Rama, http://dbpedia.org/resource/Rémo_Meyer, http://dbpedia.org/resource/Stéphane_Grichting, http://dbpedia.org/resource/Sébastien_Roth, http://dbpedia.org/resource/Fabio_Coltorti, http://dbpedia.org/resource/Adolphe_Hug, http://dbpedia.org/resource/Eugen_Corrodi, http://dbpedia.org/resource/Franco_Andreoli, http://dbpedia.org/resource/Gerhard_Lusenti, http://dbpedia.org/resource/Hans_Siegenthaler, http://dbpedia.org/resource/Jean_Tamini, http://dbpedia.org/resource/Kurt_Rey, http://dbpedia.org/resource/Roger_Quinche, http://dbpedia.org/resource/Rudolf_Gyger, http://dbpedia.org/resource/Walter_Beerli, http://dbpedia.org/resource/Walter_Schneiter, http://dbpedia.org/resource/Willi_Steffen, http://dbpedia.org/resource/Felice_Soldini, http://dbpedia.org/resource/Blerim_Džemaili, http://dbpedia.org/resource/Heinz_Schneiter, http://dbpedia.org/resource/David_Degen, http://dbpedia.org/resource/Philipp_Degen, http://dbpedia.org/resource/Philippe_Senderos, http://dbpedia.org/resource/Kubilay_Türkyilmaz, http://dbpedia.org/resource/Breel_Embolo, http://dbpedia.org/resource/Sirio_Vernati, http://dbpedia.org/resource/Ricardo_Cabanas, http://dbpedia.org/resource/Yann_Sommer, http://dbpedia.org/resource/André_Muff, http://dbpedia.org/resource/Beg_Ferati, http://dbpedia.org/resource/Benjamin_Huggel, http://dbpedia.org/resource/Boris_Smiljanić, http://dbpedia.org/resource/Erni_Maissen, http://dbpedia.org/resource/Ernst-Alfred_Thalmann, http://dbpedia.org/resource/Eugen_Rupf, http://dbpedia.org/resource/Fabian_Frei, http://dbpedia.org/resource/Jörg_Stiel, http://dbpedia.org/resource/Luca_Zuffi, http://dbpedia.org/resource/Marco_Zwyssig, http://dbpedia.org/resource/Michael_Lang_(footballer), http://dbpedia.org/resource/Renato_Steffen, http://dbpedia.org/resource/René_Hasler, http://dbpedia.org/resource/Roberto_Frigerio, http://dbpedia.org/resource/Thomas_Häberli, http://dbpedia.org/resource/Traugott_Oberer, http://dbpedia.org/resource/Valentin_Stocker, http://dbpedia.org/resource/Mickaël_Facchinetti, http://dbpedia.org/resource/Ferdinando_Riva, http://dbpedia.org/resource/Georges_Aeby, http://dbpedia.org/resource/Hans-Peter_Friedländer, http://dbpedia.org/resource/Hans-Peter_Zwicker, http://dbpedia.org/resource/Werner_Leimgruber, http://dbpedia.org/resource/André_Neury, http://dbpedia.org/resource/Charles_Casali, http://dbpedia.org/resource/Oliver_Eggimann, http://dbpedia.org/resource/Willy_Kernen, http://dbpedia.org/resource/Adolf_Stelzer, http://dbpedia.org/resource/August_Lehmann, http://dbpedia.org/resource/Fritz_Wagner_(footballer), http://dbpedia.org/resource/Hermann_Springer, http://dbpedia.org/resource/Oscar_Rauch, http://dbpedia.org/resource/Paul_Aeby, http://dbpedia.org/resource/Ernest_Peterly, http://dbpedia.org/resource/Adrian_Knup, http://dbpedia.org/resource/Adrian_Winter, http://dbpedia.org/resource/Daniel_Gygax, http://dbpedia.org/resource/Josip_Drmić, http://dbpedia.org/resource/Ricardo_Rodríguez_(footballer), http://dbpedia.org/resource/Stéphane_Chapuisat, http://dbpedia.org/resource/Almen_Abdi, http://dbpedia.org/resource/Diego_Benaglio, http://dbpedia.org/resource/Albert_Bunjaku, http://dbpedia.org/resource/Stéphane_Henchoz, http://dbpedia.org/resource/Hans_Pulver, http://dbpedia.org/resource/Walter_Weiler, http://dbpedia.org/resource/Marcel_Koller, http://dbpedia.org/resource/Michel_Morganella, http://dbpedia.org/resource/Albert_Guinchard, http://dbpedia.org/resource/Florian_Berisha, http://dbpedia.org/resource/Matías_Vitkieviez, http://dbpedia.org/resource/Elsad_Zverotić, http://dbpedia.org/resource/Vincent_Rüfli, http://dbpedia.org/resource/Daniel_Unal, http://dbpedia.org/resource/Stephan_Keller, http://dbpedia.org/resource/Francesco_Di_Jorio, http://dbpedia.org/resource/Davide_Redzepi, http://dbpedia.org/resource/Walter_Müller_(footballer,_1920), http://dbpedia.org/resource/René_Deck, http://dbpedia.org/resource/Silvan_Thüler, http://dbpedia.org/resource/Izet_Hajrović, http://dbpedia.org/resource/Ludovic_Magnin, http://dbpedia.org/resource/Valon_Behrami, http://dbpedia.org/resource/Patrick_Müller, http://dbpedia.org/resource/Johan_Djourou, http://dbpedia.org/resource/Reto_Ziegler, http://dbpedia.org/resource/Stephan_Lichtsteiner, http://dbpedia.org/resource/Alain_Nef, http://dbpedia.org/resource/Alexandre_Rey, http://dbpedia.org/resource/Gaetano_Berardi, http://dbpedia.org/resource/Giuseppe_Mazzarelli, http://dbpedia.org/resource/Jonathan_Rossini, http://dbpedia.org/resource/Xherdan_Shaqiri, http://dbpedia.org/resource/Gökhan_Inler, http://dbpedia.org/resource/Marco_Streller, http://dbpedia.org/resource/Pajtim_Kasami, http://dbpedia.org/resource/Ramon_Vega, http://dbpedia.org/resource/Steve_von_Bergen, http://dbpedia.org/resource/Stefan_Frei, http://dbpedia.org/resource/Alain_Sutter, http://dbpedia.org/resource/Timm_Klose, http://dbpedia.org/resource/Eldin_Jakupović, http://dbpedia.org/resource/Innocent_Emeghara, http://dbpedia.org/resource/Christophe_Ohrel, http://dbpedia.org/resource/Dominique_Herr, http://dbpedia.org/resource/Marco_Grassi, http://dbpedia.org/resource/Marco_Pascolo, http://dbpedia.org/resource/Nestor_Subiat, http://dbpedia.org/resource/Sébastien_Fournier, http://dbpedia.org/resource/Thomas_Bickel, http://dbpedia.org/resource/Yvan_Quentin, http://dbpedia.org/resource/Mario_Eggimann, http://dbpedia.org/resource/Heinz_Barmettler, http://dbpedia.org/resource/Kurt_Stettler, http://dbpedia.org/resource/Alain_Wiss, http://dbpedia.org/resource/Marc_Zellweger, http://dbpedia.org/resource/Sokol_Maliqi, http://dbpedia.org/resource/Xavier_Margairaz, http://dbpedia.org/resource/Peter_Risi, http://dbpedia.org/resource/Beat_Sutter, http://dbpedia.org/resource/Robert_Lüthi, http://dbpedia.org/resource/Didier_Gigon, http://dbpedia.org/resource/Joël_Corminbœuf, http://dbpedia.org/resource/Johnny_Leoni, http://dbpedia.org/resource/Peter_Marti, http://dbpedia.org/resource/Philippe_Hertig, http://dbpedia.org/resource/Alexander_Frei, http://dbpedia.org/resource/Eren_Derdiyok, http://dbpedia.org/resource/Hakan_Yakin, http://dbpedia.org/resource/Andreas_Hilfiker, http://dbpedia.org/resource/Bruno_Berner, http://dbpedia.org/resource/Roman_Bürki, http://dbpedia.org/resource/Gélson_Fernandes, http://dbpedia.org/resource/Johan_Vonlanthen, http://dbpedia.org/resource/Raphaël_Wicky, http://dbpedia.org/resource/Marco_Padalino, http://dbpedia.org/resource/Fabian_Lustenberger, http://dbpedia.org/resource/François_Affolter, http://dbpedia.org/resource/François_Moubandje, http://dbpedia.org/resource/Haris_Seferović, http://dbpedia.org/resource/Lauro_Amadò, http://dbpedia.org/resource/Marwin_Hitz, http://dbpedia.org/resource/Roger_Vonlanthen, http://dbpedia.org/resource/Stefan_Huber, http://dbpedia.org/resource/Markus_Tanner, http://dbpedia.org/resource/Pirmin_Schwegler, http://dbpedia.org/resource/Vullnet_Basha, http://dbpedia.org/resource/Albert_Büche, http://dbpedia.org/resource/Arnaldo_Ortelli, http://dbpedia.org/resource/Edmond_Loichot, http://dbpedia.org/resource/Ernst_Frick_(footballer), http://dbpedia.org/resource/Erwin_Hochsträsser, http://dbpedia.org/resource/Louis_Gobet, http://dbpedia.org/resource/Otto_Bühler, http://dbpedia.org/resource/Raymond_Passello, http://dbpedia.org/resource/Renato_Bizzozero, http://dbpedia.org/resource/Willy_Huber, http://dbpedia.org/resource/Willy_von_Känel, http://dbpedia.org/resource/Gilbert_Fesselet, http://dbpedia.org/resource/Ivo_Frosio, http://dbpedia.org/resource/Marcel_Flückiger, http://dbpedia.org/resource/Marcel_Mauron, http://dbpedia.org/resource/Alberto_Regazzoni, http://dbpedia.org/resource/Eugène_Parlier, http://dbpedia.org/resource/Antonio_Esposito_(footballer,_born_1972), http://dbpedia.org/resource/Blaise_Nkufo, http://dbpedia.org/resource/Davide_Chiumiento, http://dbpedia.org/resource/Marco_Schönbächler, http://dbpedia.org/resource/René_Botteron, http://dbpedia.org/resource/Ridge_Mobulu, http://dbpedia.org/resource/Moreno_Costanzo, http://dbpedia.org/resource/Dario_Zuffi, http://dbpedia.org/resource/Marco_Wölfli, http://dbpedia.org/resource/Walter_Eich, http://dbpedia.org/resource/Mario_Gavranović, http://dbpedia.org/resource/Ernest_Molles, http://dbpedia.org/resource/Claudio_Sulser, http://dbpedia.org/resource/Christophe_Bonvin, http://dbpedia.org/resource/David_Sesa, http://dbpedia.org/resource/Sébastien_Jeanneret, http://dbpedia.org/resource/Walter_Balmer_(footballer), http://dbpedia.org/resource/Gilbert_Rey, http://dbpedia.org/resource/Heinz_Bigler, http://dbpedia.org/resource/Hansruedi_Führer, http://dbpedia.org/resource/Heinz_Bäni, http://dbpedia.org/resource/Kurt_Armbruster, http://dbpedia.org/resource/Léo_Eichmann, http://dbpedia.org/resource/Vittore_Gottardi, http://dbpedia.org/resource/Willy_Allemann, http://dbpedia.org/resource/Xavier_Stierli, http://dbpedia.org/resource/Antonio_Permunian, http://dbpedia.org/resource/Fritz_Kehl, http://dbpedia.org/resource/Fritz_Morf, http://dbpedia.org/resource/Marcel_Vonlanthen, http://dbpedia.org/resource/Erich_Burgener, http://dbpedia.org/resource/Roger_Berbig, http://dbpedia.org/resource/Granit_Xhaka, http://dbpedia.org/resource/Alfred_Bickel, http://dbpedia.org/resource/André_Abegglen, http://dbpedia.org/resource/Ernst_Lörtscher, http://dbpedia.org/resource/Josef_Hügi, http://dbpedia.org/resource/René_Bader, http://dbpedia.org/resource/Robert_Ballaman, http://dbpedia.org/resource/Willy_Jäggi, http://dbpedia.org/resource/Silvan_Widmer, http://dbpedia.org/resource/Admir_Mehmedi, http://dbpedia.org/resource/Hans_Weber, http://dbpedia.org/resource/Ciriaco_Sforza, http://dbpedia.org/resource/Anton_Allemann, http://dbpedia.org/resource/Robert_Hosp, http://dbpedia.org/resource/Georges_Stuber, http://dbpedia.org/resource/Massimo_Lombardo, http://dbpedia.org/resource/Joseph_Bossi, http://dbpedia.org/resource/Roger_Mathis_(footballer,_born_1921), http://dbpedia.org/resource/Mattias_Schnorf, http://dbpedia.org/resource/Christophe_Jaquet, http://dbpedia.org/resource/Badile_Lubamba, http://dbpedia.org/resource/Alfred_Jäck, http://dbpedia.org/resource/Rodolfo_Kappenberger, http://dbpedia.org/resource/Jean-Pierre_Maradan, http://dbpedia.org/resource/Ernst_Kaltenbach, http://dbpedia.org/resource/Heinz_Lüdi, http://dbpedia.org/resource/Serge_Trinchero, http://dbpedia.org/resource/Walter_Seiler, http://dbpedia.org/resource/Law_Adam, http://dbpedia.org/resource/Hossein_Ali_Khan-Sardar] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 583, P: 0.518, R: 0.570, macro F1: 0.525, macro F1*: 0.543 -[INFO] Current time: 23675 ms, average: 2741.17 ms/ques, entity detection: 425.32 ms/ques (linking tool: 333.11 ms/ques), relation detection: 903.38 ms/ques (similarity: 714.53 ms/ques), EDG generation: 50.05 ms/ques +[INFO] Cumulative metrics, sample: 583, P: 0.518, R: 0.570, macro F1: 0.543 +[INFO] Current question consumed time: 20859 ms, average: 2496.90 ms/ques, entity detection: 381.61 ms/ques (linking tool: 301.77 ms/ques), relation detection: 841.42 ms/ques (similarity: 702.53 ms/ques), EDG generation: 49.96 ms/ques [INFO] Entity Linking P: 0.7495711835334476, R: 0.7041166380789022, micro F1: 0.7186963979416816, macro F1: 0.7261332645203074 [INFO] Relation Linking P: 0.601915380217267, R: 0.548885077186964, micro F1: 0.5635220125786166, macro F1: 0.5741783778497839 [INFO] Type Linking P: 0.5420240137221269, R: 0.5420240137221269, micro F1: 0.5420240137221269, macro F1: 0.5420240137221269 @@ -8185,8 +8185,8 @@ Question 583: Which nationalist group was responsible for foundation of Young Lo [INFO] Golden Answer: [http://dbpedia.org/resource/Puerto_Rico] [INFO] Predicted Answer: [http://dbpedia.org/resource/Jose_Cha_Cha_Jimenez] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 584, P: 0.517, R: 0.570, macro F1: 0.524, macro F1*: 0.542 -[INFO] Current time: 1078 ms, average: 2738.32 ms/ques, entity detection: 425.09 ms/ques (linking tool: 332.80 ms/ques), relation detection: 902.04 ms/ques (similarity: 713.43 ms/ques), EDG generation: 49.99 ms/ques +[INFO] Cumulative metrics, sample: 584, P: 0.517, R: 0.570, macro F1: 0.542 +[INFO] Current question consumed time: 831 ms, average: 2494.04 ms/ques, entity detection: 381.32 ms/ques (linking tool: 301.45 ms/ques), relation detection: 840.18 ms/ques (similarity: 701.47 ms/ques), EDG generation: 49.90 ms/ques [INFO] Entity Linking P: 0.75, R: 0.7046232876712328, micro F1: 0.7191780821917815, macro F1: 0.7266038846380224 [INFO] Relation Linking P: 0.600884703196347, R: 0.547945205479452, micro F1: 0.5625570776255711, macro F1: 0.5731951956959316 [INFO] Type Linking P: 0.5428082191780822, R: 0.5428082191780822, micro F1: 0.5428082191780822, macro F1: 0.5428082191780822 @@ -8199,8 +8199,8 @@ Question 584: What award was won by the father of Angelina Jolie? [INFO] Golden Answer: [http://dbpedia.org/resource/Academy_Awards] [INFO] Predicted Answer: [http://dbpedia.org/resource/Academy_Awards] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 585, P: 0.518, R: 0.570, macro F1: 0.525, macro F1*: 0.543 -[INFO] Current time: 2149 ms, average: 2737.31 ms/ques, entity detection: 424.75 ms/ques (linking tool: 332.47 ms/ques), relation detection: 901.96 ms/ques (similarity: 713.38 ms/ques), EDG generation: 50.06 ms/ques +[INFO] Cumulative metrics, sample: 585, P: 0.518, R: 0.570, macro F1: 0.543 +[INFO] Current question consumed time: 1842 ms, average: 2492.93 ms/ques, entity detection: 380.93 ms/ques (linking tool: 301.10 ms/ques), relation detection: 840.05 ms/ques (similarity: 701.43 ms/ques), EDG generation: 49.98 ms/ques [INFO] Entity Linking P: 0.7504273504273504, R: 0.7051282051282052, micro F1: 0.7196581196581203, macro F1: 0.7270728879654305 [INFO] Relation Linking P: 0.6007122507122507, R: 0.5478632478632479, micro F1: 0.5624501424501427, macro F1: 0.5730718879422836 [INFO] Type Linking P: 0.5418803418803418, R: 0.5418803418803418, micro F1: 0.5418803418803418, macro F1: 0.5418803418803418 @@ -8213,8 +8213,8 @@ Question 585: What are the software whose programming language are Multilingual? [INFO] Golden Answer: [http://dbpedia.org/resource/Zinf, http://dbpedia.org/resource/GIMP, http://dbpedia.org/resource/Leafpad, http://dbpedia.org/resource/Gnash, http://dbpedia.org/resource/HomeBank, http://dbpedia.org/resource/GKrellM, http://dbpedia.org/resource/J-Pilot, http://dbpedia.org/resource/ROX_Desktop, http://dbpedia.org/resource/Packetsquare, http://dbpedia.org/resource/VIPS_(software), http://dbpedia.org/resource/Sugar_(software)] [INFO] Predicted Answer: [] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 586, P: 0.517, R: 0.569, macro F1: 0.524, macro F1*: 0.542 -[INFO] Current time: 289 ms, average: 2733.13 ms/ques, entity detection: 424.03 ms/ques (linking tool: 331.90 ms/ques), relation detection: 900.74 ms/ques (similarity: 712.38 ms/ques), EDG generation: 50.00 ms/ques +[INFO] Cumulative metrics, sample: 586, P: 0.517, R: 0.569, macro F1: 0.542 +[INFO] Current question consumed time: 265 ms, average: 2489.13 ms/ques, entity detection: 380.28 ms/ques (linking tool: 300.59 ms/ques), relation detection: 838.94 ms/ques (similarity: 700.47 ms/ques), EDG generation: 49.92 ms/ques [INFO] Entity Linking P: 0.7491467576791809, R: 0.7039249146757679, micro F1: 0.7184300341296935, macro F1: 0.725832149248766 [INFO] Relation Linking P: 0.5996871444823663, R: 0.5469283276450512, micro F1: 0.5614903299203643, macro F1: 0.5720939495669555 [INFO] Type Linking P: 0.5409556313993175, R: 0.5409556313993175, micro F1: 0.5409556313993175, macro F1: 0.5409556313993175 @@ -8227,8 +8227,8 @@ Question 586: Was Marshall Arisman the cover artist in american psycho? [INFO] Golden Answer: [true] [INFO] Predicted Answer: [true] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 587, P: 0.518, R: 0.570, macro F1: 0.525, macro F1*: 0.543 -[INFO] Current time: 1551 ms, average: 2731.12 ms/ques, entity detection: 423.97 ms/ques (linking tool: 331.74 ms/ques), relation detection: 900.31 ms/ques (similarity: 711.80 ms/ques), EDG generation: 49.94 ms/ques +[INFO] Cumulative metrics, sample: 587, P: 0.518, R: 0.570, macro F1: 0.543 +[INFO] Current question consumed time: 1358 ms, average: 2487.20 ms/ques, entity detection: 380.22 ms/ques (linking tool: 300.45 ms/ques), relation detection: 838.48 ms/ques (similarity: 699.91 ms/ques), EDG generation: 49.86 ms/ques [INFO] Entity Linking P: 0.7495741056218058, R: 0.7044293015332198, micro F1: 0.7189097103918235, macro F1: 0.7263008615691078 [INFO] Relation Linking P: 0.6003691084611016, R: 0.5477001703577513, micro F1: 0.5622373651334471, macro F1: 0.5728265167411722 [INFO] Type Linking P: 0.5417376490630323, R: 0.5417376490630323, micro F1: 0.5417376490630323, macro F1: 0.5417376490630323 @@ -8241,8 +8241,8 @@ Question 587: Which birthplace of Liliya Lobanova is also the location of the Sn [INFO] Golden Answer: [http://dbpedia.org/resource/Ukraine] [INFO] Predicted Answer: [http://dbpedia.org/resource/Black_Sea] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 588, P: 0.517, R: 0.569, macro F1: 0.524, macro F1*: 0.542 -[INFO] Current time: 2132 ms, average: 2730.10 ms/ques, entity detection: 423.94 ms/ques (linking tool: 331.47 ms/ques), relation detection: 899.99 ms/ques (similarity: 711.32 ms/ques), EDG generation: 50.03 ms/ques +[INFO] Cumulative metrics, sample: 588, P: 0.517, R: 0.569, macro F1: 0.542 +[INFO] Current question consumed time: 1813 ms, average: 2486.05 ms/ques, entity detection: 380.13 ms/ques (linking tool: 300.18 ms/ques), relation detection: 838.18 ms/ques (similarity: 699.46 ms/ques), EDG generation: 49.97 ms/ques [INFO] Entity Linking P: 0.75, R: 0.7040816326530612, micro F1: 0.7188208616780053, macro F1: 0.7263157894736842 [INFO] Relation Linking P: 0.5993480725623582, R: 0.5467687074829932, micro F1: 0.5612811791383222, macro F1: 0.5718523219848097 [INFO] Type Linking P: 0.5425170068027211, R: 0.5425170068027211, micro F1: 0.5425170068027211, macro F1: 0.5425170068027211 @@ -8255,8 +8255,8 @@ Question 588: Count all the stateless people [INFO] Golden Answer: [9] [INFO] Predicted Answer: [1] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 589, P: 0.516, R: 0.568, macro F1: 0.523, macro F1*: 0.541 -[INFO] Current time: 976 ms, average: 2727.12 ms/ques, entity detection: 423.40 ms/ques (linking tool: 330.99 ms/ques), relation detection: 899.15 ms/ques (similarity: 710.50 ms/ques), EDG generation: 49.96 ms/ques +[INFO] Cumulative metrics, sample: 589, P: 0.516, R: 0.568, macro F1: 0.541 +[INFO] Current question consumed time: 791 ms, average: 2483.17 ms/ques, entity detection: 379.60 ms/ques (linking tool: 299.74 ms/ques), relation detection: 837.35 ms/ques (similarity: 698.66 ms/ques), EDG generation: 49.89 ms/ques [INFO] Entity Linking P: 0.7487266553480475, R: 0.7028862478777589, micro F1: 0.7176004527447658, macro F1: 0.7250826557054775 [INFO] Relation Linking P: 0.5983305036785511, R: 0.5458404074702886, micro F1: 0.5603282399547258, macro F1: 0.5708814351902683 [INFO] Type Linking P: 0.5432937181663837, R: 0.5432937181663837, micro F1: 0.5432937181663837, macro F1: 0.5432937181663837 @@ -8269,8 +8269,8 @@ Question 589: Josef Bhler belongs to which political party? [INFO] Golden Answer: [http://dbpedia.org/resource/Nazism] [INFO] Predicted Answer: [http://dbpedia.org/resource/Alliance_for_the_Future_of_Austria] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 590, P: 0.515, R: 0.567, macro F1: 0.522, macro F1*: 0.540 -[INFO] Current time: 1036 ms, average: 2724.26 ms/ques, entity detection: 422.98 ms/ques (linking tool: 330.58 ms/ques), relation detection: 898.11 ms/ques (similarity: 709.80 ms/ques), EDG generation: 49.89 ms/ques +[INFO] Cumulative metrics, sample: 590, P: 0.515, R: 0.567, macro F1: 0.540 +[INFO] Current question consumed time: 900 ms, average: 2480.49 ms/ques, entity detection: 379.20 ms/ques (linking tool: 299.34 ms/ques), relation detection: 836.43 ms/ques (similarity: 697.97 ms/ques), EDG generation: 49.81 ms/ques [INFO] Entity Linking P: 0.747457627118644, R: 0.7016949152542373, micro F1: 0.7163841807909611, macro F1: 0.7238537020517394 [INFO] Relation Linking P: 0.5973163841807909, R: 0.5449152542372881, micro F1: 0.5593785310734466, macro F1: 0.5699138395374036 [INFO] Type Linking P: 0.5423728813559322, R: 0.5423728813559322, micro F1: 0.5423728813559322, macro F1: 0.5423728813559322 @@ -8283,8 +8283,8 @@ Question 590: Where did the war take place where one of the commander was Zuo Zo [INFO] Golden Answer: [http://dbpedia.org/resource/Annam_(French_protectorate), http://dbpedia.org/resource/Tonkin] [INFO] Predicted Answer: [http://dbpedia.org/resource/Yueyang, http://dbpedia.org/resource/Hunan] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 591, P: 0.515, R: 0.566, macro F1: 0.521, macro F1*: 0.539 -[INFO] Current time: 1707 ms, average: 2722.53 ms/ques, entity detection: 422.58 ms/ques (linking tool: 330.20 ms/ques), relation detection: 897.66 ms/ques (similarity: 709.53 ms/ques), EDG generation: 49.86 ms/ques +[INFO] Cumulative metrics, sample: 591, P: 0.515, R: 0.566, macro F1: 0.539 +[INFO] Current question consumed time: 1466 ms, average: 2478.77 ms/ques, entity detection: 378.80 ms/ques (linking tool: 299.00 ms/ques), relation detection: 835.95 ms/ques (similarity: 697.75 ms/ques), EDG generation: 49.81 ms/ques [INFO] Entity Linking P: 0.7478849407783418, R: 0.7021996615905245, micro F1: 0.7168640721940222, macro F1: 0.7243226380933765 [INFO] Relation Linking P: 0.5963056965595036, R: 0.5439932318104906, micro F1: 0.558432036097011, macro F1: 0.5689495183199121 [INFO] Type Linking P: 0.5414551607445008, R: 0.5414551607445008, micro F1: 0.5414551607445008, macro F1: 0.5414551607445008 @@ -8297,8 +8297,8 @@ Question 591: In which place is the company which is known for Barbara bestor lo [INFO] Golden Answer: [http://dbpedia.org/resource/Chicago] [INFO] Predicted Answer: [http://dbpedia.org/resource/Beats_Electronics, http://dbpedia.org/resource/Intelligentsia_Coffee_&_Tea] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 592, P: 0.514, R: 0.565, macro F1: 0.520, macro F1*: 0.538 -[INFO] Current time: 1031 ms, average: 2719.68 ms/ques, entity detection: 422.16 ms/ques (linking tool: 329.79 ms/ques), relation detection: 896.62 ms/ques (similarity: 708.85 ms/ques), EDG generation: 49.82 ms/ques +[INFO] Cumulative metrics, sample: 592, P: 0.514, R: 0.565, macro F1: 0.538 +[INFO] Current question consumed time: 860 ms, average: 2476.04 ms/ques, entity detection: 378.38 ms/ques (linking tool: 298.61 ms/ques), relation detection: 834.99 ms/ques (similarity: 697.03 ms/ques), EDG generation: 49.75 ms/ques [INFO] Entity Linking P: 0.7483108108108109, R: 0.7027027027027027, micro F1: 0.7173423423423431, macro F1: 0.7247899820658843 [INFO] Relation Linking P: 0.5969876126126126, R: 0.543918918918919, micro F1: 0.5586148648648651, macro F1: 0.5692190339630193 [INFO] Type Linking P: 0.5405405405405406, R: 0.5405405405405406, micro F1: 0.5405405405405406, macro F1: 0.5405405405405406 @@ -8311,8 +8311,8 @@ Question 592: What novels are belong to the genre of Utopian and dystopian ficti [INFO] Golden Answer: [http://dbpedia.org/resource/We_(novel), http://dbpedia.org/resource/Equality_(novel), http://dbpedia.org/resource/Nontraditional_Love, http://dbpedia.org/resource/Delirium_(Lauren_Oliver_novel), http://dbpedia.org/resource/In_the_Country_of_Last_Things, http://dbpedia.org/resource/Save_the_Pearls:_Revealing_Eden, http://dbpedia.org/resource/Moving_the_Mountain_(novel), http://dbpedia.org/resource/With_Her_in_Ourland, http://dbpedia.org/resource/A_History_of_the_Future_(Kunstler_novel), http://dbpedia.org/resource/Winter_(novel), http://dbpedia.org/resource/Wasobyoe, http://dbpedia.org/resource/A_Clockwork_Orange, http://dbpedia.org/resource/Aristopia, http://dbpedia.org/resource/Brave_New_World, http://dbpedia.org/resource/Island_(Huxley_novel), http://dbpedia.org/resource/Nineteen_Eighty-Four, http://dbpedia.org/resource/The_Trial, http://dbpedia.org/resource/The_Castle_(novel), http://dbpedia.org/resource/A_Prophetic_Romance, http://dbpedia.org/resource/Next_(novel), http://dbpedia.org/resource/The_Martian_Chronicles, http://dbpedia.org/resource/The_Three_Stigmata_of_Palmer_Eldritch, http://dbpedia.org/resource/Timeline_of_science_fiction, http://dbpedia.org/resource/Matched, http://dbpedia.org/resource/The_Girl_with_All_the_Gifts, http://dbpedia.org/resource/World_Made_By_Hand, http://dbpedia.org/resource/The_Hunger_Games_(novel), http://dbpedia.org/resource/The_Witch_of_Hebron, http://dbpedia.org/resource/The_Prize_of_Peril, http://dbpedia.org/resource/Arqtiq, http://dbpedia.org/resource/Ionia_(novel), http://dbpedia.org/resource/Lord_of_the_World, http://dbpedia.org/resource/One_(David_Karp_novel), http://dbpedia.org/resource/The_World_a_Department_Store, http://dbpedia.org/resource/Pandemonium_(novel), http://dbpedia.org/resource/Catching_Fire, http://dbpedia.org/resource/Going,_Going,_Gone_(novel), http://dbpedia.org/resource/Looking_Backward, http://dbpedia.org/resource/Oryx_and_Crake, http://dbpedia.org/resource/Riddley_Walker, http://dbpedia.org/resource/Battle_Royale, http://dbpedia.org/resource/Kallocain, http://dbpedia.org/resource/Through_the_Eye_of_the_Needle, http://dbpedia.org/resource/334_(novel), http://dbpedia.org/resource/When_Smuts_Goes, http://dbpedia.org/resource/Acidity_(novelette), http://dbpedia.org/resource/Three_Hundred_Years_Hence, http://dbpedia.org/resource/Intermere, http://dbpedia.org/resource/Ambient_(novel), http://dbpedia.org/resource/New_Atlantis, http://dbpedia.org/resource/The_Scarlet_Empire, http://dbpedia.org/resource/The_Children_of_Men, http://dbpedia.org/resource/Truancy_(novel), http://dbpedia.org/resource/Equality;_or,_A_History_of_Lithconia, http://dbpedia.org/resource/2030_(novel), http://dbpedia.org/resource/State_of_Fear, http://dbpedia.org/resource/Virtual_Light, http://dbpedia.org/resource/The_Passage_(novel), http://dbpedia.org/resource/The_Twelve_(novel), http://dbpedia.org/resource/The_City_of_the_Sun, http://dbpedia.org/resource/Divergent_(novel), http://dbpedia.org/resource/Insurgent_(novel), http://dbpedia.org/resource/Kazohinia, http://dbpedia.org/resource/Legend_(Lu_novel), http://dbpedia.org/resource/Don't_Bite_the_Sun, http://dbpedia.org/resource/Memoirs_Found_in_a_Bathtub, http://dbpedia.org/resource/Cress_(novel), http://dbpedia.org/resource/Liminal_States, http://dbpedia.org/resource/1985_(Anthony_Burgess_novel), http://dbpedia.org/resource/Never_Let_Me_Go_(novel), http://dbpedia.org/resource/Walden_Two, http://dbpedia.org/resource/Fahrenheit_451, http://dbpedia.org/resource/Herland_(novel), http://dbpedia.org/resource/The_Handmaid's_Tale, http://dbpedia.org/resource/Woman_on_the_Edge_of_Time, http://dbpedia.org/resource/A_Crystal_Age, http://dbpedia.org/resource/The_House_of_the_Scorpion, http://dbpedia.org/resource/The_Iron_Heel, http://dbpedia.org/resource/Bend_Sinister_(novel), http://dbpedia.org/resource/The_Giver, http://dbpedia.org/resource/Allegiant_(novel), http://dbpedia.org/resource/The_Black_Coat, http://dbpedia.org/resource/The_Death_Cure, http://dbpedia.org/resource/Article_5_(novel), http://dbpedia.org/resource/Prodigy_(Marie_Lu_novel), http://dbpedia.org/resource/Shatter_Me, http://dbpedia.org/resource/Terraplane_(novel), http://dbpedia.org/resource/The_Last_Human, http://dbpedia.org/resource/Under_the_Never_Sky, http://dbpedia.org/resource/Ultraviolet_(novel), http://dbpedia.org/resource/Elvissey, http://dbpedia.org/resource/The_Foundation_Pit, http://dbpedia.org/resource/2894_(novel), http://dbpedia.org/resource/A_Traveler_from_Altruria, http://dbpedia.org/resource/Young_West, http://dbpedia.org/resource/Random_Acts_of_Senseless_Violence, http://dbpedia.org/resource/Crossed_(novel), http://dbpedia.org/resource/Reached_(novel), http://dbpedia.org/resource/Scarlet_(novel), http://dbpedia.org/resource/The_Begum's_Fortune, http://dbpedia.org/resource/Ecotopia, http://dbpedia.org/resource/Millennial_Women, http://dbpedia.org/resource/The_Eye_of_the_Heron, http://dbpedia.org/resource/The_Old_New_Land, http://dbpedia.org/resource/The_Guardians_(novel), http://dbpedia.org/resource/The_Wanting_Seed, http://dbpedia.org/resource/Blind_Faith_(novel), http://dbpedia.org/resource/Stark_(novel), http://dbpedia.org/resource/New_Amazonia, http://dbpedia.org/resource/Unwind_(novel), http://dbpedia.org/resource/Sub-Coelum, http://dbpedia.org/resource/The_Diothas, http://dbpedia.org/resource/The_New_Paul_and_Virginia, http://dbpedia.org/resource/Across_the_Universe_(novel), http://dbpedia.org/resource/A_Planet_for_the_President, http://dbpedia.org/resource/Turning_on_the_Girls, http://dbpedia.org/resource/Meccania, http://dbpedia.org/resource/Neverwas_(novel), http://dbpedia.org/resource/Free_Four:_Tobias_Tells_the_Divergent_Knife-Throwing_Scene, http://dbpedia.org/resource/Masha,_or_the_Fourth_Reich, http://dbpedia.org/resource/Notes_from_the_Internet_Apocalypse] [INFO] Predicted Answer: [http://dbpedia.org/resource/The_Trial] [INFO] P: 1.0, R: 0.008264462809917356, F: 0.01639344262295082 -[INFO] Cumulative metrics, sample: 593, P: 0.515, R: 0.564, macro F1: 0.519, macro F1*: 0.538 -[INFO] Current time: 3042 ms, average: 2720.22 ms/ques, entity detection: 422.06 ms/ques (linking tool: 329.67 ms/ques), relation detection: 897.06 ms/ques (similarity: 708.67 ms/ques), EDG generation: 49.83 ms/ques +[INFO] Cumulative metrics, sample: 593, P: 0.515, R: 0.564, macro F1: 0.538 +[INFO] Current question consumed time: 2595 ms, average: 2476.24 ms/ques, entity detection: 378.27 ms/ques (linking tool: 298.49 ms/ques), relation detection: 835.34 ms/ques (similarity: 696.87 ms/ques), EDG generation: 49.75 ms/ques [INFO] Entity Linking P: 0.7478920741989882, R: 0.7032040472175379, micro F1: 0.7172568858909507, macro F1: 0.7248599533782175 [INFO] Relation Linking P: 0.5959808881394041, R: 0.5430016863406408, micro F1: 0.5576728499156832, macro F1: 0.5682591367725252 [INFO] Type Linking P: 0.5413153456998314, R: 0.5413153456998314, micro F1: 0.5413153456998314, macro F1: 0.5413153456998314 @@ -8325,8 +8325,8 @@ Question 593: What are the movies written by Nick Castle? [INFO] Golden Answer: [http://dbpedia.org/resource/Escape_from_New_York, http://dbpedia.org/resource/August_Rush, http://dbpedia.org/resource/Tap_(film), http://dbpedia.org/resource/Tag:_The_Assassination_Game, http://dbpedia.org/resource/The_Boy_Who_Could_Fly, http://dbpedia.org/resource/Skatetown,_U.S.A., http://dbpedia.org/resource/The_Resurrection_of_Broncho_Billy] [INFO] Predicted Answer: [http://dbpedia.org/resource/Escape_from_New_York, http://dbpedia.org/resource/August_Rush, http://dbpedia.org/resource/Tap_(film), http://dbpedia.org/resource/Tag:_The_Assassination_Game, http://dbpedia.org/resource/The_Boy_Who_Could_Fly, http://dbpedia.org/resource/Skatetown,_U.S.A., http://dbpedia.org/resource/The_Resurrection_of_Broncho_Billy, http://dbpedia.org/resource/Shangri-La_Plaza_(TV_pilot)] [INFO] P: 0.875, R: 1.0, F: 0.9333333333333333 -[INFO] Cumulative metrics, sample: 594, P: 0.515, R: 0.565, macro F1: 0.520, macro F1*: 0.539 -[INFO] Current time: 1088 ms, average: 2717.47 ms/ques, entity detection: 421.88 ms/ques (linking tool: 329.49 ms/ques), relation detection: 895.88 ms/ques (similarity: 707.66 ms/ques), EDG generation: 49.76 ms/ques +[INFO] Cumulative metrics, sample: 594, P: 0.515, R: 0.565, macro F1: 0.539 +[INFO] Current question consumed time: 888 ms, average: 2473.57 ms/ques, entity detection: 378.07 ms/ques (linking tool: 298.31 ms/ques), relation detection: 834.22 ms/ques (similarity: 695.88 ms/ques), EDG generation: 49.68 ms/ques [INFO] Entity Linking P: 0.7483164983164983, R: 0.7037037037037037, micro F1: 0.7177328843995519, macro F1: 0.72532474503489 [INFO] Relation Linking P: 0.5966610549943883, R: 0.5437710437710438, micro F1: 0.5584175084175087, macro F1: 0.5689896049103824 [INFO] Type Linking P: 0.5404040404040404, R: 0.5404040404040404, micro F1: 0.5404040404040404, macro F1: 0.5404040404040404 @@ -8339,8 +8339,8 @@ Question 594: On which river are there things belonging to zambezi river authori [INFO] Golden Answer: [http://dbpedia.org/resource/Zambezi] [INFO] Predicted Answer: [http://dbpedia.org/resource/Kariba_Dam] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 595, P: 0.514, R: 0.564, macro F1: 0.519, macro F1*: 0.538 -[INFO] Current time: 1090 ms, average: 2714.74 ms/ques, entity detection: 421.60 ms/ques (linking tool: 329.13 ms/ques), relation detection: 894.66 ms/ques (similarity: 706.64 ms/ques), EDG generation: 49.70 ms/ques +[INFO] Cumulative metrics, sample: 595, P: 0.514, R: 0.564, macro F1: 0.538 +[INFO] Current question consumed time: 836 ms, average: 2470.82 ms/ques, entity detection: 377.77 ms/ques (linking tool: 297.97 ms/ques), relation detection: 833.04 ms/ques (similarity: 694.87 ms/ques), EDG generation: 49.62 ms/ques [INFO] Entity Linking P: 0.7487394957983193, R: 0.704201680672269, micro F1: 0.718207282913166, macro F1: 0.7257879670086658 [INFO] Relation Linking P: 0.5973389355742297, R: 0.5436974789915966, micro F1: 0.5585994397759106, macro F1: 0.5692573334722376 [INFO] Type Linking P: 0.5411764705882353, R: 0.5411764705882353, micro F1: 0.5411764705882353, macro F1: 0.5411764705882353 @@ -8353,8 +8353,8 @@ Question 595: Is Egyptian Arabic belong to the Central Semitic Languages family? [INFO] Golden Answer: [true] [INFO] Predicted Answer: [true] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 596, P: 0.515, R: 0.565, macro F1: 0.520, macro F1*: 0.539 -[INFO] Current time: 2095 ms, average: 2713.70 ms/ques, entity detection: 422.24 ms/ques (linking tool: 329.48 ms/ques), relation detection: 893.60 ms/ques (similarity: 705.67 ms/ques), EDG generation: 49.64 ms/ques +[INFO] Cumulative metrics, sample: 596, P: 0.515, R: 0.565, macro F1: 0.539 +[INFO] Current question consumed time: 1699 ms, average: 2469.52 ms/ques, entity detection: 378.23 ms/ques (linking tool: 298.24 ms/ques), relation detection: 832.02 ms/ques (similarity: 693.93 ms/ques), EDG generation: 49.56 ms/ques [INFO] Entity Linking P: 0.7483221476510067, R: 0.7038590604026845, micro F1: 0.7178411633109627, macro F1: 0.7254099155161118 [INFO] Relation Linking P: 0.5963366890380313, R: 0.5427852348993288, micro F1: 0.5576621923937363, macro F1: 0.5683022037180894 [INFO] Type Linking P: 0.5419463087248322, R: 0.5419463087248322, micro F1: 0.5419463087248322, macro F1: 0.5419463087248322 @@ -8367,8 +8367,8 @@ Question 596: Who is the artist of Sings Kristofferson and also performed Milk C [INFO] Golden Answer: [http://dbpedia.org/resource/Willie_Nelson] [INFO] Predicted Answer: [http://dbpedia.org/resource/Willie_Nelson] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 597, P: 0.516, R: 0.565, macro F1: 0.521, macro F1*: 0.540 -[INFO] Current time: 2858 ms, average: 2713.94 ms/ques, entity detection: 422.23 ms/ques (linking tool: 329.41 ms/ques), relation detection: 892.92 ms/ques (similarity: 704.99 ms/ques), EDG generation: 49.86 ms/ques +[INFO] Cumulative metrics, sample: 597, P: 0.516, R: 0.565, macro F1: 0.540 +[INFO] Current question consumed time: 2274 ms, average: 2469.19 ms/ques, entity detection: 378.17 ms/ques (linking tool: 298.14 ms/ques), relation detection: 831.36 ms/ques (similarity: 693.27 ms/ques), EDG generation: 49.57 ms/ques [INFO] Entity Linking P: 0.7487437185929648, R: 0.7043551088777219, micro F1: 0.7183137911781136, macro F1: 0.7258714320883958 [INFO] Relation Linking P: 0.5970128419877163, R: 0.5435510887772195, micro F1: 0.5584031267448356, macro F1: 0.5690290066577328 [INFO] Type Linking P: 0.542713567839196, R: 0.542713567839196, micro F1: 0.542713567839196, macro F1: 0.542713567839196 @@ -8382,8 +8382,8 @@ Question 597: Who has fought in wars under commander Arthur St Clair? * British North America@en] [INFO] Predicted Answer: [http://dbpedia.org/resource/World_War_I] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 598, P: 0.515, R: 0.565, macro F1: 0.520, macro F1*: 0.539 -[INFO] Current time: 799 ms, average: 2710.74 ms/ques, entity detection: 421.75 ms/ques (linking tool: 329.02 ms/ques), relation detection: 891.77 ms/ques (similarity: 704.03 ms/ques), EDG generation: 49.79 ms/ques +[INFO] Cumulative metrics, sample: 598, P: 0.515, R: 0.565, macro F1: 0.539 +[INFO] Current question consumed time: 676 ms, average: 2466.19 ms/ques, entity detection: 377.72 ms/ques (linking tool: 297.77 ms/ques), relation detection: 830.28 ms/ques (similarity: 692.31 ms/ques), EDG generation: 49.51 ms/ques [INFO] Entity Linking P: 0.7474916387959866, R: 0.7031772575250836, micro F1: 0.7171125975473809, macro F1: 0.7246576002621612 [INFO] Relation Linking P: 0.5960144927536232, R: 0.5426421404682275, micro F1: 0.5574693422519512, macro F1: 0.5680774531348938 [INFO] Type Linking P: 0.5434782608695652, R: 0.5434782608695652, micro F1: 0.5434782608695652, macro F1: 0.5434782608695652 @@ -8396,8 +8396,8 @@ Question 598: Which Paramount TV show was produced by Glen and Les Charles ? [INFO] Golden Answer: [http://dbpedia.org/resource/The_Tortellis, http://dbpedia.org/resource/Cheers, http://dbpedia.org/resource/All_is_Forgiven] [INFO] Predicted Answer: [http://dbpedia.org/resource/The_Tortellis, http://dbpedia.org/resource/Cheers, http://dbpedia.org/resource/All_is_Forgiven] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 599, P: 0.516, R: 0.565, macro F1: 0.521, macro F1*: 0.539 -[INFO] Current time: 3344 ms, average: 2711.80 ms/ques, entity detection: 421.86 ms/ques (linking tool: 329.19 ms/ques), relation detection: 892.44 ms/ques (similarity: 703.92 ms/ques), EDG generation: 49.73 ms/ques +[INFO] Cumulative metrics, sample: 599, P: 0.516, R: 0.565, macro F1: 0.539 +[INFO] Current question consumed time: 2820 ms, average: 2466.78 ms/ques, entity detection: 377.76 ms/ques (linking tool: 297.87 ms/ques), relation detection: 830.82 ms/ques (similarity: 692.22 ms/ques), EDG generation: 49.45 ms/ques [INFO] Entity Linking P: 0.7479131886477463, R: 0.7028380634390651, micro F1: 0.7170283806343914, macro F1: 0.7246753795643295 [INFO] Relation Linking P: 0.5966889259877574, R: 0.5425709515859767, micro F1: 0.5576516416249307, macro F1: 0.56834456254787 [INFO] Type Linking P: 0.5425709515859767, R: 0.5425709515859767, micro F1: 0.5425709515859767, macro F1: 0.5425709515859767 @@ -8410,8 +8410,8 @@ Question 599: Name some basketball players who have played for chicago bulls? [INFO] Golden Answer: [http://dbpedia.org/resource/Devin_Brown, http://dbpedia.org/resource/John_Salmons, http://dbpedia.org/resource/Larry_Kenon, http://dbpedia.org/resource/Trent_Tucker, http://dbpedia.org/resource/Ben_Poquette, http://dbpedia.org/resource/Bob_Boozer, http://dbpedia.org/resource/Dennis_Rodman, http://dbpedia.org/resource/Elton_Brand, http://dbpedia.org/resource/Howard_Porter_(basketball), http://dbpedia.org/resource/Keith_Bogans, http://dbpedia.org/resource/Marco_Belinelli, http://dbpedia.org/resource/Nate_Robinson, http://dbpedia.org/resource/Orlando_Woolridge, http://dbpedia.org/resource/Randy_Brown, http://dbpedia.org/resource/Rasual_Butler, http://dbpedia.org/resource/Billy_McKinney_(basketball), http://dbpedia.org/resource/David_Wood_(basketball), http://dbpedia.org/resource/Jerome_Williams_(basketball), http://dbpedia.org/resource/Paul_Shirley, http://dbpedia.org/resource/Chris_Engler, http://dbpedia.org/resource/Ronald_Dupree, http://dbpedia.org/resource/Steve_Colter, http://dbpedia.org/resource/Barry_Clemens, http://dbpedia.org/resource/Mark_Landsberger, http://dbpedia.org/resource/Jarvis_Varnado, http://dbpedia.org/resource/Frank_Williams_(basketball), http://dbpedia.org/resource/Scott_Lloyd, http://dbpedia.org/resource/Jim_Ard, http://dbpedia.org/resource/Johnny_Baum, http://dbpedia.org/resource/Wes_Matthews, http://dbpedia.org/resource/Rusty_LaRue, http://dbpedia.org/resource/Chris_Jefferies, http://dbpedia.org/resource/Keith_Starr, http://dbpedia.org/resource/Bison_Dele, http://dbpedia.org/resource/Brad_Miller_(basketball), http://dbpedia.org/resource/Dickey_Simpkins, http://dbpedia.org/resource/John_Salley, http://dbpedia.org/resource/Norm_Van_Lier, http://dbpedia.org/resource/Andre_Barrett, http://dbpedia.org/resource/Darius_Songaila, http://dbpedia.org/resource/E'Twaun_Moore, http://dbpedia.org/resource/Jimmer_Fredette, http://dbpedia.org/resource/Joakim_Noah, http://dbpedia.org/resource/Lari_Ketner, http://dbpedia.org/resource/Matt_Guokas, http://dbpedia.org/resource/Richard_Hamilton_(basketball), http://dbpedia.org/resource/Tim_Thomas_(basketball), http://dbpedia.org/resource/Brian_Scalabrine, http://dbpedia.org/resource/Chris_Carr_(basketball), http://dbpedia.org/resource/Darrell_Walker, http://dbpedia.org/resource/Ed_Manning, http://dbpedia.org/resource/John_Paxson, http://dbpedia.org/resource/Kyle_Korver, http://dbpedia.org/resource/Scott_Williams_(basketball), http://dbpedia.org/resource/Scottie_Pippen, http://dbpedia.org/resource/Steve_Kerr, http://dbpedia.org/resource/Trenton_Hassell, http://dbpedia.org/resource/Michael_Sweetney, http://dbpedia.org/resource/Daequan_Cook, http://dbpedia.org/resource/Malik_Allen, http://dbpedia.org/resource/John_Lucas_III, http://dbpedia.org/resource/Sam_Smith_(basketball,_born_1955), http://dbpedia.org/resource/James_Johnson_(basketball,_born_1987), http://dbpedia.org/resource/Dave_Johnson_(basketball), http://dbpedia.org/resource/Keith_Booth, http://dbpedia.org/resource/Gerry_Ward_(basketball), http://dbpedia.org/resource/Demetris_Nichols, http://dbpedia.org/resource/Ray_Blume, http://dbpedia.org/resource/Mike_Brown_(basketball,_born_1963), http://dbpedia.org/resource/David_Vaughn_III, http://dbpedia.org/resource/Mario_Bennett, http://dbpedia.org/resource/Joe_Courtney_(basketball), http://dbpedia.org/resource/Jackie_Dinkins, http://dbpedia.org/resource/Anthony_Jones_(basketball), http://dbpedia.org/resource/David_Nash_(basketball), http://dbpedia.org/resource/Adrian_Griffin, http://dbpedia.org/resource/Chris_Duhon, http://dbpedia.org/resource/Doug_McDermott, http://dbpedia.org/resource/Flynn_Robinson, http://dbpedia.org/resource/Fred_Hoiberg, http://dbpedia.org/resource/Luol_Deng, http://dbpedia.org/resource/Randy_Livingston, http://dbpedia.org/resource/Robert_Parish, http://dbpedia.org/resource/Rodney_McCray_(basketball), http://dbpedia.org/resource/Tyrus_Thomas, http://dbpedia.org/resource/Will_Perdue, http://dbpedia.org/resource/Derrick_Rose, http://dbpedia.org/resource/Kornél_Dávid, http://dbpedia.org/resource/Kurt_Thomas_(basketball), http://dbpedia.org/resource/P._J._Brown, http://dbpedia.org/resource/Reggie_Theus, http://dbpedia.org/resource/B._J._Armstrong, http://dbpedia.org/resource/Jawann_Oldham, http://dbpedia.org/resource/Jake_Voskuhl, http://dbpedia.org/resource/Eddie_Robinson_(basketball), http://dbpedia.org/resource/Cory_Carr, http://dbpedia.org/resource/Mike_Bratz, http://dbpedia.org/resource/Paul_Ruffner, http://dbpedia.org/resource/Reggie_Harding, http://dbpedia.org/resource/Roger_Mason,_Jr., http://dbpedia.org/resource/Nate_Bowman, http://dbpedia.org/resource/Jared_Reiner, http://dbpedia.org/resource/Dalibor_Bagarić, http://dbpedia.org/resource/Larry_Spriggs, http://dbpedia.org/resource/Tracy_Jackson, http://dbpedia.org/resource/Norman_Richardson_(basketball), http://dbpedia.org/resource/Jeff_Sanders, http://dbpedia.org/resource/Khalid_El-Amin, http://dbpedia.org/resource/Malcolm_Thomas_(basketball,_born_1988), http://dbpedia.org/resource/Lou_Amundson, http://dbpedia.org/resource/Steve_Goodrich, http://dbpedia.org/resource/Andre_Wakefield, http://dbpedia.org/resource/Caldwell_Jones, http://dbpedia.org/resource/George_Gervin, http://dbpedia.org/resource/John_Starks_(basketball), http://dbpedia.org/resource/Keith_Erickson, http://dbpedia.org/resource/Marcus_Fizer, http://dbpedia.org/resource/Mark_Bryant_(basketball), http://dbpedia.org/resource/Aaron_Gray, http://dbpedia.org/resource/Bill_Wennington, http://dbpedia.org/resource/Brent_Barry, http://dbpedia.org/resource/Charles_Oakley, http://dbpedia.org/resource/Chris_Richard_(basketball), http://dbpedia.org/resource/Eddy_Curry, http://dbpedia.org/resource/Kevin_Ollie, http://dbpedia.org/resource/Lawrence_Funderburke, http://dbpedia.org/resource/Lindsey_Hunter, http://dbpedia.org/resource/Luke_Schenscher, http://dbpedia.org/resource/Sedale_Threatt, http://dbpedia.org/resource/Bob_Hansen, http://dbpedia.org/resource/John_Brown_(basketball,_born_1951), http://dbpedia.org/resource/Ron_Harper, http://dbpedia.org/resource/Scott_Burrell, http://dbpedia.org/resource/Willie_Smith_(basketball), http://dbpedia.org/resource/Ben_Gordon, http://dbpedia.org/resource/Rod_Higgins, http://dbpedia.org/resource/Cliff_Levingston, http://dbpedia.org/resource/Dennis_Hopson, http://dbpedia.org/resource/Jackie_Robinson_(basketball,_born_1955), http://dbpedia.org/resource/Jo_Jo_English, http://dbpedia.org/resource/Charles_Davis_(basketball,_born_1958), http://dbpedia.org/resource/McCoy_McLemore, http://dbpedia.org/resource/Chris_Anstey, http://dbpedia.org/resource/Luc_Longley, http://dbpedia.org/resource/Michael_Jordan, http://dbpedia.org/resource/Toni_Kukoč, http://dbpedia.org/resource/Bill_Cartwright, http://dbpedia.org/resource/Bob_Kauffman, http://dbpedia.org/resource/Corie_Blount, http://dbpedia.org/resource/Craig_Hodges, http://dbpedia.org/resource/Hakim_Warrick, http://dbpedia.org/resource/Horace_Grant, http://dbpedia.org/resource/Jack_Haley_(basketball), http://dbpedia.org/resource/Jalen_Rose, http://dbpedia.org/resource/Jason_Caffey, http://dbpedia.org/resource/Joe_Kleine, http://dbpedia.org/resource/Joe_Smith_(basketball), http://dbpedia.org/resource/Kirk_Hinrich, http://dbpedia.org/resource/Larry_Hughes, http://dbpedia.org/resource/Mike_James_(basketball,_born_1975), http://dbpedia.org/resource/Nate_Thurmond, http://dbpedia.org/resource/Othella_Harrington, http://dbpedia.org/resource/Quintin_Dailey, http://dbpedia.org/resource/Rick_Adelman, http://dbpedia.org/resource/Rick_Brunson, http://dbpedia.org/resource/Ronald_Murray, http://dbpedia.org/resource/Rory_Sparrow, http://dbpedia.org/resource/Sidney_Green_(basketball), http://dbpedia.org/resource/Stacey_King, http://dbpedia.org/resource/Stephen_Graham_(basketball), http://dbpedia.org/resource/Travis_Best, http://dbpedia.org/resource/Ömer_Aşık, http://dbpedia.org/resource/Andrés_Nocioni, http://dbpedia.org/resource/James_Edwards_(basketball), http://dbpedia.org/resource/Shannon_Brown, http://dbpedia.org/resource/Jamal_Crawford, http://dbpedia.org/resource/Gar_Heard, http://dbpedia.org/resource/Tony_Brown_(basketball), http://dbpedia.org/resource/Aaron_Brooks_(basketball), http://dbpedia.org/resource/Acie_Law, http://dbpedia.org/resource/Andrew_Lang_(basketball), http://dbpedia.org/resource/Anthony_Roberson, http://dbpedia.org/resource/Bob_Love, http://dbpedia.org/resource/Bob_Weiss, http://dbpedia.org/resource/C._J._Watson, http://dbpedia.org/resource/Cameron_Bairstow, http://dbpedia.org/resource/Cazzie_Russell, http://dbpedia.org/resource/Cedric_Simmons, http://dbpedia.org/resource/Charlie_Paulk, http://dbpedia.org/resource/Chet_Walker, http://dbpedia.org/resource/Chuck_Nevitt, http://dbpedia.org/resource/Coby_Dietrick, http://dbpedia.org/resource/D._J._Augustin, http://dbpedia.org/resource/Dave_Corzine, http://dbpedia.org/resource/David_Greenwood, http://dbpedia.org/resource/Don_Kojis, http://dbpedia.org/resource/Dudley_Bradley, http://dbpedia.org/resource/Elston_Turner, http://dbpedia.org/resource/Erwin_Mueller, http://dbpedia.org/resource/Gene_Banks, http://dbpedia.org/resource/Granville_Waiters, http://dbpedia.org/resource/Guy_Rodgers, http://dbpedia.org/resource/Jack_Marin, http://dbpedia.org/resource/Jannero_Pargo, http://dbpedia.org/resource/Jim_King_(basketball), http://dbpedia.org/resource/Jim_Washington, http://dbpedia.org/resource/Jimmy_Butler_(basketball), http://dbpedia.org/resource/Joe_Alexander_(basketball), http://dbpedia.org/resource/John_Block_(basketball), http://dbpedia.org/resource/Jud_Buechler, http://dbpedia.org/resource/Len_Chappell, http://dbpedia.org/resource/Matt_Maloney, http://dbpedia.org/resource/Michael_Holton, http://dbpedia.org/resource/Michael_Ruffin, http://dbpedia.org/resource/Mickey_Johnson, http://dbpedia.org/resource/Nick_Weatherspoon, http://dbpedia.org/resource/Nikola_Mirotić, http://dbpedia.org/resource/Ricky_Sobers, http://dbpedia.org/resource/Ronnie_Brewer, http://dbpedia.org/resource/Ronnie_Lester, http://dbpedia.org/resource/Steve_Johnson_(basketball), http://dbpedia.org/resource/Taj_Gibson, http://dbpedia.org/resource/Tony_Snell_(basketball), http://dbpedia.org/resource/Bob_Wilkerson, http://dbpedia.org/resource/Jim_Burns_(basketball), http://dbpedia.org/resource/John_Hummer, http://dbpedia.org/resource/James_Thomas_(basketball), http://dbpedia.org/resource/Lonny_Baxter, http://dbpedia.org/resource/Mark_Randall_(basketball), http://dbpedia.org/resource/Cristiano_Felício, http://dbpedia.org/resource/Thomas_Gardner_(basketball), http://dbpedia.org/resource/Eric_Fernsten, http://dbpedia.org/resource/Jim_Fox_(basketball,_born_1943), http://dbpedia.org/resource/Ed_Nealy, http://dbpedia.org/resource/Shaler_Halimon, http://dbpedia.org/resource/Scott_May, http://dbpedia.org/resource/Thabo_Sefolosha, http://dbpedia.org/resource/Antonio_Davis, http://dbpedia.org/resource/Bryce_Drew, http://dbpedia.org/resource/Dave_Schellhase, http://dbpedia.org/resource/Jerry_Sloan, http://dbpedia.org/resource/John_Mengelt, http://dbpedia.org/resource/Kyle_Macy, http://dbpedia.org/resource/Marquis_Teague, http://dbpedia.org/resource/Roger_Burkman, http://dbpedia.org/resource/Wallace_Bryant, http://dbpedia.org/resource/Artis_Gilmore, http://dbpedia.org/resource/Walt_Wesley, http://dbpedia.org/resource/Carlos_Boozer, http://dbpedia.org/resource/Dwight_Jones_(basketball), http://dbpedia.org/resource/George_Wilson_(basketball,_born_1942), http://dbpedia.org/resource/Greg_Anthony, http://dbpedia.org/resource/Greg_Foster_(basketball), http://dbpedia.org/resource/Hersey_Hawkins, http://dbpedia.org/resource/Jim_Barnes_(basketball), http://dbpedia.org/resource/Metta_World_Peace, http://dbpedia.org/resource/Pau_Gasol, http://dbpedia.org/resource/Tyson_Chandler, http://dbpedia.org/resource/Vladimir_Radmanović, http://dbpedia.org/resource/Erik_Murphy, http://dbpedia.org/resource/Ollie_Johnson_(basketball,_born_1949), http://dbpedia.org/resource/Charles_Jones_(basketball,_born_1957), http://dbpedia.org/resource/Charles_Jones_(basketball,_born_1975), http://dbpedia.org/resource/Del_Beshore, http://dbpedia.org/resource/Mike_Smrek, http://dbpedia.org/resource/Darren_Daye, http://dbpedia.org/resource/A._J._Guyton, http://dbpedia.org/resource/Al_Tucker, http://dbpedia.org/resource/Cartier_Martin, http://dbpedia.org/resource/Ricky_Blanton, http://dbpedia.org/resource/Phil_Hicks, http://dbpedia.org/resource/Corey_Williams_(basketball,_born_1970), http://dbpedia.org/resource/Oliver_Mack, http://dbpedia.org/resource/Randy_Holcomb, http://dbpedia.org/resource/Victor_Khryapa, http://dbpedia.org/resource/Paul_McCracken_(basketball), http://dbpedia.org/resource/Donyell_Marshall, http://dbpedia.org/resource/Kendall_Gill, http://dbpedia.org/resource/Ron_Mercer, http://dbpedia.org/resource/Jay_Williams_(basketball), http://dbpedia.org/resource/Dennis_Awtrey, http://dbpedia.org/resource/Eddie_Basden, http://dbpedia.org/resource/Mitchell_Wiggins, http://dbpedia.org/resource/Linton_Johnson, http://dbpedia.org/resource/Matt_Steigenga, http://dbpedia.org/resource/Bill_Hewitt_(basketball), http://dbpedia.org/resource/Dave_Newmark, http://dbpedia.org/resource/Sam_Vincent, http://dbpedia.org/resource/Ben_Wallace, http://dbpedia.org/resource/Drew_Gooden, http://dbpedia.org/resource/Earl_Cureton, http://dbpedia.org/resource/Eric_Piatkowski, http://dbpedia.org/resource/Larry_Krystkowiak, http://dbpedia.org/resource/Mike_Dunleavy,_Jr., http://dbpedia.org/resource/Nazr_Mohammed, http://dbpedia.org/resource/Pete_Myers, http://dbpedia.org/resource/Ron_Brewer, http://dbpedia.org/resource/Khalid_Reeves, http://dbpedia.org/resource/Tornike_Shengelia, http://dbpedia.org/resource/Bobby_Portis, http://dbpedia.org/resource/Dragan_Tarlać, http://dbpedia.org/resource/Steve_Patterson_(basketball), http://dbpedia.org/resource/Derrek_Dickey, http://dbpedia.org/resource/Justin_Holiday, http://dbpedia.org/resource/Tony_White_(basketball), http://dbpedia.org/resource/Charles_Dudley_(basketball), http://dbpedia.org/resource/Cliff_Pondexter, http://dbpedia.org/resource/Dominic_Pressley, http://dbpedia.org/resource/Glenn_Hansen, http://dbpedia.org/resource/Ken_Wilburn, http://dbpedia.org/resource/Loy_Petersen, http://dbpedia.org/resource/Tom_Kropp, http://dbpedia.org/resource/Wilbur_Holland, http://dbpedia.org/resource/Roger_Brown_(basketball,_born_1950)] [INFO] Predicted Answer: [http://dbpedia.org/resource/Randy_Brown, http://dbpedia.org/resource/Bob_Kauffman__2, http://dbpedia.org/resource/Charles_Jones_(basketball,_born_1975)__1, http://dbpedia.org/resource/Chris_Jefferies__2, http://dbpedia.org/resource/Corey_Benjamin__1, http://dbpedia.org/resource/David_Wood_(basketball)__2, http://dbpedia.org/resource/Don_Kojis__3, http://dbpedia.org/resource/John_Starks_(basketball)__6, http://dbpedia.org/resource/Kornél_Dávid__7, http://dbpedia.org/resource/Kyle_Macy__2, http://dbpedia.org/resource/Luke_Schenscher__2, http://dbpedia.org/resource/Sedale_Threatt__2, http://dbpedia.org/resource/Stephen_Graham_(basketball)__3, http://dbpedia.org/resource/Will_Perdue__1, http://dbpedia.org/resource/Will_Perdue__3, http://dbpedia.org/resource/Billy_McKinney_(basketball)__5, http://dbpedia.org/resource/Bob_Wilkerson__3, http://dbpedia.org/resource/Jim_Fox_(basketball,_born_1943)__6, http://dbpedia.org/resource/Lou_Amundson__10, http://dbpedia.org/resource/Lou_Amundson__8, http://dbpedia.org/resource/Tony_Brown_(basketball)__4, http://dbpedia.org/resource/Chris_Engler__4, http://dbpedia.org/resource/Linton_Johnson__10, http://dbpedia.org/resource/Linton_Johnson__2, http://dbpedia.org/resource/Erik_Murphy__1, http://dbpedia.org/resource/Cristiano_Felício__3, http://dbpedia.org/resource/E'Twaun_Moore, http://dbpedia.org/resource/Joakim_Noah, http://dbpedia.org/resource/Doug_McDermott, http://dbpedia.org/resource/Fred_Hoiberg, http://dbpedia.org/resource/Derrick_Rose, http://dbpedia.org/resource/Aaron_Brooks_(basketball), http://dbpedia.org/resource/Cameron_Bairstow, http://dbpedia.org/resource/Jimmy_Butler_(basketball), http://dbpedia.org/resource/Nikola_Mirotić, http://dbpedia.org/resource/Taj_Gibson, http://dbpedia.org/resource/Tony_Snell_(basketball), http://dbpedia.org/resource/Cristiano_Felício, http://dbpedia.org/resource/Pau_Gasol, http://dbpedia.org/resource/Mike_Wilhelm_(basketball), http://dbpedia.org/resource/Jim_Boylen, http://dbpedia.org/resource/Mike_Dunleavy,_Jr., http://dbpedia.org/resource/Pete_Myers, http://dbpedia.org/resource/Bobby_Portis, http://dbpedia.org/resource/Justin_Holiday, http://dbpedia.org/resource/Andrés_Nocioni__6, http://dbpedia.org/resource/Antonio_Davis__5, http://dbpedia.org/resource/Artis_Gilmore__2, http://dbpedia.org/resource/Artis_Gilmore__4, http://dbpedia.org/resource/Charles_Oakley__1, http://dbpedia.org/resource/Charles_Oakley__4, http://dbpedia.org/resource/Chet_Walker__2, http://dbpedia.org/resource/Darius_Songaila__3, http://dbpedia.org/resource/Drew_Gooden__4, http://dbpedia.org/resource/Eddy_Curry__1, http://dbpedia.org/resource/Eric_Piatkowski__3, http://dbpedia.org/resource/Greg_Anthony__5, http://dbpedia.org/resource/Guy_Rodgers__2, http://dbpedia.org/resource/Hakim_Warrick__3, http://dbpedia.org/resource/Jannero_Pargo__4, http://dbpedia.org/resource/Jannero_Pargo__8, http://dbpedia.org/resource/Jared_Reiner__1, http://dbpedia.org/resource/Jerome_Williams_(basketball)__3, http://dbpedia.org/resource/John_Lucas_III__10, http://dbpedia.org/resource/John_Salley__4, http://dbpedia.org/resource/Kevin_Ollie__9, http://dbpedia.org/resource/Kyle_Korver__3, http://dbpedia.org/resource/Larry_Hughes__5, http://dbpedia.org/resource/Lindsey_Hunter__6, http://dbpedia.org/resource/Luol_Deng__1, http://dbpedia.org/resource/Matt_Guokas__2, http://dbpedia.org/resource/Matt_Guokas__6, http://dbpedia.org/resource/Mike_Dunleavy,_Jr.__4, http://dbpedia.org/resource/Nazr_Mohammed__8, http://dbpedia.org/resource/Paul_Shirley__6, http://dbpedia.org/resource/Robert_Parish__4, http://dbpedia.org/resource/Tyson_Chandler__1, http://dbpedia.org/resource/Andrew_Lang_(basketball)__6, http://dbpedia.org/resource/Barry_Clemens__2, http://dbpedia.org/resource/Bob_Boozer__5, http://dbpedia.org/resource/Charles_Jones_(basketball,_born_1957)__8, http://dbpedia.org/resource/Chris_Anstey__4, http://dbpedia.org/resource/Chuck_Nevitt__6, http://dbpedia.org/resource/Cliff_Levingston__3, http://dbpedia.org/resource/Clifford_Ray__1, http://dbpedia.org/resource/Dave_Corzine__3, http://dbpedia.org/resource/David_Greenwood__1, http://dbpedia.org/resource/Dennis_Awtrey__2, http://dbpedia.org/resource/Dennis_Awtrey__6, http://dbpedia.org/resource/Hersey_Hawkins__4, http://dbpedia.org/resource/Jack_Marin__4, http://dbpedia.org/resource/James_Edwards_(basketball)__9, http://dbpedia.org/resource/James_Thomas_(basketball)__5, http://dbpedia.org/resource/Jim_King_(basketball)__4, http://dbpedia.org/resource/Jim_Washington__2, http://dbpedia.org/resource/Joe_Kleine__6, http://dbpedia.org/resource/John_Mengelt__3, http://dbpedia.org/resource/Jud_Buechler__4, http://dbpedia.org/resource/Lawrence_Funderburke__6, http://dbpedia.org/resource/McCoy_McLemore__2, http://dbpedia.org/resource/Mickey_Johnson__1, http://dbpedia.org/resource/Mike_Brown_(basketball,_born_1963)__2, http://dbpedia.org/resource/Mike_Smrek__1, http://dbpedia.org/resource/Orlando_Woolridge__1, http://dbpedia.org/resource/Pete_Myers__1, http://dbpedia.org/resource/Randy_Brown__2, http://dbpedia.org/resource/Reggie_Theus__1, http://dbpedia.org/resource/Rick_Brunson__10, http://dbpedia.org/resource/Ron_Brewer__7, http://dbpedia.org/resource/Scott_May__1, http://dbpedia.org/resource/Scott_Williams_(basketball)__1, http://dbpedia.org/resource/Steve_Johnson_(basketball)__2, http://dbpedia.org/resource/A._J._Guyton__1, http://dbpedia.org/resource/Cedric_Simmons__4, http://dbpedia.org/resource/Chris_Carr_(basketball)__5, http://dbpedia.org/resource/Cory_Carr__1, http://dbpedia.org/resource/Dave_Johnson_(basketball)__2, http://dbpedia.org/resource/David_Vaughn_III__3, http://dbpedia.org/resource/Elston_Turner__3, http://dbpedia.org/resource/Flynn_Robinson__2, http://dbpedia.org/resource/Jack_Haley_(basketball)__2, http://dbpedia.org/resource/Jack_Haley_(basketball)__7, http://dbpedia.org/resource/Jawann_Oldham__4, http://dbpedia.org/resource/Jeff_Sanders__1, http://dbpedia.org/resource/Khalid_Reeves__8, http://dbpedia.org/resource/Larry_Krystkowiak__5, http://dbpedia.org/resource/Mark_Bryant_(basketball)__4, http://dbpedia.org/resource/Mark_Randall_(basketball)__1, http://dbpedia.org/resource/Matt_Maloney__3, http://dbpedia.org/resource/Mitchell_Wiggins__1, http://dbpedia.org/resource/Randy_Holcomb__9, http://dbpedia.org/resource/Ricky_Blanton__3, http://dbpedia.org/resource/Rod_Higgins__1, http://dbpedia.org/resource/Rod_Higgins__7, http://dbpedia.org/resource/Roger_Mason,_Jr.__1, http://dbpedia.org/resource/Ronnie_Brewer__3, http://dbpedia.org/resource/Ronnie_Brewer__7, http://dbpedia.org/resource/Shannon_Brown__3, http://dbpedia.org/resource/Sidney_Green_(basketball)__1, http://dbpedia.org/resource/Steve_Colter__2, http://dbpedia.org/resource/Thabo_Sefolosha__4, http://dbpedia.org/resource/James_Johnson_(basketball,_born_1987)__1, http://dbpedia.org/resource/Ben_Gordon__1, http://dbpedia.org/resource/Ben_Wallace__4, http://dbpedia.org/resource/Bill_Cartwright__2, http://dbpedia.org/resource/Brent_Barry__3, http://dbpedia.org/resource/Chris_Duhon__1, http://dbpedia.org/resource/Elton_Brand__1, http://dbpedia.org/resource/Jerry_Sloan__2, http://dbpedia.org/resource/Kirk_Hinrich__1, http://dbpedia.org/resource/Kirk_Hinrich__4, http://dbpedia.org/resource/Richard_Hamilton_(basketball)__3, http://dbpedia.org/resource/Brian_Scalabrine__3, http://dbpedia.org/resource/Cazzie_Russell__4, http://dbpedia.org/resource/Craig_Hodges__4, http://dbpedia.org/resource/Devin_Brown__10, http://dbpedia.org/resource/Frank_Williams_(basketball)__2, http://dbpedia.org/resource/Jamal_Crawford__1, http://dbpedia.org/resource/Jay_Williams_(basketball)__1, http://dbpedia.org/resource/Joe_Smith_(basketball)__9, http://dbpedia.org/resource/Keith_Bogans__7, http://dbpedia.org/resource/Kendall_Gill__7, http://dbpedia.org/resource/Kurt_Thomas_(basketball)__8, http://dbpedia.org/resource/Michael_Sweetney__2, http://dbpedia.org/resource/Ron_Harper__3, http://dbpedia.org/resource/Ronald_Dupree__2, http://dbpedia.org/resource/Vladimir_Radmanović__9, http://dbpedia.org/resource/Anthony_Roberson__6, http://dbpedia.org/resource/Ben_Poquette__4, http://dbpedia.org/resource/Bob_Hansen__3, http://dbpedia.org/resource/Keith_Starr__1, http://dbpedia.org/resource/Aaron_Gray__1, http://dbpedia.org/resource/Darren_Daye__2, http://dbpedia.org/resource/Dudley_Bradley__3, http://dbpedia.org/resource/Granville_Waiters__3, http://dbpedia.org/resource/Jim_Barnes_(basketball)__4, http://dbpedia.org/resource/Steve_Patterson_(basketball)__2, http://dbpedia.org/resource/Wes_Matthews__6, http://dbpedia.org/resource/Al_Tucker__3, http://dbpedia.org/resource/Charles_Davis_(basketball,_born_1958)__6, http://dbpedia.org/resource/Larry_Spriggs__5, http://dbpedia.org/resource/Michael_Holton__4, http://dbpedia.org/resource/Taj_Gibson__1, http://dbpedia.org/resource/Wilbur_Holland__2, http://dbpedia.org/resource/Ömer_Aşık__5, http://dbpedia.org/resource/Andre_Wakefield__1, http://dbpedia.org/resource/Thomas_Gardner_(basketball)__2, http://dbpedia.org/resource/Tom_Kropp__2, http://dbpedia.org/resource/Willie_Smith_(basketball)__1, http://dbpedia.org/resource/Jarvis_Varnado__9, http://dbpedia.org/resource/Phil_Hicks__2, http://dbpedia.org/resource/Derrick_Rose__1, http://dbpedia.org/resource/Glenn_Hansen__2, http://dbpedia.org/resource/Joe_Alexander_(basketball)__3, http://dbpedia.org/resource/Paul_Ruffner__1, http://dbpedia.org/resource/Steve_Goodrich__5, http://dbpedia.org/resource/Gerry_Ward_(basketball)__4, http://dbpedia.org/resource/John_Hummer__2, http://dbpedia.org/resource/Dave_Schellhase__1, http://dbpedia.org/resource/Jimmy_Butler_(basketball)__1, http://dbpedia.org/resource/Jimmer_Fredette__2, http://dbpedia.org/resource/Sam_Smith_(basketball,_born_1955)__4, http://dbpedia.org/resource/Tim_Thomas_(basketball)__4, http://dbpedia.org/resource/Tim_Thomas_(basketball)__8, http://dbpedia.org/resource/Nikola_Mirotić__4, http://dbpedia.org/resource/John_Paxson__2, http://dbpedia.org/resource/Luc_Longley__3, http://dbpedia.org/resource/Pau_Gasol__4, http://dbpedia.org/resource/Tornike_Shengelia__7, http://dbpedia.org/resource/Ken_Wilburn__2, http://dbpedia.org/resource/Justin_Holiday__8, http://dbpedia.org/resource/Shaler_Halimon__2, http://dbpedia.org/resource/Gene_Banks__2, http://dbpedia.org/resource/Marco_Belinelli__6, http://dbpedia.org/resource/Aaron_Brooks_(basketball)__7, http://dbpedia.org/resource/Bobby_Wilson_(basketball,_born_1951)__1, http://dbpedia.org/resource/Corey_Williams_(basketball,_born_1970)__1, http://dbpedia.org/resource/Ed_Nealy__6, http://dbpedia.org/resource/Ed_Nealy__8, http://dbpedia.org/resource/John_Brown_(basketball,_born_1951)__2, http://dbpedia.org/resource/Kennedy_McIntosh__1, http://dbpedia.org/resource/Mark_Landsberger__1, http://dbpedia.org/resource/Mike_Bratz__4, http://dbpedia.org/resource/Nate_Bowman__2, http://dbpedia.org/resource/Reggie_Harding__2, http://dbpedia.org/resource/Steve_Sheppard__1, http://dbpedia.org/resource/Tate_Armstrong__1, http://dbpedia.org/resource/Bill_Hewitt_(basketball)__4, http://dbpedia.org/resource/Charles_Dudley_(basketball)__3, http://dbpedia.org/resource/Chris_Richard_(basketball)__4, http://dbpedia.org/resource/Coby_Dietrick__3, http://dbpedia.org/resource/Derrek_Dickey__2, http://dbpedia.org/resource/Ed_Manning__2, http://dbpedia.org/resource/Eric_Fernsten__2, http://dbpedia.org/resource/George_Wilson_(basketball,_born_1942)__2, http://dbpedia.org/resource/Jackie_Robinson_(basketball,_born_1955)__4, http://dbpedia.org/resource/John_Laskowski__1, http://dbpedia.org/resource/Norman_Richardson_(basketball)__2, http://dbpedia.org/resource/Oliver_Mack__2, http://dbpedia.org/resource/Scott_Lloyd__3, http://dbpedia.org/resource/E'Twaun_Moore__4, http://dbpedia.org/resource/Acie_Law__4, http://dbpedia.org/resource/Brad_Sellers__1, http://dbpedia.org/resource/Corie_Blount__1, http://dbpedia.org/resource/Corie_Blount__7, http://dbpedia.org/resource/Dennis_Hopson__2, http://dbpedia.org/resource/Jo_Jo_English__1, http://dbpedia.org/resource/Jo_Jo_English__3, http://dbpedia.org/resource/Jo_Jo_English__6, http://dbpedia.org/resource/Joe_Courtney_(basketball)__3, http://dbpedia.org/resource/John_Block_(basketball)__7, http://dbpedia.org/resource/Mario_Bennett__5, http://dbpedia.org/resource/Matt_Steigenga__6, http://dbpedia.org/resource/Ronnie_Lester__1, http://dbpedia.org/resource/Rusty_LaRue__5, http://dbpedia.org/resource/Sam_Vincent__3, http://dbpedia.org/resource/Sam_Worthen__1, http://dbpedia.org/resource/James_Wilkes_(basketball)__1, http://dbpedia.org/resource/Keith_Erickson__2, http://dbpedia.org/resource/Lari_Ketner__1, http://dbpedia.org/resource/Nick_Weatherspoon__3, http://dbpedia.org/resource/Cameron_Bairstow__1, http://dbpedia.org/resource/Bill_Wennington__4, http://dbpedia.org/resource/Bison_Dele__4, http://dbpedia.org/resource/Brad_Miller_(basketball)__3, http://dbpedia.org/resource/Brad_Miller_(basketball)__6, http://dbpedia.org/resource/Carlos_Boozer__3, http://dbpedia.org/resource/George_Gervin__3, http://dbpedia.org/resource/Jalen_Rose__3, http://dbpedia.org/resource/Metta_World_Peace__1, http://dbpedia.org/resource/Ron_Mercer__4, http://dbpedia.org/resource/Adrian_Griffin__10, http://dbpedia.org/resource/Andre_Barrett__6, http://dbpedia.org/resource/Bob_Love__4, http://dbpedia.org/resource/Bob_Weiss__6, http://dbpedia.org/resource/Bryce_Drew__2, http://dbpedia.org/resource/Caldwell_Jones__7, http://dbpedia.org/resource/Dalibor_Bagarić__5, http://dbpedia.org/resource/Donyell_Marshall__4, http://dbpedia.org/resource/Eddie_Basden__1, http://dbpedia.org/resource/Fred_Hoiberg__2, http://dbpedia.org/resource/Jake_Voskuhl__1, http://dbpedia.org/resource/John_Salmons__3, http://dbpedia.org/resource/Lonny_Baxter__1, http://dbpedia.org/resource/Malik_Allen__4, http://dbpedia.org/resource/Marcus_Fizer__1, http://dbpedia.org/resource/Michael_Ruffin__1, http://dbpedia.org/resource/Michael_Ruffin__7, http://dbpedia.org/resource/Nate_Robinson__5, http://dbpedia.org/resource/Norm_Van_Lier__2, http://dbpedia.org/resource/Othella_Harrington__4, http://dbpedia.org/resource/P._J._Brown__5, http://dbpedia.org/resource/Rasual_Butler__4, http://dbpedia.org/resource/Rick_Adelman__3, http://dbpedia.org/resource/Ronald_Murray__8, http://dbpedia.org/resource/Stacey_King__1, http://dbpedia.org/resource/Tom_Boerwinkle__1, http://dbpedia.org/resource/Travis_Best__2, http://dbpedia.org/resource/Trenton_Hassell__1, http://dbpedia.org/resource/Victor_Khryapa__5, http://dbpedia.org/resource/Charlie_Paulk__3, http://dbpedia.org/resource/Paul_McCracken_(basketball)__4, http://dbpedia.org/resource/Dominic_Pressley__5, http://dbpedia.org/resource/Jackie_Dinkins__1, http://dbpedia.org/resource/David_Nash_(basketball)__1, http://dbpedia.org/resource/B._J._Armstrong__1, http://dbpedia.org/resource/B._J._Armstrong__5, http://dbpedia.org/resource/Dennis_Rodman__3, http://dbpedia.org/resource/Horace_Grant__1, http://dbpedia.org/resource/Nate_Thurmond__2, http://dbpedia.org/resource/Scottie_Pippen__1, http://dbpedia.org/resource/Scottie_Pippen__4, http://dbpedia.org/resource/Steve_Kerr__4, http://dbpedia.org/resource/Toni_Kukoč__3, http://dbpedia.org/resource/D._J._Augustin__4, http://dbpedia.org/resource/Daequan_Cook__5, http://dbpedia.org/resource/Del_Beshore__3, http://dbpedia.org/resource/Demetris_Nichols__2, http://dbpedia.org/resource/Dwight_Jones_(basketball)__3, http://dbpedia.org/resource/Howard_Porter_(basketball)__1, http://dbpedia.org/resource/Jim_Ard__4, http://dbpedia.org/resource/Loy_Petersen__1, http://dbpedia.org/resource/Tracy_Jackson__2, http://dbpedia.org/resource/Wallace_Bryant__2, http://dbpedia.org/resource/Anthony_Jones_(basketball)__5, http://dbpedia.org/resource/Jim_Burns_(basketball)__1, http://dbpedia.org/resource/Johnny_Baum__1, http://dbpedia.org/resource/Leon_Benbow__1, http://dbpedia.org/resource/Ollie_Johnson_(basketball,_born_1949)__5, http://dbpedia.org/resource/Ray_Blume__1, http://dbpedia.org/resource/Roger_Burkman__1, http://dbpedia.org/resource/Tony_White_(basketball)__1, http://dbpedia.org/resource/Walt_Wesley__2, http://dbpedia.org/resource/Doug_McDermott__1, http://dbpedia.org/resource/Tony_Snell_(basketball)__1, http://dbpedia.org/resource/Clem_Haskins__1, http://dbpedia.org/resource/Darrell_Walker__5, http://dbpedia.org/resource/Dickey_Simpkins__1, http://dbpedia.org/resource/Dickey_Simpkins__3, http://dbpedia.org/resource/Dragan_Tarlać__3, http://dbpedia.org/resource/Earl_Cureton__4, http://dbpedia.org/resource/Eddie_Robinson_(basketball)__2, http://dbpedia.org/resource/Erwin_Mueller__1, http://dbpedia.org/resource/Erwin_Mueller__3, http://dbpedia.org/resource/Gar_Heard__2, http://dbpedia.org/resource/Greg_Foster_(basketball)__6, http://dbpedia.org/resource/Jason_Caffey__1, http://dbpedia.org/resource/Joakim_Noah__1, http://dbpedia.org/resource/Keith_Booth__1, http://dbpedia.org/resource/Khalid_El-Amin__1, http://dbpedia.org/resource/Larry_Kenon__3, http://dbpedia.org/resource/Len_Chappell__3, http://dbpedia.org/resource/Quintin_Dailey__1, http://dbpedia.org/resource/Ricky_Sobers__3, http://dbpedia.org/resource/Rodney_McCray_(basketball)__4, http://dbpedia.org/resource/Roger_Brown_(basketball,_born_1950)__8, http://dbpedia.org/resource/Rory_Sparrow__5, http://dbpedia.org/resource/Rory_Sparrow__8, http://dbpedia.org/resource/Scott_Burrell__3, http://dbpedia.org/resource/Trent_Tucker__3, http://dbpedia.org/resource/Tyrus_Thomas__1, http://dbpedia.org/resource/C._J._Watson__5, http://dbpedia.org/resource/Dave_Newmark__1, http://dbpedia.org/resource/Bobby_Portis__1, http://dbpedia.org/resource/Cliff_Pondexter__1, http://dbpedia.org/resource/Marquis_Teague__1, http://dbpedia.org/resource/Rowland_Garrett__1, http://dbpedia.org/resource/Devin_Brown, http://dbpedia.org/resource/John_Salmons, http://dbpedia.org/resource/Larry_Kenon, http://dbpedia.org/resource/Trent_Tucker, http://dbpedia.org/resource/Ben_Poquette, http://dbpedia.org/resource/Bob_Boozer, http://dbpedia.org/resource/Dennis_Rodman, http://dbpedia.org/resource/Elton_Brand, http://dbpedia.org/resource/Howard_Porter_(basketball), http://dbpedia.org/resource/Keith_Bogans, http://dbpedia.org/resource/Marco_Belinelli, http://dbpedia.org/resource/Nate_Robinson, http://dbpedia.org/resource/Orlando_Woolridge, http://dbpedia.org/resource/Rasual_Butler, http://dbpedia.org/resource/Billy_McKinney_(basketball), http://dbpedia.org/resource/David_Wood_(basketball), http://dbpedia.org/resource/Jerome_Williams_(basketball), http://dbpedia.org/resource/Paul_Shirley, http://dbpedia.org/resource/Chris_Engler, http://dbpedia.org/resource/Ronald_Dupree, http://dbpedia.org/resource/Steve_Colter, http://dbpedia.org/resource/Barry_Clemens, http://dbpedia.org/resource/Mark_Landsberger, http://dbpedia.org/resource/Jarvis_Varnado, http://dbpedia.org/resource/Frank_Williams_(basketball), http://dbpedia.org/resource/Scott_Lloyd, http://dbpedia.org/resource/2015_NBA_Summer_League, http://dbpedia.org/resource/Jim_Ard, http://dbpedia.org/resource/Johnny_Baum, http://dbpedia.org/resource/Wes_Matthews, http://dbpedia.org/resource/Rusty_LaRue, http://dbpedia.org/resource/Chris_Jefferies, http://dbpedia.org/resource/Keith_Starr, http://dbpedia.org/resource/Bison_Dele, http://dbpedia.org/resource/Brad_Miller_(basketball), http://dbpedia.org/resource/Dickey_Simpkins, http://dbpedia.org/resource/John_Salley, http://dbpedia.org/resource/Norm_Van_Lier, http://dbpedia.org/resource/Andre_Barrett, http://dbpedia.org/resource/Darius_Songaila, http://dbpedia.org/resource/Jimmer_Fredette, http://dbpedia.org/resource/Lari_Ketner, http://dbpedia.org/resource/Matt_Guokas, http://dbpedia.org/resource/Richard_Hamilton_(basketball), http://dbpedia.org/resource/Tim_Thomas_(basketball), http://dbpedia.org/resource/1990_NBA_Playoffs, http://dbpedia.org/resource/1991_NBA_Finals, http://dbpedia.org/resource/1991_NBA_Playoffs, http://dbpedia.org/resource/1992_NBA_Finals, http://dbpedia.org/resource/1992_NBA_Playoffs, http://dbpedia.org/resource/1993_NBA_Playoffs, http://dbpedia.org/resource/1994_NBA_Playoffs, http://dbpedia.org/resource/1995_NBA_Playoffs, http://dbpedia.org/resource/1996_NBA_Finals, http://dbpedia.org/resource/1996_NBA_Playoffs, http://dbpedia.org/resource/1997_NBA_Finals, http://dbpedia.org/resource/1997_NBA_Playoffs, http://dbpedia.org/resource/1998_NBA_Playoffs, http://dbpedia.org/resource/Brian_Scalabrine, http://dbpedia.org/resource/Chris_Carr_(basketball), http://dbpedia.org/resource/Darrell_Walker, http://dbpedia.org/resource/Ed_Manning, http://dbpedia.org/resource/John_Paxson, http://dbpedia.org/resource/Kyle_Korver, http://dbpedia.org/resource/Scott_Williams_(basketball), http://dbpedia.org/resource/Scottie_Pippen, http://dbpedia.org/resource/Steve_Kerr, http://dbpedia.org/resource/Trenton_Hassell, http://dbpedia.org/resource/Michael_Sweetney, http://dbpedia.org/resource/Daequan_Cook, http://dbpedia.org/resource/Malik_Allen, http://dbpedia.org/resource/John_Lucas_III, http://dbpedia.org/resource/Sam_Smith_(basketball,_born_1955), http://dbpedia.org/resource/James_Johnson_(basketball,_born_1987), http://dbpedia.org/resource/Dave_Johnson_(basketball), http://dbpedia.org/resource/Keith_Booth, http://dbpedia.org/resource/Gerry_Ward_(basketball), http://dbpedia.org/resource/Demetris_Nichols, http://dbpedia.org/resource/Ray_Blume, http://dbpedia.org/resource/Mike_Brown_(basketball,_born_1963), http://dbpedia.org/resource/David_Vaughn_III, http://dbpedia.org/resource/Mario_Bennett, http://dbpedia.org/resource/Joe_Courtney_(basketball), http://dbpedia.org/resource/Jackie_Dinkins, http://dbpedia.org/resource/Anthony_Jones_(basketball), http://dbpedia.org/resource/David_Nash_(basketball), http://dbpedia.org/resource/Adrian_Griffin, http://dbpedia.org/resource/Chris_Duhon, http://dbpedia.org/resource/Flynn_Robinson, http://dbpedia.org/resource/Luol_Deng, http://dbpedia.org/resource/Randy_Livingston, http://dbpedia.org/resource/Robert_Parish, http://dbpedia.org/resource/Rodney_McCray_(basketball), http://dbpedia.org/resource/Tyrus_Thomas, http://dbpedia.org/resource/Will_Perdue, http://dbpedia.org/resource/Kornél_Dávid, http://dbpedia.org/resource/Kurt_Thomas_(basketball), http://dbpedia.org/resource/P._J._Brown, http://dbpedia.org/resource/Reggie_Theus, http://dbpedia.org/resource/B._J._Armstrong, http://dbpedia.org/resource/Jawann_Oldham, http://dbpedia.org/resource/Jake_Voskuhl, http://dbpedia.org/resource/Eddie_Robinson_(basketball), http://dbpedia.org/resource/Cory_Carr, http://dbpedia.org/resource/Mike_Bratz, http://dbpedia.org/resource/Paul_Ruffner, http://dbpedia.org/resource/Reggie_Harding, http://dbpedia.org/resource/Roger_Mason,_Jr., http://dbpedia.org/resource/Nate_Bowman, http://dbpedia.org/resource/Jared_Reiner, http://dbpedia.org/resource/Dalibor_Bagarić, http://dbpedia.org/resource/2006–07_Detroit_Pistons_season, http://dbpedia.org/resource/2005–06_Chicago_Bulls_season, http://dbpedia.org/resource/Larry_Spriggs, http://dbpedia.org/resource/Tracy_Jackson, http://dbpedia.org/resource/Norman_Richardson_(basketball), http://dbpedia.org/resource/Jeff_Sanders, http://dbpedia.org/resource/Khalid_El-Amin, http://dbpedia.org/resource/Malcolm_Thomas_(basketball,_born_1988), http://dbpedia.org/resource/Lou_Amundson, http://dbpedia.org/resource/Steve_Goodrich, http://dbpedia.org/resource/Andre_Wakefield, http://dbpedia.org/resource/Caldwell_Jones, http://dbpedia.org/resource/George_Gervin, http://dbpedia.org/resource/John_Starks_(basketball), http://dbpedia.org/resource/Keith_Erickson, http://dbpedia.org/resource/Marcus_Fizer, http://dbpedia.org/resource/Mark_Bryant_(basketball), http://dbpedia.org/resource/Aaron_Gray, http://dbpedia.org/resource/Bill_Wennington, http://dbpedia.org/resource/Brent_Barry, http://dbpedia.org/resource/Charles_Oakley, http://dbpedia.org/resource/Chris_Richard_(basketball), http://dbpedia.org/resource/Eddy_Curry, http://dbpedia.org/resource/Kevin_Ollie, http://dbpedia.org/resource/Lawrence_Funderburke, http://dbpedia.org/resource/Lindsey_Hunter, http://dbpedia.org/resource/Luke_Schenscher, http://dbpedia.org/resource/Sedale_Threatt, http://dbpedia.org/resource/Bob_Hansen, http://dbpedia.org/resource/John_Brown_(basketball,_born_1951), http://dbpedia.org/resource/Ron_Harper, http://dbpedia.org/resource/Scott_Burrell, http://dbpedia.org/resource/Willie_Smith_(basketball), http://dbpedia.org/resource/Ben_Gordon, http://dbpedia.org/resource/Rod_Higgins, http://dbpedia.org/resource/2004–05_Chicago_Bulls_season, http://dbpedia.org/resource/Cliff_Levingston, http://dbpedia.org/resource/Dennis_Hopson, http://dbpedia.org/resource/2014_NBA_Summer_League, http://dbpedia.org/resource/Jackie_Robinson_(basketball,_born_1955), http://dbpedia.org/resource/Jo_Jo_English, http://dbpedia.org/resource/Charles_Davis_(basketball,_born_1958), http://dbpedia.org/resource/McCoy_McLemore, http://dbpedia.org/resource/1993_NBA_Finals, http://dbpedia.org/resource/Chris_Anstey, http://dbpedia.org/resource/Luc_Longley, http://dbpedia.org/resource/Michael_Jordan, http://dbpedia.org/resource/Toni_Kukoč, http://dbpedia.org/resource/Bill_Cartwright, http://dbpedia.org/resource/Bob_Kauffman, http://dbpedia.org/resource/Corie_Blount, http://dbpedia.org/resource/Craig_Hodges, http://dbpedia.org/resource/Hakim_Warrick, http://dbpedia.org/resource/Horace_Grant, http://dbpedia.org/resource/Jack_Haley_(basketball), http://dbpedia.org/resource/Jalen_Rose, http://dbpedia.org/resource/Jason_Caffey, http://dbpedia.org/resource/Joe_Kleine, http://dbpedia.org/resource/Joe_Smith_(basketball), http://dbpedia.org/resource/Kirk_Hinrich, http://dbpedia.org/resource/Larry_Hughes, http://dbpedia.org/resource/Mike_James_(basketball,_born_1975), http://dbpedia.org/resource/Nate_Thurmond, http://dbpedia.org/resource/Othella_Harrington, http://dbpedia.org/resource/Quintin_Dailey, http://dbpedia.org/resource/Rick_Adelman, http://dbpedia.org/resource/Rick_Brunson, http://dbpedia.org/resource/Ronald_Murray, http://dbpedia.org/resource/Rory_Sparrow, http://dbpedia.org/resource/Sidney_Green_(basketball), http://dbpedia.org/resource/Stacey_King, http://dbpedia.org/resource/Stephen_Graham_(basketball), http://dbpedia.org/resource/Travis_Best, http://dbpedia.org/resource/Ömer_Aşık, http://dbpedia.org/resource/Andrés_Nocioni, http://dbpedia.org/resource/James_Edwards_(basketball), http://dbpedia.org/resource/Shannon_Brown, http://dbpedia.org/resource/Jamal_Crawford, http://dbpedia.org/resource/2010_NBA_Playoffs, http://dbpedia.org/resource/2011_NBA_Playoffs, http://dbpedia.org/resource/2013_NBA_Playoffs, http://dbpedia.org/resource/2015_NBA_Playoffs, http://dbpedia.org/resource/Gar_Heard, http://dbpedia.org/resource/Tony_Brown_(basketball), http://dbpedia.org/resource/2005_NBA_Playoffs, http://dbpedia.org/resource/2006_NBA_Playoffs, http://dbpedia.org/resource/2006–07_Chicago_Bulls_season, http://dbpedia.org/resource/2007_NBA_Playoffs, http://dbpedia.org/resource/2009_NBA_Playoffs, http://dbpedia.org/resource/2012_NBA_Playoffs, http://dbpedia.org/resource/2014_NBA_Playoffs, http://dbpedia.org/resource/Acie_Law, http://dbpedia.org/resource/Andrew_Lang_(basketball), http://dbpedia.org/resource/Anthony_Roberson, http://dbpedia.org/resource/Bob_Love, http://dbpedia.org/resource/Bob_Weiss, http://dbpedia.org/resource/C._J._Watson, http://dbpedia.org/resource/Cazzie_Russell, http://dbpedia.org/resource/Cedric_Simmons, http://dbpedia.org/resource/Charlie_Paulk, http://dbpedia.org/resource/Chet_Walker, http://dbpedia.org/resource/Chuck_Nevitt, http://dbpedia.org/resource/Coby_Dietrick, http://dbpedia.org/resource/D._J._Augustin, http://dbpedia.org/resource/Dave_Corzine, http://dbpedia.org/resource/David_Greenwood, http://dbpedia.org/resource/Don_Kojis, http://dbpedia.org/resource/Dudley_Bradley, http://dbpedia.org/resource/Elston_Turner, http://dbpedia.org/resource/Erwin_Mueller, http://dbpedia.org/resource/Gene_Banks, http://dbpedia.org/resource/Granville_Waiters, http://dbpedia.org/resource/Guy_Rodgers, http://dbpedia.org/resource/Jack_Marin, http://dbpedia.org/resource/Jannero_Pargo, http://dbpedia.org/resource/Jim_King_(basketball), http://dbpedia.org/resource/Jim_Washington, http://dbpedia.org/resource/Joe_Alexander_(basketball), http://dbpedia.org/resource/John_Block_(basketball), http://dbpedia.org/resource/Jud_Buechler, http://dbpedia.org/resource/Len_Chappell, http://dbpedia.org/resource/Matt_Maloney, http://dbpedia.org/resource/Michael_Holton, http://dbpedia.org/resource/Michael_Ruffin, http://dbpedia.org/resource/Mickey_Johnson, http://dbpedia.org/resource/Nick_Weatherspoon, http://dbpedia.org/resource/Ricky_Sobers, http://dbpedia.org/resource/Ronnie_Brewer, http://dbpedia.org/resource/Ronnie_Lester, http://dbpedia.org/resource/Steve_Johnson_(basketball), http://dbpedia.org/resource/Bob_Wilkerson, http://dbpedia.org/resource/Jim_Burns_(basketball), http://dbpedia.org/resource/2009–10_Cleveland_Cavaliers_season, http://dbpedia.org/resource/John_Hummer, http://dbpedia.org/resource/James_Thomas_(basketball), http://dbpedia.org/resource/Lonny_Baxter, http://dbpedia.org/resource/2004–05_Washington_Wizards_season, http://dbpedia.org/resource/Mark_Randall_(basketball), http://dbpedia.org/resource/Thomas_Gardner_(basketball), http://dbpedia.org/resource/Eric_Fernsten, http://dbpedia.org/resource/Jim_Fox_(basketball,_born_1943), http://dbpedia.org/resource/Ed_Nealy, http://dbpedia.org/resource/Shaler_Halimon, http://dbpedia.org/resource/1998_NBA_Finals, http://dbpedia.org/resource/Scott_May, http://dbpedia.org/resource/Thabo_Sefolosha, http://dbpedia.org/resource/Antonio_Davis, http://dbpedia.org/resource/Bryce_Drew, http://dbpedia.org/resource/Dave_Schellhase, http://dbpedia.org/resource/Jerry_Sloan, http://dbpedia.org/resource/John_Mengelt, http://dbpedia.org/resource/Kyle_Macy, http://dbpedia.org/resource/Marquis_Teague, http://dbpedia.org/resource/Roger_Burkman, http://dbpedia.org/resource/Wallace_Bryant, http://dbpedia.org/resource/1981_NBA_Playoffs, http://dbpedia.org/resource/1985_NBA_Playoffs, http://dbpedia.org/resource/1986_NBA_Playoffs, http://dbpedia.org/resource/1987_NBA_Playoffs, http://dbpedia.org/resource/1988_NBA_Playoffs, http://dbpedia.org/resource/1989_NBA_Playoffs, http://dbpedia.org/resource/Artis_Gilmore, http://dbpedia.org/resource/Walt_Wesley, http://dbpedia.org/resource/Carlos_Boozer, http://dbpedia.org/resource/Dwight_Jones_(basketball), http://dbpedia.org/resource/George_Wilson_(basketball,_born_1942), http://dbpedia.org/resource/Greg_Anthony, http://dbpedia.org/resource/Greg_Foster_(basketball), http://dbpedia.org/resource/Hersey_Hawkins, http://dbpedia.org/resource/Jim_Barnes_(basketball), http://dbpedia.org/resource/Metta_World_Peace, http://dbpedia.org/resource/Tyson_Chandler, http://dbpedia.org/resource/Vladimir_Radmanović, http://dbpedia.org/resource/Erik_Murphy, http://dbpedia.org/resource/Ollie_Johnson_(basketball,_born_1949), http://dbpedia.org/resource/Charles_Jones_(basketball,_born_1957), http://dbpedia.org/resource/Charles_Jones_(basketball,_born_1975), http://dbpedia.org/resource/Del_Beshore, http://dbpedia.org/resource/Mike_Smrek, http://dbpedia.org/resource/Darren_Daye, http://dbpedia.org/resource/A._J._Guyton, http://dbpedia.org/resource/1969–70_New_York_Knicks_season, http://dbpedia.org/resource/Al_Tucker, http://dbpedia.org/resource/Cartier_Martin, http://dbpedia.org/resource/Ricky_Blanton, http://dbpedia.org/resource/Phil_Hicks, http://dbpedia.org/resource/Corey_Williams_(basketball,_born_1970), http://dbpedia.org/resource/Oliver_Mack, http://dbpedia.org/resource/Randy_Holcomb, http://dbpedia.org/resource/Victor_Khryapa, http://dbpedia.org/resource/Paul_McCracken_(basketball), http://dbpedia.org/resource/Donyell_Marshall, http://dbpedia.org/resource/Kendall_Gill, http://dbpedia.org/resource/Ron_Mercer, http://dbpedia.org/resource/Jay_Williams_(basketball), http://dbpedia.org/resource/Dennis_Awtrey, http://dbpedia.org/resource/Eddie_Basden, http://dbpedia.org/resource/Mitchell_Wiggins, http://dbpedia.org/resource/Linton_Johnson, http://dbpedia.org/resource/Matt_Steigenga, http://dbpedia.org/resource/Bill_Hewitt_(basketball), http://dbpedia.org/resource/Dave_Newmark, http://dbpedia.org/resource/Sam_Vincent, http://dbpedia.org/resource/Ben_Wallace, http://dbpedia.org/resource/Drew_Gooden, http://dbpedia.org/resource/Earl_Cureton, http://dbpedia.org/resource/Eric_Piatkowski, http://dbpedia.org/resource/Larry_Krystkowiak, http://dbpedia.org/resource/Nazr_Mohammed, http://dbpedia.org/resource/Ron_Brewer, http://dbpedia.org/resource/Khalid_Reeves, http://dbpedia.org/resource/Tornike_Shengelia, http://dbpedia.org/resource/Dragan_Tarlać, http://dbpedia.org/resource/Steve_Patterson_(basketball), http://dbpedia.org/resource/Derrek_Dickey, http://dbpedia.org/resource/2013_NBA_Summer_League, http://dbpedia.org/resource/Tony_White_(basketball), http://dbpedia.org/resource/Charles_Dudley_(basketball), http://dbpedia.org/resource/Cliff_Pondexter, http://dbpedia.org/resource/Dominic_Pressley, http://dbpedia.org/resource/Glenn_Hansen, http://dbpedia.org/resource/Ken_Wilburn, http://dbpedia.org/resource/Loy_Petersen, http://dbpedia.org/resource/Tom_Kropp, http://dbpedia.org/resource/Wilbur_Holland, http://dbpedia.org/resource/Roger_Brown_(basketball,_born_1950)] [INFO] P: 0.4460227272727273, R: 1.0, F: 0.6168958742632613 -[INFO] Cumulative metrics, sample: 600, P: 0.516, R: 0.566, macro F1: 0.521, macro F1*: 0.540 -[INFO] Current time: 2211 ms, average: 2710.96 ms/ques, entity detection: 421.64 ms/ques (linking tool: 328.95 ms/ques), relation detection: 892.24 ms/ques (similarity: 704.12 ms/ques), EDG generation: 49.67 ms/ques +[INFO] Cumulative metrics, sample: 600, P: 0.516, R: 0.566, macro F1: 0.540 +[INFO] Current question consumed time: 1874 ms, average: 2465.80 ms/ques, entity detection: 377.48 ms/ques (linking tool: 297.62 ms/ques), relation detection: 830.56 ms/ques (similarity: 692.44 ms/ques), EDG generation: 49.39 ms/ques [INFO] Entity Linking P: 0.7483333333333333, R: 0.7033333333333334, micro F1: 0.7175000000000008, macro F1: 0.7251358591657099 [INFO] Relation Linking P: 0.597361111111111, R: 0.5433333333333333, micro F1: 0.5583888888888893, macro F1: 0.5690677381387231 [INFO] Type Linking P: 0.5416666666666666, R: 0.5416666666666666, micro F1: 0.5416666666666666, macro F1: 0.5416666666666666 @@ -8424,8 +8424,8 @@ Question 600: Which owner of the national Herald india is the leader of Kumta ? [INFO] Golden Answer: [http://dbpedia.org/resource/Indian_National_Congress] [INFO] Predicted Answer: [G. Anuradha KAS@en, Smt. Sharadha Mohan Shetty@en] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 601, P: 0.515, R: 0.565, macro F1: 0.520, macro F1*: 0.539 -[INFO] Current time: 17429 ms, average: 2735.45 ms/ques, entity detection: 425.63 ms/ques (linking tool: 333.04 ms/ques), relation detection: 902.46 ms/ques (similarity: 707.38 ms/ques), EDG generation: 49.63 ms/ques +[INFO] Cumulative metrics, sample: 601, P: 0.515, R: 0.565, macro F1: 0.539 +[INFO] Current question consumed time: 14556 ms, average: 2485.91 ms/ques, entity detection: 380.56 ms/ques (linking tool: 300.79 ms/ques), relation detection: 839.20 ms/ques (similarity: 695.74 ms/ques), EDG generation: 49.35 ms/ques [INFO] Entity Linking P: 0.7487520798668885, R: 0.7029950083194676, micro F1: 0.7174154187465344, macro F1: 0.7251524441117718 [INFO] Relation Linking P: 0.5963671658347198, R: 0.5424292845257903, micro F1: 0.5574597892401556, macro F1: 0.5681208700220198 [INFO] Type Linking P: 0.5424292845257903, R: 0.5424292845257903, micro F1: 0.5424292845257903, macro F1: 0.5424292845257903 @@ -8438,8 +8438,8 @@ Question 601: Was Francisco Pizarro of the monarch of Charles 5, of the roman em [INFO] Golden Answer: [true] [INFO] Predicted Answer: [false] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 602, P: 0.514, R: 0.564, macro F1: 0.519, macro F1*: 0.538 -[INFO] Current time: 1287 ms, average: 2733.04 ms/ques, entity detection: 424.92 ms/ques (linking tool: 332.48 ms/ques), relation detection: 900.96 ms/ques (similarity: 706.20 ms/ques), EDG generation: 49.74 ms/ques +[INFO] Cumulative metrics, sample: 602, P: 0.514, R: 0.564, macro F1: 0.538 +[INFO] Current question consumed time: 1114 ms, average: 2483.64 ms/ques, entity detection: 379.93 ms/ques (linking tool: 300.29 ms/ques), relation detection: 837.81 ms/ques (similarity: 694.59 ms/ques), EDG generation: 49.46 ms/ques [INFO] Entity Linking P: 0.7475083056478405, R: 0.7018272425249169, micro F1: 0.7162236987818391, macro F1: 0.7239478719454729 [INFO] Relation Linking P: 0.5953765227021041, R: 0.5415282392026578, micro F1: 0.5565337763012185, macro F1: 0.5671771476465679 [INFO] Type Linking P: 0.5431893687707641, R: 0.5431893687707641, micro F1: 0.5431893687707641, macro F1: 0.5431893687707641 @@ -8452,8 +8452,8 @@ Question 602: What is the largest city of Pulau Ubin ? [INFO] Golden Answer: [http://dbpedia.org/resource/Singapore] [INFO] Predicted Answer: [http://dbpedia.org/resource/Singapore] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 603, P: 0.515, R: 0.565, macro F1: 0.520, macro F1*: 0.539 -[INFO] Current time: 805 ms, average: 2729.85 ms/ques, entity detection: 424.47 ms/ques (linking tool: 332.08 ms/ques), relation detection: 899.99 ms/ques (similarity: 705.35 ms/ques), EDG generation: 49.68 ms/ques +[INFO] Cumulative metrics, sample: 603, P: 0.515, R: 0.565, macro F1: 0.539 +[INFO] Current question consumed time: 684 ms, average: 2480.65 ms/ques, entity detection: 379.49 ms/ques (linking tool: 299.92 ms/ques), relation detection: 836.90 ms/ques (similarity: 693.76 ms/ques), EDG generation: 49.40 ms/ques [INFO] Entity Linking P: 0.7479270315091211, R: 0.7023217247097844, micro F1: 0.7166943062465458, macro F1: 0.7244073135371362 [INFO] Relation Linking P: 0.5960475400773908, R: 0.5422885572139303, micro F1: 0.5572692095080158, macro F1: 0.5678986396172591 [INFO] Type Linking P: 0.5439469320066335, R: 0.5439469320066335, micro F1: 0.5439469320066335, macro F1: 0.5439469320066335 @@ -8466,8 +8466,8 @@ Question 603: What is the origin of the musical artist of A Todo Romantic Style [INFO] Golden Answer: [http://dbpedia.org/resource/Panama, http://dbpedia.org/resource/Chitré] [INFO] Predicted Answer: [http://dbpedia.org/resource/Panama, http://dbpedia.org/resource/Chitré] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 604, P: 0.516, R: 0.566, macro F1: 0.521, macro F1*: 0.539 -[INFO] Current time: 1160 ms, average: 2727.25 ms/ques, entity detection: 424.05 ms/ques (linking tool: 331.68 ms/ques), relation detection: 899.21 ms/ques (similarity: 704.61 ms/ques), EDG generation: 49.64 ms/ques +[INFO] Cumulative metrics, sample: 604, P: 0.516, R: 0.566, macro F1: 0.539 +[INFO] Current question consumed time: 1049 ms, average: 2478.28 ms/ques, entity detection: 379.11 ms/ques (linking tool: 299.55 ms/ques), relation detection: 836.17 ms/ques (similarity: 693.05 ms/ques), EDG generation: 49.36 ms/ques [INFO] Entity Linking P: 0.7483443708609272, R: 0.7028145695364238, micro F1: 0.7171633554083893, macro F1: 0.7248652263102421 [INFO] Relation Linking P: 0.5967163355408388, R: 0.543046357615894, micro F1: 0.5580022075055191, macro F1: 0.5686177210237844 [INFO] Type Linking P: 0.5447019867549668, R: 0.5447019867549668, micro F1: 0.5447019867549668, macro F1: 0.5447019867549668 @@ -8480,8 +8480,8 @@ Question 604: Horses sired by Karinga Bay have participated in which races? [INFO] Golden Answer: [http://dbpedia.org/resource/Bristol_Novices'_Hurdle, http://dbpedia.org/resource/Hyde_Novices'_Hurdle, http://dbpedia.org/resource/Kauto_Star_Novices'_Chase, http://dbpedia.org/resource/Berkshire_Novices'_Chase, http://dbpedia.org/resource/Denman_Chase, http://dbpedia.org/resource/Cheltenham_Gold_Cup] [INFO] Predicted Answer: [http://dbpedia.org/resource/Central_Province_(Kenya)] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 605, P: 0.515, R: 0.565, macro F1: 0.520, macro F1*: 0.539 -[INFO] Current time: 953 ms, average: 2724.32 ms/ques, entity detection: 423.61 ms/ques (linking tool: 331.27 ms/ques), relation detection: 898.10 ms/ques (similarity: 703.81 ms/ques), EDG generation: 49.60 ms/ques +[INFO] Cumulative metrics, sample: 605, P: 0.515, R: 0.565, macro F1: 0.539 +[INFO] Current question consumed time: 829 ms, average: 2475.56 ms/ques, entity detection: 378.68 ms/ques (linking tool: 299.17 ms/ques), relation detection: 835.13 ms/ques (similarity: 692.27 ms/ques), EDG generation: 49.32 ms/ques [INFO] Entity Linking P: 0.7471074380165289, R: 0.7016528925619835, micro F1: 0.7159779614325077, macro F1: 0.7236671019692333 [INFO] Relation Linking P: 0.5957300275482093, R: 0.5421487603305785, micro F1: 0.5570798898071628, macro F1: 0.5676778570220922 [INFO] Type Linking P: 0.543801652892562, R: 0.543801652892562, micro F1: 0.543801652892562, macro F1: 0.543801652892562 @@ -8494,8 +8494,8 @@ Question 605: What is the alma mater of the scientists known for String theory [INFO] Golden Answer: [http://dbpedia.org/resource/California_Institute_of_Technology, http://dbpedia.org/resource/Indian_Institute_of_Technology_Kanpur, Institute for Advanced Study@en, http://dbpedia.org/resource/Princeton_University, http://dbpedia.org/resource/University_of_Illinois_at_Urbana–Champaign, http://dbpedia.org/resource/University_of_Minnesota, http://dbpedia.org/resource/Massachusetts_Institute_of_Technology, University of Chicago@en, http://dbpedia.org/resource/Mumbai, http://dbpedia.org/resource/List_of_schools_named_after_Francis_Xavier, http://dbpedia.org/resource/Stony_Brook_University, http://dbpedia.org/resource/Cornell_University, http://dbpedia.org/resource/City_College_of_New_York, http://dbpedia.org/resource/Moscow_Institute_of_Physics_and_Technology, http://dbpedia.org/resource/Brandeis_University, http://dbpedia.org/resource/University_of_Wisconsin–Madison, Princeton University@en, http://dbpedia.org/resource/Utrecht_University, Harvard University@en, Massachusetts Institute of Technology@en, University of California, Berkeley@en, http://dbpedia.org/resource/Harvard_University, http://dbpedia.org/resource/Magdalen_College,_Oxford, http://dbpedia.org/resource/Stuyvesant_High_School, Harvard University Cornell University@en, University of Cambridge@en, http://dbpedia.org/resource/University_of_Delhi, http://dbpedia.org/resource/New_Delhi, http://dbpedia.org/resource/Stony_Brook,_New_York, Valparaiso University@en, http://dbpedia.org/resource/University_of_Cape_Town, http://dbpedia.org/resource/Stanford_University] [INFO] Predicted Answer: [http://dbpedia.org/resource/Sandip_Trivedi, http://dbpedia.org/resource/Sergei_Gukov, http://dbpedia.org/resource/Gordon_L._Kane, http://dbpedia.org/resource/Sylvester_James_Gates, http://dbpedia.org/resource/Burt_Ovrut, http://dbpedia.org/resource/Sunil_Mukhi, http://dbpedia.org/resource/Leonard_Susskind, http://dbpedia.org/resource/Alexander_Markovich_Polyakov, http://dbpedia.org/resource/Edward_Witten, http://dbpedia.org/resource/Herman_Verlinde, http://dbpedia.org/resource/Andrew_Strominger, http://dbpedia.org/resource/Brian_Greene, http://dbpedia.org/resource/Stephen_Shenker, http://dbpedia.org/resource/Malcolm_Perry_(physicist), http://dbpedia.org/resource/Ashok_Das, http://dbpedia.org/resource/Gabriele_Veneziano, http://dbpedia.org/resource/Gerald_B._Cleaver, http://dbpedia.org/resource/Peter_Goddard_(physicist), http://dbpedia.org/resource/Robbert_Dijkgraaf, http://dbpedia.org/resource/Claud_Lovelace, http://dbpedia.org/resource/Matthew_Kleban] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 606, P: 0.514, R: 0.564, macro F1: 0.519, macro F1*: 0.538 -[INFO] Current time: 721 ms, average: 2721.01 ms/ques, entity detection: 423.22 ms/ques (linking tool: 330.92 ms/ques), relation detection: 896.76 ms/ques (similarity: 702.74 ms/ques), EDG generation: 49.56 ms/ques +[INFO] Cumulative metrics, sample: 606, P: 0.514, R: 0.564, macro F1: 0.538 +[INFO] Current question consumed time: 617 ms, average: 2472.49 ms/ques, entity detection: 378.31 ms/ques (linking tool: 298.84 ms/ques), relation detection: 833.88 ms/ques (similarity: 691.21 ms/ques), EDG generation: 49.28 ms/ques [INFO] Entity Linking P: 0.7475247524752475, R: 0.7021452145214522, micro F1: 0.7164466446644673, macro F1: 0.7241247175372062 [INFO] Relation Linking P: 0.5963971397139713, R: 0.5420792079207921, micro F1: 0.5572607260726076, macro F1: 0.5679423903255164 [INFO] Type Linking P: 0.5429042904290429, R: 0.5429042904290429, micro F1: 0.5429042904290429, macro F1: 0.5429042904290429 @@ -8508,8 +8508,8 @@ Question 606: How many apes are there? [INFO] Golden Answer: [41] [INFO] Predicted Answer: [] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 607, P: 0.513, R: 0.563, macro F1: 0.518, macro F1*: 0.537 -[INFO] Current time: 363 ms, average: 2717.13 ms/ques, entity detection: 422.70 ms/ques (linking tool: 330.45 ms/ques), relation detection: 895.29 ms/ques (similarity: 701.59 ms/ques), EDG generation: 49.49 ms/ques +[INFO] Cumulative metrics, sample: 607, P: 0.513, R: 0.563, macro F1: 0.537 +[INFO] Current question consumed time: 302 ms, average: 2468.91 ms/ques, entity detection: 377.83 ms/ques (linking tool: 298.41 ms/ques), relation detection: 832.51 ms/ques (similarity: 690.08 ms/ques), EDG generation: 49.20 ms/ques [INFO] Entity Linking P: 0.7462932454695222, R: 0.7009884678747941, micro F1: 0.7152663371773759, macro F1: 0.7229317608361564 [INFO] Relation Linking P: 0.5954146073585941, R: 0.5411861614497528, micro F1: 0.5563426688632623, macro F1: 0.5670067356462323 [INFO] Type Linking P: 0.5436573311367381, R: 0.5436573311367381, micro F1: 0.5436573311367381, macro F1: 0.5436573311367381 @@ -8522,8 +8522,8 @@ Question 607: What is a common nickname given to both Lyons Township high school [INFO] Golden Answer: [http://dbpedia.org/resource/Lion] [INFO] Predicted Answer: [http://dbpedia.org/resource/Lion] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 608, P: 0.514, R: 0.563, macro F1: 0.519, macro F1*: 0.538 -[INFO] Current time: 8127 ms, average: 2726.03 ms/ques, entity detection: 423.46 ms/ques (linking tool: 331.02 ms/ques), relation detection: 899.36 ms/ques (similarity: 703.30 ms/ques), EDG generation: 49.67 ms/ques +[INFO] Cumulative metrics, sample: 608, P: 0.514, R: 0.563, macro F1: 0.538 +[INFO] Current question consumed time: 6978 ms, average: 2476.33 ms/ques, entity detection: 378.39 ms/ques (linking tool: 298.85 ms/ques), relation detection: 836.11 ms/ques (similarity: 691.80 ms/ques), EDG generation: 49.40 ms/ques [INFO] Entity Linking P: 0.7467105263157895, R: 0.7006578947368421, micro F1: 0.7151864035087727, macro F1: 0.7229515550239235 [INFO] Relation Linking P: 0.5960800438596491, R: 0.5419407894736842, micro F1: 0.557072368421053, macro F1: 0.5677226287898477 [INFO] Type Linking P: 0.5444078947368421, R: 0.5444078947368421, micro F1: 0.5444078947368421, macro F1: 0.5444078947368421 @@ -8536,8 +8536,8 @@ Question 608: Where do beauty queens with brown hair reside? [INFO] Golden Answer: [http://dbpedia.org/resource/Minneapolis–Saint_Paul, http://dbpedia.org/resource/Lansing,_Michigan, http://dbpedia.org/resource/Panama, http://dbpedia.org/resource/Panama_City, http://dbpedia.org/resource/Japan, http://dbpedia.org/resource/Tokyo, http://dbpedia.org/resource/Los_Angeles, http://dbpedia.org/resource/Dominican_Republic, http://dbpedia.org/resource/Santo_Domingo, http://dbpedia.org/resource/Ensenada,_Baja_California, http://dbpedia.org/resource/South_Africa, http://dbpedia.org/resource/Johannesburg, http://dbpedia.org/resource/Atlanta, http://dbpedia.org/resource/Florida, http://dbpedia.org/resource/Miami, http://dbpedia.org/resource/Frisco,_Texas, http://dbpedia.org/resource/Utah, http://dbpedia.org/resource/Salt_Lake_City_metropolitan_area, http://dbpedia.org/resource/Mexico, http://dbpedia.org/resource/Mexico_City] [INFO] Predicted Answer: [http://dbpedia.org/resource/Belgrade, http://dbpedia.org/resource/Serbia] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 609, P: 0.513, R: 0.563, macro F1: 0.518, macro F1*: 0.537 -[INFO] Current time: 1052 ms, average: 2723.28 ms/ques, entity detection: 423.14 ms/ques (linking tool: 330.66 ms/ques), relation detection: 898.16 ms/ques (similarity: 702.33 ms/ques), EDG generation: 49.62 ms/ques +[INFO] Cumulative metrics, sample: 609, P: 0.513, R: 0.563, macro F1: 0.537 +[INFO] Current question consumed time: 893 ms, average: 2473.73 ms/ques, entity detection: 378.06 ms/ques (linking tool: 298.50 ms/ques), relation detection: 834.99 ms/ques (similarity: 690.83 ms/ques), EDG generation: 49.35 ms/ques [INFO] Entity Linking P: 0.7454844006568144, R: 0.6995073891625616, micro F1: 0.7140120415982493, macro F1: 0.7217644424540975 [INFO] Relation Linking P: 0.5951012588943623, R: 0.5410509031198686, micro F1: 0.5561576354679807, macro F1: 0.5667904077245112 [INFO] Type Linking P: 0.5451559934318555, R: 0.5451559934318555, micro F1: 0.5451559934318555, macro F1: 0.5451559934318555 @@ -8550,8 +8550,8 @@ Question 609: List the television shows one of whose distributor's divisions is [INFO] Golden Answer: [http://dbpedia.org/resource/Constantine_(TV_series), http://dbpedia.org/resource/Tiny_Toon_Adventures, http://dbpedia.org/resource/Access_Hollywood, http://dbpedia.org/resource/Arrow_(TV_series), http://dbpedia.org/resource/Mad_(TV_series), http://dbpedia.org/resource/Shazam!_(TV_series), http://dbpedia.org/resource/The_Jetsons, http://dbpedia.org/resource/The_Man_from_U.N.C.L.E., http://dbpedia.org/resource/The_New_Scooby-Doo_Movies, http://dbpedia.org/resource/Kung_Fu:_The_Legend_Continues, http://dbpedia.org/resource/Kung_Fu_(TV_series), http://dbpedia.org/resource/Superboy_(TV_series), http://dbpedia.org/resource/The_Parent_'Hood, http://dbpedia.org/resource/The_Vampire_Diaries, http://dbpedia.org/resource/The_Wayans_Bros., http://dbpedia.org/resource/Ice_&_Coco, http://dbpedia.org/resource/Life_Unexpected, http://dbpedia.org/resource/Mad_TV, http://dbpedia.org/resource/Mike_&_Molly, http://dbpedia.org/resource/Welcome_Back,_Kotter, http://dbpedia.org/resource/Don't_Go_to_Sleep, http://dbpedia.org/resource/Steven_Universe, http://dbpedia.org/resource/Captain_Planet_and_the_Planeteers, http://dbpedia.org/resource/Please_Don't_Eat_the_Daisies_(TV_series), http://dbpedia.org/resource/Queer_as_Folk_(2000_TV_series), http://dbpedia.org/resource/Taz-Mania, http://dbpedia.org/resource/The_Perils_of_Penelope_Pitstop, http://dbpedia.org/resource/The_Jenny_Jones_Show, http://dbpedia.org/resource/Uncle_Grandpa, http://dbpedia.org/resource/A_Pup_Named_Scooby-Doo, http://dbpedia.org/resource/Be_Cool,_Scooby-Doo!, http://dbpedia.org/resource/Butch_Cassidy_(TV_series), http://dbpedia.org/resource/Devlin_(TV_series), http://dbpedia.org/resource/Dynomutt,_Dog_Wonder, http://dbpedia.org/resource/Goober_and_the_Ghost_Chasers, http://dbpedia.org/resource/Jabberjaw, http://dbpedia.org/resource/Josie_and_the_Pussycats_(TV_series), http://dbpedia.org/resource/Life_Goes_On_(TV_series), http://dbpedia.org/resource/Richie_Rich_(1980_TV_series), http://dbpedia.org/resource/Scooby-Doo!_Legend_of_the_Phantosaur, http://dbpedia.org/resource/Scooby-Doo!_and_the_Reluctant_Werewolf, http://dbpedia.org/resource/Scooby-Doo!_in_Arabian_Nights, http://dbpedia.org/resource/Scooby-Doo_Meets_the_Boo_Brothers, http://dbpedia.org/resource/Scooby-Doo_and_Scrappy-Doo_(1979_TV_series), http://dbpedia.org/resource/Scooby-Doo_and_Scrappy-Doo_(1980_TV_series), http://dbpedia.org/resource/Scooby-Doo_and_the_Ghoul_School, http://dbpedia.org/resource/Scooby_Goes_Hollywood, http://dbpedia.org/resource/Significant_Mother, http://dbpedia.org/resource/Speed_Buggy, http://dbpedia.org/resource/The_Amazing_Chan_and_the_Chan_Clan, http://dbpedia.org/resource/The_Funky_Phantom, http://dbpedia.org/resource/The_Scooby-Doo/Dynomutt_Hour, http://dbpedia.org/resource/The_Scooby-Doo_Show, http://dbpedia.org/resource/What's_New,_Scooby-Doo%3F, http://dbpedia.org/resource/Without_a_Trace, http://dbpedia.org/resource/Wonder_Wheels, http://dbpedia.org/resource/My_Sister_Sam, http://dbpedia.org/resource/Privileged_(TV_series), http://dbpedia.org/resource/The_Addams_Family_(1973_animated_series), http://dbpedia.org/resource/The_Wizard_of_Oz_(TV_series), http://dbpedia.org/resource/Way_Out_(TV_series), http://dbpedia.org/resource/Let's_Ask_America, http://dbpedia.org/resource/Love_Connection, http://dbpedia.org/resource/5_Para_A_Meia-Noite, http://dbpedia.org/resource/Big_Shamus,_Little_Shamus, http://dbpedia.org/resource/The_Forgotten_(TV_series), http://dbpedia.org/resource/Trump_Card, http://dbpedia.org/resource/Bunnicula_(TV_series), http://dbpedia.org/resource/Dink,_the_Little_Dinosaur, http://dbpedia.org/resource/Droopy,_Master_Detective, http://dbpedia.org/resource/Frosty's_Winter_Wonderland, http://dbpedia.org/resource/Tales_from_the_Cryptkeeper, http://dbpedia.org/resource/Apple's_Way, http://dbpedia.org/resource/Golden_Boy_(TV_series), http://dbpedia.org/resource/The_Lying_Game, http://dbpedia.org/resource/He's_Your_Dog,_Charlie_Brown, http://dbpedia.org/resource/You're_in_Love,_Charlie_Brown, http://dbpedia.org/resource/Jack_&_Bobby, http://dbpedia.org/resource/3rd_Degree_(game_show), http://dbpedia.org/resource/Mortal_Kombat:_Defenders_of_the_Realm, http://dbpedia.org/resource/The_Little_Troll_Prince, http://dbpedia.org/resource/Yogi_the_Easter_Bear, http://dbpedia.org/resource/The_Pac-Man/Little_Rascals/Richie_Rich_Show, http://dbpedia.org/resource/History_of_Rock_'n'_Roll_(TV_series), http://dbpedia.org/resource/The_Pussycat_Dolls_Present:_Girlicious, http://dbpedia.org/resource/Containment_(TV_series), http://dbpedia.org/resource/Believe_(TV_series), http://dbpedia.org/resource/Sex_and_the_City, http://dbpedia.org/resource/77_Sunset_Strip, http://dbpedia.org/resource/Crime_Story_(TV_series), http://dbpedia.org/resource/Falcon_Crest, http://dbpedia.org/resource/Rizzoli_&_Isles, http://dbpedia.org/resource/Samurai_Jack, http://dbpedia.org/resource/The_Mentalist, http://dbpedia.org/resource/Beyond_Westworld, http://dbpedia.org/resource/The_Paul_Reiser_Show, http://dbpedia.org/resource/The_Voice_(U.S._TV_series), http://dbpedia.org/resource/The_Whole_Truth_(TV_series), http://dbpedia.org/resource/Westworld_(TV_series), http://dbpedia.org/resource/2_Stupid_Dogs, http://dbpedia.org/resource/Almost_Human_(TV_series), http://dbpedia.org/resource/Mystery_Island, http://dbpedia.org/resource/Rubik,_the_Amazing_Cube, http://dbpedia.org/resource/SWAT_Kats:_The_Radical_Squadron, http://dbpedia.org/resource/Step_by_Step_(TV_series), http://dbpedia.org/resource/Stroker_and_Hoop, http://dbpedia.org/resource/Superjail!, http://dbpedia.org/resource/The_Chimp_Channel, http://dbpedia.org/resource/The_Problem_Solverz, http://dbpedia.org/resource/The_Thin_Man_(TV_series), http://dbpedia.org/resource/Wake,_Rattle,_and_Roll, http://dbpedia.org/resource/Scooby-Doo,_Where_Are_You!, http://dbpedia.org/resource/Suddenly_Susan, http://dbpedia.org/resource/Bugs_Bunny's_Bustin'_Out_All_Over, http://dbpedia.org/resource/Daffy_Duck_and_Porky_Pig_Meet_the_Groovie_Goolies, http://dbpedia.org/resource/Fred_and_Barney_Meet_The_Thing, http://dbpedia.org/resource/Fred_and_Barney_Meet_the_Shmoo, http://dbpedia.org/resource/Freddy's_Nightmares, http://dbpedia.org/resource/Galaxy_Goof-Ups, http://dbpedia.org/resource/Hanna-Barbera's_50th:_A_Yabba_Dabba_Doo_Celebration, http://dbpedia.org/resource/Heathcliff_(1980_TV_series), http://dbpedia.org/resource/Rockin'_with_Judy_Jetson, http://dbpedia.org/resource/The_Atom_Ant/Secret_Squirrel_Show, http://dbpedia.org/resource/The_Flintstone_Comedy_Hour, http://dbpedia.org/resource/The_Flintstone_Funnies, http://dbpedia.org/resource/The_Flintstone_Kids, http://dbpedia.org/resource/The_Jetsons_Meet_the_Flintstones, http://dbpedia.org/resource/The_Middle_(TV_series), http://dbpedia.org/resource/The_New_Fred_and_Barney_Show, http://dbpedia.org/resource/The_Pebbles_and_Bamm-Bamm_Show, http://dbpedia.org/resource/A_Flintstones_Christmas_Carol, http://dbpedia.org/resource/Cave_Kids, http://dbpedia.org/resource/Sealab_2020, http://dbpedia.org/resource/The_Further_Adventures_of_SuperTed, http://dbpedia.org/resource/The_Huckleberry_Hound_Show, http://dbpedia.org/resource/The_Quick_Draw_McGraw_Show, http://dbpedia.org/resource/The_Roman_Holidays, http://dbpedia.org/resource/Doc_Elliot, http://dbpedia.org/resource/Dr._Seuss'_How_the_Grinch_Stole_Christmas!_(TV_special), http://dbpedia.org/resource/The_Real_(TV_series), http://dbpedia.org/resource/Lifechangers, http://dbpedia.org/resource/Crime_Watch_Daily, http://dbpedia.org/resource/Eunice_(film), http://dbpedia.org/resource/Casper's_First_Christmas, http://dbpedia.org/resource/Legally_Mad, http://dbpedia.org/resource/Jack_Frost_(TV_special), http://dbpedia.org/resource/Teen_Titans:_Trouble_in_Tokyo, http://dbpedia.org/resource/Ann_Jillian_(TV_series), http://dbpedia.org/resource/Yogi_Bear_&_Friends, http://dbpedia.org/resource/Supernatural:_Bloodlines, http://dbpedia.org/resource/Gossip_Girl:_Acapulco, http://dbpedia.org/resource/Babylon_5, http://dbpedia.org/resource/Max_Headroom_(TV_series), http://dbpedia.org/resource/Supernatural_(U.S._TV_series), http://dbpedia.org/resource/The_Flintstones, http://dbpedia.org/resource/ALF_(TV_series), http://dbpedia.org/resource/Adventures_of_Superman_(TV_series), http://dbpedia.org/resource/Beetlejuice_(TV_series), http://dbpedia.org/resource/Cold_Case, http://dbpedia.org/resource/Hellcats, http://dbpedia.org/resource/Horton_Hears_a_Who!_(TV_special), http://dbpedia.org/resource/Morton_&_Hayes, http://dbpedia.org/resource/Murphy_Brown, http://dbpedia.org/resource/Night_Visions_(TV_series), http://dbpedia.org/resource/Perfect_Strangers_(TV_series), http://dbpedia.org/resource/Person_of_Interest_(TV_series), http://dbpedia.org/resource/Pinky_and_the_Brain, http://dbpedia.org/resource/Shameless_(U.S._TV_series), http://dbpedia.org/resource/Smallville, http://dbpedia.org/resource/Supergirl_(TV_series), http://dbpedia.org/resource/Terminator:_The_Sarah_Connor_Chronicles, http://dbpedia.org/resource/The_New_Adventures_of_Superman_(TV_series), http://dbpedia.org/resource/Fringe_(TV_series), http://dbpedia.org/resource/Time_Trax, http://dbpedia.org/resource/Challenge_of_the_Super_Friends, http://dbpedia.org/resource/DC_Nation_Shorts, http://dbpedia.org/resource/Super_Friends:_The_Legendary_Super_Powers_Show, http://dbpedia.org/resource/The_World's_Greatest_Super_Friends, http://dbpedia.org/resource/Bethenny_(TV_series), http://dbpedia.org/resource/Judge_Mathis, http://dbpedia.org/resource/The_Comeback_(TV_series), http://dbpedia.org/resource/The_Originals_(TV_series), http://dbpedia.org/resource/The_Secret_Circle_(TV_series), http://dbpedia.org/resource/Baby_Looney_Tunes, http://dbpedia.org/resource/Wabbit_(TV_series), http://dbpedia.org/resource/Anderson_Live, http://dbpedia.org/resource/Second_Chance_(game_show), http://dbpedia.org/resource/GE_True, http://dbpedia.org/resource/Memphis_Beat, http://dbpedia.org/resource/Southland_(TV_series), http://dbpedia.org/resource/The_Last_Ship_(TV_series), http://dbpedia.org/resource/Gilligan's_Planet, http://dbpedia.org/resource/Jesse_(TV_series), http://dbpedia.org/resource/Mickey_(TV_series), http://dbpedia.org/resource/Notes_from_the_Underbelly, http://dbpedia.org/resource/The_Life_&_Times_of_Tim, http://dbpedia.org/resource/The_New_Adventures_of_Gilligan, http://dbpedia.org/resource/The_War_at_Home_(TV_series), http://dbpedia.org/resource/The_Year_Without_a_Santa_Claus, http://dbpedia.org/resource/Traveler_(TV_series), http://dbpedia.org/resource/TMZ_on_TV, http://dbpedia.org/resource/Hostages_(American_TV_series), http://dbpedia.org/resource/The_Hero_(1966_TV_series), http://dbpedia.org/resource/Black_Sash_(TV_series), http://dbpedia.org/resource/Chase_(2010_TV_series), http://dbpedia.org/resource/Harry's_Girls, http://dbpedia.org/resource/Lucky_(TV_series), http://dbpedia.org/resource/Undercovers_(TV_series), http://dbpedia.org/resource/In_Person_(1996_TV_series), http://dbpedia.org/resource/Just_Keke, http://dbpedia.org/resource/Casper's_Halloween_Special, http://dbpedia.org/resource/Family_Matters, http://dbpedia.org/resource/Martin_(TV_series), http://dbpedia.org/resource/My_Name_Is_Bill_W., http://dbpedia.org/resource/Nikita_(TV_series), http://dbpedia.org/resource/The_100_(TV_series), http://dbpedia.org/resource/The_Bugs_Bunny_Show, http://dbpedia.org/resource/The_People's_Court, http://dbpedia.org/resource/The_Powerpuff_Girls, http://dbpedia.org/resource/The_Waltons, http://dbpedia.org/resource/Better_Days_(TV_series), http://dbpedia.org/resource/Medical_Center_(TV_series), http://dbpedia.org/resource/My_Favorite_Martian, http://dbpedia.org/resource/One_Tree_Hill_(TV_series), http://dbpedia.org/resource/Search_(TV_series), http://dbpedia.org/resource/CHiPs, http://dbpedia.org/resource/Scooby-Doo!_Mystery_Incorporated, http://dbpedia.org/resource/Elf:_Buddy's_Musical_Christmas, http://dbpedia.org/resource/Everwood, http://dbpedia.org/resource/Harry's_Law, http://dbpedia.org/resource/Karate_Kommandos, http://dbpedia.org/resource/The_Following, http://dbpedia.org/resource/The_Jimmy_Stewart_Show, http://dbpedia.org/resource/The_New_Adventures_of_Old_Christine, http://dbpedia.org/resource/The_Tom_and_Jerry_Show_(2014_TV_series), http://dbpedia.org/resource/Veronica's_Closet, http://dbpedia.org/resource/Greetings_from_Tucson, http://dbpedia.org/resource/IZombie_(TV_series), http://dbpedia.org/resource/Our_House_(1986_TV_series), http://dbpedia.org/resource/Run,_Buddy,_Run, http://dbpedia.org/resource/Fuller_House_(TV_series), http://dbpedia.org/resource/Invasion_(TV_series), http://dbpedia.org/resource/Two_of_a_Kind_(U.S._TV_series), http://dbpedia.org/resource/Bugs_Bunny's_Howl-oween_Special, http://dbpedia.org/resource/Ed,_Edd_n_Eddy's_Big_Picture_Show, http://dbpedia.org/resource/Christmas_Comes_to_Pac-Land, http://dbpedia.org/resource/Tupu, http://dbpedia.org/resource/Batman_(TV_series), http://dbpedia.org/resource/Dallas_(1978_TV_series), http://dbpedia.org/resource/Lois_&_Clark:_The_New_Adventures_of_Superman, http://dbpedia.org/resource/The_Fresh_Prince_of_Bel-Air, http://dbpedia.org/resource/The_New_Dick_Van_Dyke_Show, http://dbpedia.org/resource/The_Rosie_O'Donnell_Show, http://dbpedia.org/resource/The_West_Wing, http://dbpedia.org/resource/Will_&_Grace, http://dbpedia.org/resource/Gilligan's_Island, http://dbpedia.org/resource/Mayberry_R.F.D., http://dbpedia.org/resource/The_Pirates_of_Dark_Water, http://dbpedia.org/resource/F_Troop, http://dbpedia.org/resource/Teen_Titans_(TV_series), http://dbpedia.org/resource/Gavilan_(TV_series), http://dbpedia.org/resource/Foster's_Home_for_Imaginary_Friends, http://dbpedia.org/resource/Clue_Club, http://dbpedia.org/resource/Nip/Tuck, http://dbpedia.org/resource/The_Hanna-Barbera_Happy_Hour, http://dbpedia.org/resource/The_Robonic_Stooges, http://dbpedia.org/resource/The_Skatebirds, http://dbpedia.org/resource/$h*!_My_Dad_Says, http://dbpedia.org/resource/Eastwick_(TV_series), http://dbpedia.org/resource/Mom_(TV_series), http://dbpedia.org/resource/The_Asphalt_Jungle_(TV_series), http://dbpedia.org/resource/The_Funtastic_World_of_Hanna-Barbera, http://dbpedia.org/resource/The_Tyra_Banks_Show, http://dbpedia.org/resource/Benji,_Zax_&_the_Alien_Prince, http://dbpedia.org/resource/Birdman_and_the_Galaxy_Trio, http://dbpedia.org/resource/Casper_and_the_Angels, http://dbpedia.org/resource/Cattanooga_Cats, http://dbpedia.org/resource/Dastardly_and_Muttley_in_Their_Flying_Machines, http://dbpedia.org/resource/Dumb_and_Dumber_(TV_series), http://dbpedia.org/resource/Fantastic_Max, http://dbpedia.org/resource/Going_Bananas_(U.S._series), http://dbpedia.org/resource/Help!..._It's_the_Hair_Bear_Bunch!, http://dbpedia.org/resource/Hong_Kong_Phooey, http://dbpedia.org/resource/Inch_High,_Private_Eye, http://dbpedia.org/resource/Jokebook, http://dbpedia.org/resource/Korg:_70,000_B.C., http://dbpedia.org/resource/Laurel_and_Hardy_(animated_series), http://dbpedia.org/resource/Longmire_(TV_series), http://dbpedia.org/resource/Moby_Dick_and_Mighty_Mightor, http://dbpedia.org/resource/Popeye_and_Son, http://dbpedia.org/resource/Sky_Commanders, http://dbpedia.org/resource/Snorks, http://dbpedia.org/resource/Space_Ghost_(TV_series), http://dbpedia.org/resource/Space_Stars, http://dbpedia.org/resource/Super_Friends_(1973_TV_series), http://dbpedia.org/resource/Super_Friends_(1980_TV_series), http://dbpedia.org/resource/The_Abbott_and_Costello_Cartoon_Show, http://dbpedia.org/resource/The_Adventures_of_Don_Coyote_and_Sancho_Panda, http://dbpedia.org/resource/The_All-New_Super_Friends_Hour, http://dbpedia.org/resource/The_Biskitts, http://dbpedia.org/resource/The_Little_Rascals_(animated_TV_series), http://dbpedia.org/resource/The_New_Yogi_Bear_Show, http://dbpedia.org/resource/The_Super_Globetrotters, http://dbpedia.org/resource/The_Super_Powers_Team:_Galactic_Guardians, http://dbpedia.org/resource/The_Yogi_Bear_Show, http://dbpedia.org/resource/These_Are_the_Days_(TV_series), http://dbpedia.org/resource/Tom_and_Jerry:_The_Mansion_Cat, http://dbpedia.org/resource/Wheelie_and_the_Chopper_Bunch, http://dbpedia.org/resource/Where's_Huddles%3F, http://dbpedia.org/resource/Wildfire_(1986_TV_series), http://dbpedia.org/resource/Yo_Yogi!, http://dbpedia.org/resource/Yogi's_Space_Race, http://dbpedia.org/resource/Yogi's_Treasure_Hunt, http://dbpedia.org/resource/You_Don't_Say!, http://dbpedia.org/resource/Young_Robin_Hood, http://dbpedia.org/resource/Bachelor_in_Paradise_(TV_series), http://dbpedia.org/resource/Hangin'_with_Mr._Cooper, http://dbpedia.org/resource/Just_the_Ten_of_Us, http://dbpedia.org/resource/Suburgatory, http://dbpedia.org/resource/The_Bachelor_(U.S._TV_series), http://dbpedia.org/resource/V_(2009_TV_series), http://dbpedia.org/resource/Growing_Pains:_Return_of_the_Seavers, http://dbpedia.org/resource/Surviving:_A_Family_in_Crisis, http://dbpedia.org/resource/The_Growing_Pains_Movie, http://dbpedia.org/resource/The_Jackie_Thomas_Show, http://dbpedia.org/resource/Hart_of_Dixie, http://dbpedia.org/resource/Wonder_Woman_(1974_film), http://dbpedia.org/resource/For_Your_Love_(TV_series), http://dbpedia.org/resource/Men_in_Trees, http://dbpedia.org/resource/Super_Fun_Night, http://dbpedia.org/resource/Carol_Burnett_&_Company, http://dbpedia.org/resource/Rituals_(TV_series), http://dbpedia.org/resource/The_Bonnie_Hunt_Show, http://dbpedia.org/resource/Baby_Blues_(U.S._TV_series), http://dbpedia.org/resource/Mina_and_the_Count, http://dbpedia.org/resource/Judge_Jeanine_Pirro, http://dbpedia.org/resource/Tom_and_Jerry's_Funhouse_on_TBS, http://dbpedia.org/resource/Queen_Sugar, http://dbpedia.org/resource/Adam's_Rib_(TV_series), http://dbpedia.org/resource/Animaniacs, http://dbpedia.org/resource/Batman:_The_Animated_Series, http://dbpedia.org/resource/ER_(TV_series), http://dbpedia.org/resource/Gilmore_Girls, http://dbpedia.org/resource/Gossip_Girl, http://dbpedia.org/resource/Gotham_(TV_series), http://dbpedia.org/resource/Knots_Landing, http://dbpedia.org/resource/Megas_XLR, http://dbpedia.org/resource/The_Adventures_of_Batman, http://dbpedia.org/resource/The_Batman/Superman_Hour, http://dbpedia.org/resource/The_Big_Bang_Theory, http://dbpedia.org/resource/The_Eleventh_Hour_(1962_TV_series), http://dbpedia.org/resource/The_Flash_(1990_TV_series), http://dbpedia.org/resource/The_Great_Grape_Ape_Show, http://dbpedia.org/resource/The_New_Adventures_of_Batman, http://dbpedia.org/resource/The_Twilight_Zone_(2002_TV_series), http://dbpedia.org/resource/Two_and_a_Half_Men, http://dbpedia.org/resource/Veronica_Mars, http://dbpedia.org/resource/2_Broke_Girls, http://dbpedia.org/resource/Fantastic_Four_(1967_TV_series), http://dbpedia.org/resource/Joey_(TV_series), http://dbpedia.org/resource/Jonny_Quest_(TV_series), http://dbpedia.org/resource/The_Bachelorette, http://dbpedia.org/resource/The_Ellen_DeGeneres_Show, http://dbpedia.org/resource/The_Real_Adventures_of_Jonny_Quest, http://dbpedia.org/resource/What_I_Like_About_You_(TV_series), http://dbpedia.org/resource/The_Dukes_of_Hazzard, http://dbpedia.org/resource/Chuck_(TV_series), http://dbpedia.org/resource/Eight_Is_Enough, http://dbpedia.org/resource/Hank_(2009_TV_series), http://dbpedia.org/resource/La_Femme_Nikita, http://dbpedia.org/resource/Living_Single, http://dbpedia.org/resource/Man_from_Atlantis, http://dbpedia.org/resource/Roc_(TV_series), http://dbpedia.org/resource/Superman_(TV_series), http://dbpedia.org/resource/The_Kwicky_Koala_Show, http://dbpedia.org/resource/Third_Watch, http://dbpedia.org/resource/Blindspot_(TV_series), http://dbpedia.org/resource/Chico_and_the_Man, http://dbpedia.org/resource/China_Beach, http://dbpedia.org/resource/Regular_Show, http://dbpedia.org/resource/Revolution_(TV_series), http://dbpedia.org/resource/The_Banana_Splits, http://dbpedia.org/resource/The_Hogan_Family, http://dbpedia.org/resource/The_Mysteries_of_Laura, http://dbpedia.org/resource/The_Smurfs_(TV_series), http://dbpedia.org/resource/Falling_Skies, http://dbpedia.org/resource/Spenser:_For_Hire, http://dbpedia.org/resource/The_Kid_Super_Power_Hour_with_Shazam!, http://dbpedia.org/resource/All_of_Us, http://dbpedia.org/resource/In_the_House_(TV_series), http://dbpedia.org/resource/Moonlight_(TV_series), http://dbpedia.org/resource/Wacky_Races, http://dbpedia.org/resource/Charlie_Brown's_All-Stars, http://dbpedia.org/resource/Mission_Hill, http://dbpedia.org/resource/The_Adventures_of_Gulliver, http://dbpedia.org/resource/The_Carrie_Diaries_(TV_series), http://dbpedia.org/resource/The_Dukes, http://dbpedia.org/resource/Daddy's_Girls_(1994_TV_series), http://dbpedia.org/resource/Duck_Dodgers_(TV_series), http://dbpedia.org/resource/Forever_(U.S._TV_series), http://dbpedia.org/resource/The_Leftovers_(TV_series), http://dbpedia.org/resource/Godzilla_(animated_series), http://dbpedia.org/resource/Pretty_Little_Liars_(TV_series), http://dbpedia.org/resource/The_Nightmare_Room, http://dbpedia.org/resource/The_Real_Housewives_of_New_York_City, http://dbpedia.org/resource/Twas_the_Night_Before_Christmas_(1974_TV_special), http://dbpedia.org/resource/The_Superman/Aquaman_Hour_of_Adventure, http://dbpedia.org/resource/It's_Me_or_the_Dog, http://dbpedia.org/resource/Ace_Ventura:_Pet_Detective_(TV_series), http://dbpedia.org/resource/Are_You_There,_Chelsea%3F, http://dbpedia.org/resource/Ball_Four_(TV_series), http://dbpedia.org/resource/Buford_and_the_Galloping_Ghost, http://dbpedia.org/resource/Carnie!, http://dbpedia.org/resource/Cult_(TV_series), http://dbpedia.org/resource/Dark_Justice, http://dbpedia.org/resource/Fun_House_(U.S._game_show), http://dbpedia.org/resource/Ground_Floor, http://dbpedia.org/resource/He_&_She, http://dbpedia.org/resource/I_Hate_My_Teenage_Daughter, http://dbpedia.org/resource/Little_Big_Shots, http://dbpedia.org/resource/Meatballs_&_Spaghetti, http://dbpedia.org/resource/Miami_Medical, http://dbpedia.org/resource/Moral_Court, http://dbpedia.org/resource/On_Our_Own_(1977_TV_series), http://dbpedia.org/resource/Pandamonium_(TV_series), http://dbpedia.org/resource/Partners_(2012_TV_series), http://dbpedia.org/resource/Police_Academy_(TV_series), http://dbpedia.org/resource/Ravenswood_(TV_series), http://dbpedia.org/resource/Superior_Court_(TV_series), http://dbpedia.org/resource/Surviving_Jack, http://dbpedia.org/resource/The_Army_Show, http://dbpedia.org/resource/The_Blue_Knight_(TV_series), http://dbpedia.org/resource/The_Family_Man_(1990_TV_series), http://dbpedia.org/resource/The_Mountain_(TV_series), http://dbpedia.org/resource/The_Practice_(1976_TV_series), http://dbpedia.org/resource/The_Tomorrow_People_(U.S._TV_series), http://dbpedia.org/resource/The_Trouble_with_Larry, http://dbpedia.org/resource/A_to_Z_(TV_series), http://dbpedia.org/resource/Moral_Orel, http://dbpedia.org/resource/Hot_Wheels_World_Race, http://dbpedia.org/resource/Scruffy_(1980_film), http://dbpedia.org/resource/Kodiak_(TV_series), http://dbpedia.org/resource/The_Best_Times_(TV_show), http://dbpedia.org/resource/Eye_to_Eye_(U.S._TV_series), http://dbpedia.org/resource/Helter_Skelter_(1976_film), http://dbpedia.org/resource/Mister_T_(TV_series), http://dbpedia.org/resource/PopClips, http://dbpedia.org/resource/Secret_Squirrel, http://dbpedia.org/resource/Pushing_Daisies, http://dbpedia.org/resource/The_Flintstone_Comedy_Show_(1980_TV_series), http://dbpedia.org/resource/Eleventh_Hour_(U.S._TV_series), http://dbpedia.org/resource/George_Lopez_(TV_series), http://dbpedia.org/resource/The_Colbys, http://dbpedia.org/resource/The_Tom_and_Jerry_Comedy_Show, http://dbpedia.org/resource/The_Tom_and_Jerry_Show_(1975_TV_series), http://dbpedia.org/resource/Tom_&_Jerry_Kids, http://dbpedia.org/resource/Tom_and_Jerry_Tales, http://dbpedia.org/resource/Wait_Till_Your_Father_Gets_Home, http://dbpedia.org/resource/Grim_&_Evil, http://dbpedia.org/resource/I_Am_Weasel, http://dbpedia.org/resource/Mike_Tyson_Mysteries, http://dbpedia.org/resource/Secret_Mountain_Fort_Awesome, http://dbpedia.org/resource/The_Flintstones:_On_the_Rocks, http://dbpedia.org/resource/Tower_Prep, http://dbpedia.org/resource/We_Bare_Bears, http://dbpedia.org/resource/What_a_Cartoon!, http://dbpedia.org/resource/Alcatraz_(TV_series), http://dbpedia.org/resource/Salem's_Lot_(1979_miniseries), http://dbpedia.org/resource/Green_Lantern:_The_Animated_Series, http://dbpedia.org/resource/Legends_of_Tomorrow, http://dbpedia.org/resource/Planet_Earth_(film), http://dbpedia.org/resource/Extra_(TV_program), http://dbpedia.org/resource/Fish_Police_(TV_series), http://dbpedia.org/resource/Better_with_You, http://dbpedia.org/resource/The_Deliberate_Stranger, http://dbpedia.org/resource/Stalker_(TV_series), http://dbpedia.org/resource/666_Park_Avenue, http://dbpedia.org/resource/Fangface, http://dbpedia.org/resource/Going_Places_(U.S._TV_series), http://dbpedia.org/resource/Goldie_Gold_and_Action_Jack, http://dbpedia.org/resource/Just_Our_Luck, http://dbpedia.org/resource/Newsreaders, http://dbpedia.org/resource/Our_Family_Honor, http://dbpedia.org/resource/Romantically_Challenged, http://dbpedia.org/resource/Selfie_(TV_series), http://dbpedia.org/resource/The_Neighbors_(game_show), http://dbpedia.org/resource/The_Nine_(TV_series), http://dbpedia.org/resource/The_Norm_Show, http://dbpedia.org/resource/The_Puppy's_Further_Adventures, http://dbpedia.org/resource/Under_Cover_(TV_series), http://dbpedia.org/resource/Work_It_(TV_series), http://dbpedia.org/resource/Danger_Island_(TV_series), http://dbpedia.org/resource/Hot_Wheels_Battle_Force_5, http://dbpedia.org/resource/Jana_of_the_Jungle, http://dbpedia.org/resource/Hot_Wheels_AcceleRacers, http://dbpedia.org/resource/Mortal_Kombat:_Conquest, http://dbpedia.org/resource/The_Office_(1995_TV_series), http://dbpedia.org/resource/The_Comic_Strip_(TV_series), http://dbpedia.org/resource/Green_Eggs_and_Ham_(TV_series), http://dbpedia.org/resource/Valley_of_the_Dinosaurs, http://dbpedia.org/resource/Family_Affair, http://dbpedia.org/resource/It_(1990_film), http://dbpedia.org/resource/The_Flash_(2014_TV_series), http://dbpedia.org/resource/Human_Target_(2010_TV_series), http://dbpedia.org/resource/Brimstone_(TV_series), http://dbpedia.org/resource/Riverdale_(2016_TV_series), http://dbpedia.org/resource/Freddie, http://dbpedia.org/resource/Major_Crimes, http://dbpedia.org/resource/Pinocchio's_Christmas, http://dbpedia.org/resource/The_Closer, http://dbpedia.org/resource/Laff-A-Lympics, http://dbpedia.org/resource/Birds_of_Prey_(TV_series), http://dbpedia.org/resource/Big_Shots_(TV_series), http://dbpedia.org/resource/The_George_Carlin_Show, http://dbpedia.org/resource/The_Mystic_Warrior, http://dbpedia.org/resource/Police_Academy:_The_Series, http://dbpedia.org/resource/Committed_(2001_TV_series), http://dbpedia.org/resource/ThunderCats_(2011_TV_series), http://dbpedia.org/resource/The_Mullets, http://dbpedia.org/resource/The_First_Christmas:_The_Story_of_the_First_Christmas_Snow, http://dbpedia.org/resource/The_New_Adventures_of_Lucky_Luke, http://dbpedia.org/resource/The_All_New_Popeye_Hour] [INFO] Predicted Answer: [http://dbpedia.org/resource/Constantine_(TV_series), http://dbpedia.org/resource/Friends, http://dbpedia.org/resource/Tiny_Toon_Adventures, http://dbpedia.org/resource/Access_Hollywood, http://dbpedia.org/resource/Arrow_(TV_series), http://dbpedia.org/resource/Mad_(TV_series), http://dbpedia.org/resource/The_Jetsons, http://dbpedia.org/resource/The_Man_from_U.N.C.L.E., http://dbpedia.org/resource/The_New_Scooby-Doo_Movies, http://dbpedia.org/resource/Kung_Fu:_The_Legend_Continues, http://dbpedia.org/resource/Kung_Fu_(TV_series), http://dbpedia.org/resource/Superboy_(TV_series), http://dbpedia.org/resource/The_Parent_'Hood, http://dbpedia.org/resource/The_Vampire_Diaries, http://dbpedia.org/resource/The_Wayans_Bros., http://dbpedia.org/resource/Ice_&_Coco, http://dbpedia.org/resource/Life_Unexpected, http://dbpedia.org/resource/Mad_TV, http://dbpedia.org/resource/Mike_&_Molly, http://dbpedia.org/resource/Welcome_Back,_Kotter, http://dbpedia.org/resource/Don't_Go_to_Sleep, http://dbpedia.org/resource/Steven_Universe, http://dbpedia.org/resource/Captain_Planet_and_the_Planeteers, http://dbpedia.org/resource/Please_Don't_Eat_the_Daisies_(TV_series), http://dbpedia.org/resource/Queer_as_Folk_(2000_TV_series), http://dbpedia.org/resource/Taz-Mania, http://dbpedia.org/resource/The_Perils_of_Penelope_Pitstop, http://dbpedia.org/resource/Mama's_Family, http://dbpedia.org/resource/The_Jenny_Jones_Show, http://dbpedia.org/resource/Uncle_Grandpa, http://dbpedia.org/resource/A_Pup_Named_Scooby-Doo, http://dbpedia.org/resource/Be_Cool,_Scooby-Doo!, http://dbpedia.org/resource/Devlin_(TV_series), http://dbpedia.org/resource/Dynomutt,_Dog_Wonder, http://dbpedia.org/resource/Goober_and_the_Ghost_Chasers, http://dbpedia.org/resource/Jabberjaw, http://dbpedia.org/resource/Josie_and_the_Pussycats_(TV_series), http://dbpedia.org/resource/Life_Goes_On_(TV_series), http://dbpedia.org/resource/Richie_Rich_(1980_TV_series), http://dbpedia.org/resource/Scooby-Doo!_Legend_of_the_Phantosaur, http://dbpedia.org/resource/Scooby-Doo!_and_the_Reluctant_Werewolf, http://dbpedia.org/resource/Scooby-Doo!_in_Arabian_Nights, http://dbpedia.org/resource/Scooby-Doo_Meets_the_Boo_Brothers, http://dbpedia.org/resource/Scooby-Doo_and_Scrappy-Doo_(1979_TV_series), http://dbpedia.org/resource/Scooby-Doo_and_Scrappy-Doo_(1980_TV_series), http://dbpedia.org/resource/Scooby-Doo_and_the_Ghoul_School, http://dbpedia.org/resource/Scooby_Goes_Hollywood, http://dbpedia.org/resource/Significant_Mother, http://dbpedia.org/resource/Speed_Buggy, http://dbpedia.org/resource/The_Amazing_Chan_and_the_Chan_Clan, http://dbpedia.org/resource/The_Scooby-Doo/Dynomutt_Hour, http://dbpedia.org/resource/The_Scooby-Doo_Show, http://dbpedia.org/resource/What's_New,_Scooby-Doo%3F, http://dbpedia.org/resource/Without_a_Trace, http://dbpedia.org/resource/Wonder_Wheels, http://dbpedia.org/resource/Cave-In!, http://dbpedia.org/resource/My_Sister_Sam, http://dbpedia.org/resource/Privileged_(TV_series), http://dbpedia.org/resource/The_Addams_Family_(1973_animated_series), http://dbpedia.org/resource/The_Night_the_Bridge_Fell_Down, http://dbpedia.org/resource/The_Wizard_of_Oz_(TV_series), http://dbpedia.org/resource/Way_Out_(TV_series), http://dbpedia.org/resource/Let's_Ask_America, http://dbpedia.org/resource/Love_Connection, http://dbpedia.org/resource/5_Para_A_Meia-Noite, http://dbpedia.org/resource/Big_Shamus,_Little_Shamus, http://dbpedia.org/resource/The_Forgotten_(TV_series), http://dbpedia.org/resource/Trump_Card, http://dbpedia.org/resource/Bunnicula_(TV_series), http://dbpedia.org/resource/Dink,_the_Little_Dinosaur, http://dbpedia.org/resource/Droopy,_Master_Detective, http://dbpedia.org/resource/Frosty's_Winter_Wonderland, http://dbpedia.org/resource/Tales_from_the_Cryptkeeper, http://dbpedia.org/resource/Apple's_Way, http://dbpedia.org/resource/Golden_Boy_(TV_series), http://dbpedia.org/resource/The_Lying_Game, http://dbpedia.org/resource/He's_Your_Dog,_Charlie_Brown, http://dbpedia.org/resource/You're_in_Love,_Charlie_Brown, http://dbpedia.org/resource/Jack_&_Bobby, http://dbpedia.org/resource/3rd_Degree_(game_show), http://dbpedia.org/resource/Mortal_Kombat:_Defenders_of_the_Realm, http://dbpedia.org/resource/The_Little_Troll_Prince, http://dbpedia.org/resource/Yogi_the_Easter_Bear, http://dbpedia.org/resource/The_Pac-Man/Little_Rascals/Richie_Rich_Show, http://dbpedia.org/resource/History_of_Rock_'n'_Roll_(TV_series), http://dbpedia.org/resource/The_Pussycat_Dolls_Present:_Girlicious, http://dbpedia.org/resource/Containment_(TV_series), http://dbpedia.org/resource/Believe_(TV_series), http://dbpedia.org/resource/Sex_and_the_City, http://dbpedia.org/resource/77_Sunset_Strip, http://dbpedia.org/resource/Crime_Story_(TV_series), http://dbpedia.org/resource/Falcon_Crest, http://dbpedia.org/resource/Rizzoli_&_Isles, http://dbpedia.org/resource/Samurai_Jack, http://dbpedia.org/resource/The_Mentalist, http://dbpedia.org/resource/Beyond_Westworld, http://dbpedia.org/resource/The_Paul_Reiser_Show, http://dbpedia.org/resource/The_Voice_(U.S._TV_series), http://dbpedia.org/resource/The_Whole_Truth_(TV_series), http://dbpedia.org/resource/Westworld_(TV_series), http://dbpedia.org/resource/2_Stupid_Dogs, http://dbpedia.org/resource/Almost_Human_(TV_series), http://dbpedia.org/resource/Mystery_Island, http://dbpedia.org/resource/Rubik,_the_Amazing_Cube, http://dbpedia.org/resource/SWAT_Kats:_The_Radical_Squadron, http://dbpedia.org/resource/Step_by_Step_(TV_series), http://dbpedia.org/resource/Stroker_and_Hoop, http://dbpedia.org/resource/Superjail!, http://dbpedia.org/resource/The_Chimp_Channel, http://dbpedia.org/resource/The_Problem_Solverz, http://dbpedia.org/resource/The_Thin_Man_(TV_series), http://dbpedia.org/resource/Wake,_Rattle,_and_Roll, http://dbpedia.org/resource/Scooby-Doo,_Where_Are_You!, http://dbpedia.org/resource/Suddenly_Susan, http://dbpedia.org/resource/ThunderCats_(1985_TV_series), http://dbpedia.org/resource/Bugs_Bunny's_Bustin'_Out_All_Over, http://dbpedia.org/resource/Daffy_Duck_and_Porky_Pig_Meet_the_Groovie_Goolies, http://dbpedia.org/resource/Fred_Flintstone_and_Friends, http://dbpedia.org/resource/Fred_and_Barney_Meet_The_Thing, http://dbpedia.org/resource/Fred_and_Barney_Meet_the_Shmoo, http://dbpedia.org/resource/Freddy's_Nightmares, http://dbpedia.org/resource/Galaxy_Goof-Ups, http://dbpedia.org/resource/Hanna-Barbera's_50th:_A_Yabba_Dabba_Doo_Celebration, http://dbpedia.org/resource/Heathcliff_(1980_TV_series), http://dbpedia.org/resource/Rockin'_with_Judy_Jetson, http://dbpedia.org/resource/The_Atom_Ant/Secret_Squirrel_Show, http://dbpedia.org/resource/The_Flintstone_Comedy_Hour, http://dbpedia.org/resource/The_Flintstone_Funnies, http://dbpedia.org/resource/The_Flintstone_Kids, http://dbpedia.org/resource/The_Jetsons_Meet_the_Flintstones, http://dbpedia.org/resource/The_Middle_(TV_series), http://dbpedia.org/resource/The_New_Fred_and_Barney_Show, http://dbpedia.org/resource/A_Flintstones_Christmas_Carol, http://dbpedia.org/resource/Cave_Kids, http://dbpedia.org/resource/Sealab_2020, http://dbpedia.org/resource/The_Further_Adventures_of_SuperTed, http://dbpedia.org/resource/The_Huckleberry_Hound_Show, http://dbpedia.org/resource/The_Quick_Draw_McGraw_Show, http://dbpedia.org/resource/The_Roman_Holidays, http://dbpedia.org/resource/Doc_Elliot, http://dbpedia.org/resource/Dr._Seuss'_How_the_Grinch_Stole_Christmas!_(TV_special), http://dbpedia.org/resource/The_Real_(TV_series), http://dbpedia.org/resource/Lifechangers, http://dbpedia.org/resource/Murder_in_Mississippi, http://dbpedia.org/resource/The_Bourne_Identity_(1988_film), http://dbpedia.org/resource/Crime_Watch_Daily, http://dbpedia.org/resource/Eunice_(film), http://dbpedia.org/resource/Casper's_First_Christmas, http://dbpedia.org/resource/Legally_Mad, http://dbpedia.org/resource/Jack_Frost_(TV_special), http://dbpedia.org/resource/Teen_Titans:_Trouble_in_Tokyo, http://dbpedia.org/resource/Ann_Jillian_(TV_series), http://dbpedia.org/resource/Yogi_Bear_&_Friends, http://dbpedia.org/resource/Supernatural:_Bloodlines, http://dbpedia.org/resource/Gossip_Girl:_Acapulco, http://dbpedia.org/resource/Babylon_5, http://dbpedia.org/resource/Max_Headroom_(TV_series), http://dbpedia.org/resource/Supernatural_(U.S._TV_series), http://dbpedia.org/resource/The_Flintstones, http://dbpedia.org/resource/ALF_(TV_series), http://dbpedia.org/resource/Adventures_of_Superman_(TV_series), http://dbpedia.org/resource/Attack_of_the_50_Ft._Woman_(1993_film), http://dbpedia.org/resource/Beetlejuice_(TV_series), http://dbpedia.org/resource/Cold_Case, http://dbpedia.org/resource/Hellcats, http://dbpedia.org/resource/Horton_Hears_a_Who!_(TV_special), http://dbpedia.org/resource/Morton_&_Hayes, http://dbpedia.org/resource/Murphy_Brown, http://dbpedia.org/resource/Night_Visions_(TV_series), http://dbpedia.org/resource/Perfect_Strangers_(TV_series), http://dbpedia.org/resource/Person_of_Interest_(TV_series), http://dbpedia.org/resource/Pinky_and_the_Brain, http://dbpedia.org/resource/Shameless_(U.S._TV_series), http://dbpedia.org/resource/Smallville, http://dbpedia.org/resource/Supergirl_(TV_series), http://dbpedia.org/resource/Terminator:_The_Sarah_Connor_Chronicles, http://dbpedia.org/resource/The_Big_Bus, http://dbpedia.org/resource/The_New_Adventures_of_Superman_(TV_series), http://dbpedia.org/resource/Eve_(U.S._TV_series), http://dbpedia.org/resource/Fringe_(TV_series), http://dbpedia.org/resource/Time_Trax, http://dbpedia.org/resource/Challenge_of_the_Super_Friends, http://dbpedia.org/resource/DC_Nation_Shorts, http://dbpedia.org/resource/Super_Friends:_The_Legendary_Super_Powers_Show, http://dbpedia.org/resource/The_World's_Greatest_Super_Friends, http://dbpedia.org/resource/Bethenny_(TV_series), http://dbpedia.org/resource/Judge_Mathis, http://dbpedia.org/resource/The_Comeback_(TV_series), http://dbpedia.org/resource/The_Originals_(TV_series), http://dbpedia.org/resource/The_Secret_Circle_(TV_series), http://dbpedia.org/resource/Baby_Looney_Tunes, http://dbpedia.org/resource/Wabbit_(TV_series), http://dbpedia.org/resource/Anderson_Live, http://dbpedia.org/resource/Second_Chance_(game_show), http://dbpedia.org/resource/GE_True, http://dbpedia.org/resource/Memphis_Beat, http://dbpedia.org/resource/Southland_(TV_series), http://dbpedia.org/resource/The_Last_Ship_(TV_series), http://dbpedia.org/resource/Gilligan's_Planet, http://dbpedia.org/resource/Jesse_(TV_series), http://dbpedia.org/resource/Mickey_(TV_series), http://dbpedia.org/resource/Notes_from_the_Underbelly, http://dbpedia.org/resource/The_Life_&_Times_of_Tim, http://dbpedia.org/resource/The_New_Adventures_of_Gilligan, http://dbpedia.org/resource/The_War_at_Home_(TV_series), http://dbpedia.org/resource/The_Year_Without_a_Santa_Claus, http://dbpedia.org/resource/Traveler_(TV_series), http://dbpedia.org/resource/Wanda_at_Large, http://dbpedia.org/resource/TMZ_on_TV, http://dbpedia.org/resource/Hostages_(American_TV_series), http://dbpedia.org/resource/Change_of_Heart_(TV_series), http://dbpedia.org/resource/The_Hero_(1966_TV_series), http://dbpedia.org/resource/Black_Sash_(TV_series), http://dbpedia.org/resource/Chase_(2010_TV_series), http://dbpedia.org/resource/Harry's_Girls, http://dbpedia.org/resource/Lucky_(TV_series), http://dbpedia.org/resource/The_Montefuscos, http://dbpedia.org/resource/Undercovers_(TV_series), http://dbpedia.org/resource/One_Big_Happy_(TV_series), http://dbpedia.org/resource/In_Person_(1996_TV_series), http://dbpedia.org/resource/Just_Keke, http://dbpedia.org/resource/Casper's_Halloween_Special, http://dbpedia.org/resource/Family_Matters, http://dbpedia.org/resource/Martin_(TV_series), http://dbpedia.org/resource/My_Name_Is_Bill_W., http://dbpedia.org/resource/Nikita_(TV_series), http://dbpedia.org/resource/The_100_(TV_series), http://dbpedia.org/resource/The_Bugs_Bunny_Show, http://dbpedia.org/resource/The_People's_Court, http://dbpedia.org/resource/The_Powerpuff_Girls, http://dbpedia.org/resource/The_Waltons, http://dbpedia.org/resource/Better_Days_(TV_series), http://dbpedia.org/resource/Medical_Center_(TV_series), http://dbpedia.org/resource/My_Favorite_Martian, http://dbpedia.org/resource/One_Tree_Hill_(TV_series), http://dbpedia.org/resource/Search_(TV_series), http://dbpedia.org/resource/CHiPs, http://dbpedia.org/resource/Scooby-Doo!_Mystery_Incorporated, http://dbpedia.org/resource/This_Old_House, http://dbpedia.org/resource/Wonder_Woman_(TV_series), http://dbpedia.org/resource/City_Beneath_the_Sea_(1971_film), http://dbpedia.org/resource/Elf:_Buddy's_Musical_Christmas, http://dbpedia.org/resource/Everwood, http://dbpedia.org/resource/Harry's_Law, http://dbpedia.org/resource/Karate_Kommandos, http://dbpedia.org/resource/The_Following, http://dbpedia.org/resource/The_Jimmy_Stewart_Show, http://dbpedia.org/resource/The_New_Adventures_of_Old_Christine, http://dbpedia.org/resource/The_Tom_and_Jerry_Show_(2014_TV_series), http://dbpedia.org/resource/Veronica's_Closet, http://dbpedia.org/resource/Greetings_from_Tucson, http://dbpedia.org/resource/IZombie_(TV_series), http://dbpedia.org/resource/Our_House_(1986_TV_series), http://dbpedia.org/resource/Run,_Buddy,_Run, http://dbpedia.org/resource/Double,_Double,_Toil_and_Trouble, http://dbpedia.org/resource/Fuller_House_(TV_series), http://dbpedia.org/resource/Invasion_(TV_series), http://dbpedia.org/resource/Two_of_a_Kind_(U.S._TV_series), http://dbpedia.org/resource/Bugs_Bunny's_Howl-oween_Special, http://dbpedia.org/resource/Ed,_Edd_n_Eddy's_Big_Picture_Show, http://dbpedia.org/resource/Max_Steel:_Bio_Crisis, http://dbpedia.org/resource/Max_Steel:_Countdown, http://dbpedia.org/resource/Max_Steel:_Dark_Rival, http://dbpedia.org/resource/Max_Steel:_Endangered_Species, http://dbpedia.org/resource/Max_Steel:_Forces_of_Nature, http://dbpedia.org/resource/Christmas_Comes_to_Pac-Land, http://dbpedia.org/resource/Tupu, http://dbpedia.org/resource/Batman_(TV_series), http://dbpedia.org/resource/Dallas_(1978_TV_series), http://dbpedia.org/resource/Lois_&_Clark:_The_New_Adventures_of_Superman, http://dbpedia.org/resource/The_Fresh_Prince_of_Bel-Air, http://dbpedia.org/resource/The_New_Dick_Van_Dyke_Show, http://dbpedia.org/resource/The_Rosie_O'Donnell_Show, http://dbpedia.org/resource/The_West_Wing, http://dbpedia.org/resource/Will_&_Grace, http://dbpedia.org/resource/Gilligan's_Island, http://dbpedia.org/resource/Mayberry_R.F.D., http://dbpedia.org/resource/The_Pirates_of_Dark_Water, http://dbpedia.org/resource/F_Troop, http://dbpedia.org/resource/Teen_Titans_(TV_series), http://dbpedia.org/resource/Gavilan_(TV_series), http://dbpedia.org/resource/Foster's_Home_for_Imaginary_Friends, http://dbpedia.org/resource/Clue_Club, http://dbpedia.org/resource/Nip/Tuck, http://dbpedia.org/resource/The_Hanna-Barbera_Happy_Hour, http://dbpedia.org/resource/The_Robonic_Stooges, http://dbpedia.org/resource/The_Skatebirds, http://dbpedia.org/resource/$h*!_My_Dad_Says, http://dbpedia.org/resource/Eastwick_(TV_series), http://dbpedia.org/resource/Mom_(TV_series), http://dbpedia.org/resource/The_Asphalt_Jungle_(TV_series), http://dbpedia.org/resource/The_Funtastic_World_of_Hanna-Barbera, http://dbpedia.org/resource/The_Tyra_Banks_Show, http://dbpedia.org/resource/Benji,_Zax_&_the_Alien_Prince, http://dbpedia.org/resource/Birdman_and_the_Galaxy_Trio, http://dbpedia.org/resource/Casper_and_the_Angels, http://dbpedia.org/resource/Cattanooga_Cats, http://dbpedia.org/resource/Dastardly_and_Muttley_in_Their_Flying_Machines, http://dbpedia.org/resource/Dumb_and_Dumber_(TV_series), http://dbpedia.org/resource/Fantastic_Max, http://dbpedia.org/resource/Going_Bananas_(U.S._series), http://dbpedia.org/resource/Help!..._It's_the_Hair_Bear_Bunch!, http://dbpedia.org/resource/Hong_Kong_Phooey, http://dbpedia.org/resource/Inch_High,_Private_Eye, http://dbpedia.org/resource/Jokebook, http://dbpedia.org/resource/Korg:_70,000_B.C., http://dbpedia.org/resource/Laurel_and_Hardy_(animated_series), http://dbpedia.org/resource/Longmire_(TV_series), http://dbpedia.org/resource/Popeye_and_Son, http://dbpedia.org/resource/Pound_Puppies_(1986_TV_series), http://dbpedia.org/resource/Shazzan, http://dbpedia.org/resource/Sky_Commanders, http://dbpedia.org/resource/Snorks, http://dbpedia.org/resource/Space_Ghost_(TV_series), http://dbpedia.org/resource/Space_Stars, http://dbpedia.org/resource/Super_Friends_(1973_TV_series), http://dbpedia.org/resource/Super_Friends_(1980_TV_series), http://dbpedia.org/resource/The_Abbott_and_Costello_Cartoon_Show, http://dbpedia.org/resource/The_Adventures_of_Don_Coyote_and_Sancho_Panda, http://dbpedia.org/resource/The_All-New_Super_Friends_Hour, http://dbpedia.org/resource/The_Biskitts, http://dbpedia.org/resource/The_Little_Rascals_(animated_TV_series), http://dbpedia.org/resource/The_New_Yogi_Bear_Show, http://dbpedia.org/resource/The_Super_Globetrotters, http://dbpedia.org/resource/The_Super_Powers_Team:_Galactic_Guardians, http://dbpedia.org/resource/The_Yogi_Bear_Show, http://dbpedia.org/resource/These_Are_the_Days_(TV_series), http://dbpedia.org/resource/Tom_and_Jerry:_The_Mansion_Cat, http://dbpedia.org/resource/Wheelie_and_the_Chopper_Bunch, http://dbpedia.org/resource/Where's_Huddles%3F, http://dbpedia.org/resource/Wildfire_(1986_TV_series), http://dbpedia.org/resource/Yo_Yogi!, http://dbpedia.org/resource/Yogi's_Space_Race, http://dbpedia.org/resource/Yogi's_Treasure_Hunt, http://dbpedia.org/resource/You_Don't_Say!, http://dbpedia.org/resource/Young_Robin_Hood, http://dbpedia.org/resource/Bachelor_in_Paradise_(TV_series), http://dbpedia.org/resource/Hangin'_with_Mr._Cooper, http://dbpedia.org/resource/Just_the_Ten_of_Us, http://dbpedia.org/resource/Suburgatory, http://dbpedia.org/resource/The_Bachelor_(U.S._TV_series), http://dbpedia.org/resource/V_(2009_TV_series), http://dbpedia.org/resource/Fire!_(1977_film), http://dbpedia.org/resource/Growing_Pains:_Return_of_the_Seavers, http://dbpedia.org/resource/Surviving:_A_Family_in_Crisis, http://dbpedia.org/resource/The_Growing_Pains_Movie, http://dbpedia.org/resource/The_Jackie_Thomas_Show, http://dbpedia.org/resource/Hart_of_Dixie, http://dbpedia.org/resource/Wonder_Woman_(1974_film), http://dbpedia.org/resource/For_Your_Love_(TV_series), http://dbpedia.org/resource/Men_in_Trees, http://dbpedia.org/resource/Super_Fun_Night, http://dbpedia.org/resource/Carol_Burnett_&_Company, http://dbpedia.org/resource/Rituals_(TV_series), http://dbpedia.org/resource/The_Bonnie_Hunt_Show, http://dbpedia.org/resource/Baby_Blues_(U.S._TV_series), http://dbpedia.org/resource/Mina_and_the_Count, http://dbpedia.org/resource/Judge_Jeanine_Pirro, http://dbpedia.org/resource/Tom_and_Jerry's_Funhouse_on_TBS, http://dbpedia.org/resource/Queen_Sugar, http://dbpedia.org/resource/Adam's_Rib_(TV_series), http://dbpedia.org/resource/Animaniacs, http://dbpedia.org/resource/Batman:_The_Animated_Series, http://dbpedia.org/resource/ER_(TV_series), http://dbpedia.org/resource/Full_House, http://dbpedia.org/resource/Gilmore_Girls, http://dbpedia.org/resource/Gossip_Girl, http://dbpedia.org/resource/Gotham_(TV_series), http://dbpedia.org/resource/Knots_Landing, http://dbpedia.org/resource/Megas_XLR, http://dbpedia.org/resource/Super_Friends, http://dbpedia.org/resource/The_Adventures_of_Batman, http://dbpedia.org/resource/The_Batman/Superman_Hour, http://dbpedia.org/resource/The_Big_Bang_Theory, http://dbpedia.org/resource/The_Eleventh_Hour_(1962_TV_series), http://dbpedia.org/resource/The_Flash_(1990_TV_series), http://dbpedia.org/resource/The_Great_Grape_Ape_Show, http://dbpedia.org/resource/The_New_Adventures_of_Batman, http://dbpedia.org/resource/The_Twilight_Zone_(2002_TV_series), http://dbpedia.org/resource/The_Wonder_Years, http://dbpedia.org/resource/Two_and_a_Half_Men, http://dbpedia.org/resource/Veronica_Mars, http://dbpedia.org/resource/2_Broke_Girls, http://dbpedia.org/resource/Fantastic_Four_(1967_TV_series), http://dbpedia.org/resource/Joey_(TV_series), http://dbpedia.org/resource/Jonny_Quest_(TV_series), http://dbpedia.org/resource/The_Bachelorette, http://dbpedia.org/resource/The_Ellen_DeGeneres_Show, http://dbpedia.org/resource/The_Real_Adventures_of_Jonny_Quest, http://dbpedia.org/resource/What_I_Like_About_You_(TV_series), http://dbpedia.org/resource/The_Dukes_of_Hazzard, http://dbpedia.org/resource/Chuck_(TV_series), http://dbpedia.org/resource/Eight_Is_Enough, http://dbpedia.org/resource/Hank_(2009_TV_series), http://dbpedia.org/resource/La_Femme_Nikita, http://dbpedia.org/resource/Living_Single, http://dbpedia.org/resource/Man_from_Atlantis, http://dbpedia.org/resource/Roc_(TV_series), http://dbpedia.org/resource/Superman_(TV_series), http://dbpedia.org/resource/The_Kwicky_Koala_Show, http://dbpedia.org/resource/Third_Watch, http://dbpedia.org/resource/Blindspot_(TV_series), http://dbpedia.org/resource/Chico_and_the_Man, http://dbpedia.org/resource/China_Beach, http://dbpedia.org/resource/Regular_Show, http://dbpedia.org/resource/Revolution_(TV_series), http://dbpedia.org/resource/The_Banana_Splits, http://dbpedia.org/resource/The_Hogan_Family, http://dbpedia.org/resource/The_Mysteries_of_Laura, http://dbpedia.org/resource/The_Smurfs_(TV_series), http://dbpedia.org/resource/Falling_Skies, http://dbpedia.org/resource/Spenser:_For_Hire, http://dbpedia.org/resource/The_Kid_Super_Power_Hour_with_Shazam!, http://dbpedia.org/resource/All_of_Us, http://dbpedia.org/resource/In_the_House_(TV_series), http://dbpedia.org/resource/Moonlight_(TV_series), http://dbpedia.org/resource/Wacky_Races, http://dbpedia.org/resource/Charlie_Brown's_All-Stars, http://dbpedia.org/resource/Mission_Hill, http://dbpedia.org/resource/The_Adventures_of_Gulliver, http://dbpedia.org/resource/The_Carrie_Diaries_(TV_series), http://dbpedia.org/resource/The_Dukes, http://dbpedia.org/resource/Daddy's_Girls_(1994_TV_series), http://dbpedia.org/resource/Duck_Dodgers_(TV_series), http://dbpedia.org/resource/Forever_(U.S._TV_series), http://dbpedia.org/resource/The_Leftovers_(TV_series), http://dbpedia.org/resource/Godzilla_(animated_series), http://dbpedia.org/resource/Pretty_Little_Liars_(TV_series), http://dbpedia.org/resource/The_Nightmare_Room, http://dbpedia.org/resource/The_Real_Housewives_of_New_York_City, http://dbpedia.org/resource/Twas_the_Night_Before_Christmas_(1974_TV_special), http://dbpedia.org/resource/The_Superman/Aquaman_Hour_of_Adventure, http://dbpedia.org/resource/It's_Me_or_the_Dog, http://dbpedia.org/resource/Ace_Ventura:_Pet_Detective_(TV_series), http://dbpedia.org/resource/Are_You_There,_Chelsea%3F, http://dbpedia.org/resource/Ball_Four_(TV_series), http://dbpedia.org/resource/Big_Bag_(TV_series), http://dbpedia.org/resource/Buford_and_the_Galloping_Ghost, http://dbpedia.org/resource/Carnie!, http://dbpedia.org/resource/Cult_(TV_series), http://dbpedia.org/resource/Dark_Justice, http://dbpedia.org/resource/Fun_House_(U.S._game_show), http://dbpedia.org/resource/Ground_Floor, http://dbpedia.org/resource/He_&_She, http://dbpedia.org/resource/I_Hate_My_Teenage_Daughter, http://dbpedia.org/resource/Little_Big_Shots, http://dbpedia.org/resource/Meatballs_&_Spaghetti, http://dbpedia.org/resource/Miami_Medical, http://dbpedia.org/resource/Moral_Court, http://dbpedia.org/resource/Pandamonium_(TV_series), http://dbpedia.org/resource/Partners_(2012_TV_series), http://dbpedia.org/resource/Police_Academy_(TV_series), http://dbpedia.org/resource/Ravenswood_(TV_series), http://dbpedia.org/resource/Superior_Court_(TV_series), http://dbpedia.org/resource/Surviving_Jack, http://dbpedia.org/resource/The_Army_Show, http://dbpedia.org/resource/The_Blue_Knight_(TV_series), http://dbpedia.org/resource/The_Family_Man_(1990_TV_series), http://dbpedia.org/resource/The_Mountain_(TV_series), http://dbpedia.org/resource/The_Practice_(1976_TV_series), http://dbpedia.org/resource/The_Tomorrow_People_(U.S._TV_series), http://dbpedia.org/resource/The_Trouble_with_Larry, http://dbpedia.org/resource/A_to_Z_(TV_series), http://dbpedia.org/resource/Moral_Orel, http://dbpedia.org/resource/Hot_Wheels_World_Race, http://dbpedia.org/resource/Scruffy_(1980_film), http://dbpedia.org/resource/Kodiak_(TV_series), http://dbpedia.org/resource/The_Best_Times_(TV_show), http://dbpedia.org/resource/The_Casual_Vacancy_(miniseries), http://dbpedia.org/resource/Eye_to_Eye_(U.S._TV_series), http://dbpedia.org/resource/Helter_Skelter_(1976_film), http://dbpedia.org/resource/Mister_T_(TV_series), http://dbpedia.org/resource/PopClips, http://dbpedia.org/resource/Secret_Squirrel, http://dbpedia.org/resource/The_Travels_of_Jaimie_McPheeters_(TV_series), http://dbpedia.org/resource/Pushing_Daisies, http://dbpedia.org/resource/Eleventh_Hour_(U.S._TV_series), http://dbpedia.org/resource/George_Lopez_(TV_series), http://dbpedia.org/resource/The_Colbys, http://dbpedia.org/resource/The_Tom_and_Jerry_Comedy_Show, http://dbpedia.org/resource/The_Tom_and_Jerry_Show_(1975_TV_series), http://dbpedia.org/resource/Tom_&_Jerry_Kids, http://dbpedia.org/resource/Tom_and_Jerry_Tales, http://dbpedia.org/resource/Wait_Till_Your_Father_Gets_Home, http://dbpedia.org/resource/Grim_&_Evil, http://dbpedia.org/resource/I_Am_Weasel, http://dbpedia.org/resource/Mike_Tyson_Mysteries, http://dbpedia.org/resource/Secret_Mountain_Fort_Awesome, http://dbpedia.org/resource/Sybil_(2007_film), http://dbpedia.org/resource/The_Flintstones:_On_the_Rocks, http://dbpedia.org/resource/Tower_Prep, http://dbpedia.org/resource/We_Bare_Bears, http://dbpedia.org/resource/What_a_Cartoon!, http://dbpedia.org/resource/Alcatraz_(TV_series), http://dbpedia.org/resource/Salem's_Lot_(1979_miniseries), http://dbpedia.org/resource/Green_Lantern:_The_Animated_Series, http://dbpedia.org/resource/Legends_of_Tomorrow, http://dbpedia.org/resource/Planet_Earth_(film), http://dbpedia.org/resource/Extra_(TV_program), http://dbpedia.org/resource/Fish_Police_(TV_series), http://dbpedia.org/resource/Better_with_You, http://dbpedia.org/resource/The_Deliberate_Stranger, http://dbpedia.org/resource/Stalker_(TV_series), http://dbpedia.org/resource/666_Park_Avenue, http://dbpedia.org/resource/Fangface, http://dbpedia.org/resource/Going_Places_(U.S._TV_series), http://dbpedia.org/resource/Goldie_Gold_and_Action_Jack, http://dbpedia.org/resource/Newsreaders, http://dbpedia.org/resource/Our_Family_Honor, http://dbpedia.org/resource/Romantically_Challenged, http://dbpedia.org/resource/Selfie_(TV_series), http://dbpedia.org/resource/The_Neighbors_(game_show), http://dbpedia.org/resource/The_Nine_(TV_series), http://dbpedia.org/resource/The_Norm_Show, http://dbpedia.org/resource/The_Puppy's_Further_Adventures, http://dbpedia.org/resource/Under_Cover_(TV_series), http://dbpedia.org/resource/Work_It_(TV_series), http://dbpedia.org/resource/Danger_Island_(TV_series), http://dbpedia.org/resource/Hot_Wheels_Battle_Force_5, http://dbpedia.org/resource/Jana_of_the_Jungle, http://dbpedia.org/resource/Hot_Wheels_AcceleRacers, http://dbpedia.org/resource/Mortal_Kombat:_Conquest, http://dbpedia.org/resource/The_Superman/Batman_Adventures, http://dbpedia.org/resource/The_Office_(1995_TV_series), http://dbpedia.org/resource/The_Comic_Strip_(TV_series), http://dbpedia.org/resource/Green_Eggs_and_Ham_(TV_series), http://dbpedia.org/resource/Valley_of_the_Dinosaurs, http://dbpedia.org/resource/Family_Affair, http://dbpedia.org/resource/It_(1990_film), http://dbpedia.org/resource/The_Flash_(2014_TV_series), http://dbpedia.org/resource/Human_Target_(2010_TV_series), http://dbpedia.org/resource/La_Classe_américaine, http://dbpedia.org/resource/Brimstone_(TV_series), http://dbpedia.org/resource/Riverdale_(2016_TV_series), http://dbpedia.org/resource/Freddie, http://dbpedia.org/resource/Major_Crimes, http://dbpedia.org/resource/Pinocchio's_Christmas, http://dbpedia.org/resource/The_Closer, http://dbpedia.org/resource/Seven_Brides_for_Seven_Brothers_(TV_series), http://dbpedia.org/resource/Birds_of_Prey_(TV_series), http://dbpedia.org/resource/Big_Shots_(TV_series), http://dbpedia.org/resource/The_George_Carlin_Show, http://dbpedia.org/resource/The_Mystic_Warrior, http://dbpedia.org/resource/Police_Academy:_The_Series, http://dbpedia.org/resource/Committed_(2001_TV_series), http://dbpedia.org/resource/ThunderCats_(2011_TV_series), http://dbpedia.org/resource/The_Mullets, http://dbpedia.org/resource/The_First_Christmas:_The_Story_of_the_First_Christmas_Snow, http://dbpedia.org/resource/The_New_Adventures_of_Lucky_Luke, http://dbpedia.org/resource/The_All_New_Popeye_Hour, http://dbpedia.org/resource/Ed_Grimley__The_Completely_Mental_Misadventures_of_Ed_Grimley__1, http://dbpedia.org/resource/Shazam!_(TV_series), http://dbpedia.org/resource/Butch_Cassidy_(TV_series), http://dbpedia.org/resource/The_Funky_Phantom, http://dbpedia.org/resource/Martha_Stewart_Living, http://dbpedia.org/resource/The_Pebbles_and_Bamm-Bamm_Show, http://dbpedia.org/resource/Ed_Grimley, http://dbpedia.org/resource/Moby_Dick_and_Mighty_Mightor, http://dbpedia.org/resource/On_Our_Own_(1977_TV_series), http://dbpedia.org/resource/The_Flintstone_Comedy_Show_(1980_TV_series), http://dbpedia.org/resource/Just_Our_Luck, http://dbpedia.org/resource/Laff-A-Lympics] [INFO] P: 0.9288256227758007, R: 1.0, F: 0.96309963099631 -[INFO] Cumulative metrics, sample: 610, P: 0.514, R: 0.563, macro F1: 0.519, macro F1*: 0.537 -[INFO] Current time: 3358 ms, average: 2724.32 ms/ques, entity detection: 424.32 ms/ques (linking tool: 331.87 ms/ques), relation detection: 898.00 ms/ques (similarity: 702.19 ms/ques), EDG generation: 49.57 ms/ques +[INFO] Cumulative metrics, sample: 610, P: 0.514, R: 0.563, macro F1: 0.537 +[INFO] Current question consumed time: 2957 ms, average: 2474.52 ms/ques, entity detection: 379.07 ms/ques (linking tool: 299.56 ms/ques), relation detection: 834.78 ms/ques (similarity: 690.71 ms/ques), EDG generation: 49.31 ms/ques [INFO] Entity Linking P: 0.7459016393442623, R: 0.7, micro F1: 0.7144808743169407, macro F1: 0.7222222222222223 [INFO] Relation Linking P: 0.5949453551912568, R: 0.5409836065573771, micro F1: 0.5560655737704921, macro F1: 0.5666827676626283 [INFO] Type Linking P: 0.5442622950819672, R: 0.5442622950819672, micro F1: 0.5442622950819672, macro F1: 0.5442622950819672 @@ -8564,8 +8564,8 @@ Question 610: Name the television show directed by Simon Fuller and judged by Je [INFO] Golden Answer: [http://dbpedia.org/resource/American_Idol] [INFO] Predicted Answer: [http://dbpedia.org/resource/Idols_(TV_series)] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 611, P: 0.513, R: 0.562, macro F1: 0.518, macro F1*: 0.536 -[INFO] Current time: 2645 ms, average: 2724.19 ms/ques, entity detection: 424.13 ms/ques (linking tool: 331.59 ms/ques), relation detection: 897.83 ms/ques (similarity: 702.43 ms/ques), EDG generation: 49.54 ms/ques +[INFO] Cumulative metrics, sample: 611, P: 0.513, R: 0.562, macro F1: 0.536 +[INFO] Current question consumed time: 2205 ms, average: 2474.08 ms/ques, entity detection: 378.84 ms/ques (linking tool: 299.29 ms/ques), relation detection: 834.55 ms/ques (similarity: 690.94 ms/ques), EDG generation: 49.27 ms/ques [INFO] Entity Linking P: 0.7463175122749591, R: 0.6996726677577741, micro F1: 0.7144026186579386, macro F1: 0.7222427538144766 [INFO] Relation Linking P: 0.5939716312056736, R: 0.5400981996726678, micro F1: 0.5551554828150577, macro F1: 0.5657552999577794 [INFO] Type Linking P: 0.5433715220949263, R: 0.5433715220949263, micro F1: 0.5433715220949263, macro F1: 0.5433715220949263 @@ -8578,8 +8578,8 @@ Question 611: Name the appointer of William Clark ? [INFO] Golden Answer: [http://dbpedia.org/resource/James_Madison] [INFO] Predicted Answer: [http://dbpedia.org/resource/James_Madison] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 612, P: 0.514, R: 0.563, macro F1: 0.519, macro F1*: 0.537 -[INFO] Current time: 1172 ms, average: 2721.65 ms/ques, entity detection: 423.87 ms/ques (linking tool: 331.28 ms/ques), relation detection: 897.00 ms/ques (similarity: 701.66 ms/ques), EDG generation: 49.47 ms/ques +[INFO] Cumulative metrics, sample: 612, P: 0.514, R: 0.563, macro F1: 0.537 +[INFO] Current question consumed time: 1005 ms, average: 2471.68 ms/ques, entity detection: 378.59 ms/ques (linking tool: 299.00 ms/ques), relation detection: 833.77 ms/ques (similarity: 690.18 ms/ques), EDG generation: 49.21 ms/ques [INFO] Entity Linking P: 0.7467320261437909, R: 0.7001633986928104, micro F1: 0.7148692810457524, macro F1: 0.7226983019821894 [INFO] Relation Linking P: 0.5946350762527233, R: 0.5408496732026143, micro F1: 0.5558823529411768, macro F1: 0.5664685268919094 [INFO] Type Linking P: 0.5441176470588235, R: 0.5441176470588235, micro F1: 0.5441176470588235, macro F1: 0.5441176470588235 @@ -8592,8 +8592,8 @@ Question 612: in which movies have momoki kochi acted [INFO] Golden Answer: [http://dbpedia.org/resource/Half_Human, http://dbpedia.org/resource/Godzilla_(1954_film), http://dbpedia.org/resource/Godzilla,_King_of_the_Monsters!] [INFO] Predicted Answer: [http://dbpedia.org/resource/Rajan_P._Dev] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 613, P: 0.513, R: 0.562, macro F1: 0.518, macro F1*: 0.536 -[INFO] Current time: 4244 ms, average: 2724.14 ms/ques, entity detection: 423.62 ms/ques (linking tool: 331.05 ms/ques), relation detection: 898.47 ms/ques (similarity: 703.81 ms/ques), EDG generation: 49.41 ms/ques +[INFO] Cumulative metrics, sample: 613, P: 0.513, R: 0.562, macro F1: 0.536 +[INFO] Current question consumed time: 3771 ms, average: 2473.80 ms/ques, entity detection: 378.32 ms/ques (linking tool: 298.77 ms/ques), relation detection: 835.04 ms/ques (similarity: 692.36 ms/ques), EDG generation: 49.15 ms/ques [INFO] Entity Linking P: 0.7455138662316476, R: 0.699021207177814, micro F1: 0.7137030995106044, macro F1: 0.7215193487978792 [INFO] Relation Linking P: 0.5936650353452962, R: 0.5399673735725938, micro F1: 0.5549755301794457, macro F1: 0.5655444346783828 [INFO] Type Linking P: 0.5432300163132137, R: 0.5432300163132137, micro F1: 0.5432300163132137, macro F1: 0.5432300163132137 @@ -8606,8 +8606,8 @@ Question 613: What is the base currency of the Benelux and is also used as money [INFO] Golden Answer: [http://dbpedia.org/resource/Euro] [INFO] Predicted Answer: [http://dbpedia.org/resource/Euro] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 614, P: 0.514, R: 0.563, macro F1: 0.519, macro F1*: 0.537 -[INFO] Current time: 4465 ms, average: 2726.97 ms/ques, entity detection: 423.73 ms/ques (linking tool: 331.00 ms/ques), relation detection: 900.18 ms/ques (similarity: 704.51 ms/ques), EDG generation: 49.43 ms/ques +[INFO] Cumulative metrics, sample: 614, P: 0.514, R: 0.563, macro F1: 0.537 +[INFO] Current question consumed time: 3831 ms, average: 2476.01 ms/ques, entity detection: 378.37 ms/ques (linking tool: 298.70 ms/ques), relation detection: 836.59 ms/ques (similarity: 693.08 ms/ques), EDG generation: 49.19 ms/ques [INFO] Entity Linking P: 0.745928338762215, R: 0.6995114006514658, micro F1: 0.7141693811074927, macro F1: 0.7219745836583016 [INFO] Relation Linking P: 0.5943268186753529, R: 0.5407166123778502, micro F1: 0.5557003257328994, macro F1: 0.5662556607922052 [INFO] Type Linking P: 0.5439739413680782, R: 0.5439739413680782, micro F1: 0.5439739413680782, macro F1: 0.5439739413680782 @@ -8620,8 +8620,8 @@ Question 614: List the mammals whose taxonomy is Diprotodontia and phylum is Cho [INFO] Golden Answer: [http://dbpedia.org/resource/Bettong, http://dbpedia.org/resource/Pademelon, http://dbpedia.org/resource/Diprotodontidae, http://dbpedia.org/resource/Vombatiformes, http://dbpedia.org/resource/Cuscus, http://dbpedia.org/resource/Tree-kangaroo, http://dbpedia.org/resource/Wallaby, http://dbpedia.org/resource/Common_wombat, http://dbpedia.org/resource/Northern_common_cuscus, http://dbpedia.org/resource/Lasiorhinus, http://dbpedia.org/resource/Northern_hairy-nosed_wombat, http://dbpedia.org/resource/Ramasayia, http://dbpedia.org/resource/Rhizophascolomus, http://dbpedia.org/resource/Warendja, http://dbpedia.org/resource/Allied_rock-wallaby, http://dbpedia.org/resource/Banded_hare-wallaby, http://dbpedia.org/resource/Black-flanked_rock-wallaby, http://dbpedia.org/resource/Black-striped_wallaby, http://dbpedia.org/resource/Black_dorcopsis, http://dbpedia.org/resource/Bridled_nail-tail_wallaby, http://dbpedia.org/resource/Brown's_pademelon, http://dbpedia.org/resource/Brown_dorcopsis, http://dbpedia.org/resource/Brush-tailed_rock-wallaby, http://dbpedia.org/resource/Calaby's_pademelon, http://dbpedia.org/resource/Cape_York_rock-wallaby, http://dbpedia.org/resource/Crescent_nail-tail_wallaby, http://dbpedia.org/resource/Dorcopsis, http://dbpedia.org/resource/Dorcopsulus, http://dbpedia.org/resource/Dusky_pademelon, http://dbpedia.org/resource/Eastern_hare-wallaby, http://dbpedia.org/resource/Godman's_rock-wallaby, http://dbpedia.org/resource/Gray_dorcopsis, http://dbpedia.org/resource/Herbert's_rock-wallaby, http://dbpedia.org/resource/Lagorchestes, http://dbpedia.org/resource/Macleay's_dorcopsis, http://dbpedia.org/resource/Mareeba_rock-wallaby, http://dbpedia.org/resource/Monjon, http://dbpedia.org/resource/Mountain_pademelon, http://dbpedia.org/resource/Nabarlek, http://dbpedia.org/resource/Northern_nail-tail_wallaby, http://dbpedia.org/resource/Parma_wallaby, http://dbpedia.org/resource/Proserpine_rock-wallaby, http://dbpedia.org/resource/Purple-necked_rock-wallaby, http://dbpedia.org/resource/Red-legged_pademelon, http://dbpedia.org/resource/Red-necked_pademelon, http://dbpedia.org/resource/Rock-wallaby, http://dbpedia.org/resource/Rothschild's_rock-wallaby, http://dbpedia.org/resource/Short-eared_rock-wallaby, http://dbpedia.org/resource/Tasmanian_pademelon, http://dbpedia.org/resource/Toolache_wallaby, http://dbpedia.org/resource/Unadorned_rock-wallaby, http://dbpedia.org/resource/Western_brush_wallaby, http://dbpedia.org/resource/Whiptail_wallaby, http://dbpedia.org/resource/White-striped_dorcopsis, http://dbpedia.org/resource/Yellow-footed_rock-wallaby, http://dbpedia.org/resource/Brachalletes, http://dbpedia.org/resource/Ilariidae, http://dbpedia.org/resource/Nimbadon, http://dbpedia.org/resource/Palorchestidae, http://dbpedia.org/resource/Wynyardiidae, http://dbpedia.org/resource/Bennett's_tree-kangaroo, http://dbpedia.org/resource/Buergers'_tree-kangaroo, http://dbpedia.org/resource/Dingiso, http://dbpedia.org/resource/Doria's_tree-kangaroo, http://dbpedia.org/resource/Golden-mantled_tree-kangaroo, http://dbpedia.org/resource/Grizzled_tree-kangaroo, http://dbpedia.org/resource/Lowlands_tree-kangaroo, http://dbpedia.org/resource/Matschie's_tree-kangaroo, http://dbpedia.org/resource/Seri's_tree-kangaroo, http://dbpedia.org/resource/Ursine_tree-kangaroo, http://dbpedia.org/resource/Wakaleo_vanderleuri, http://dbpedia.org/resource/Ifola_tree-kangaroo, http://dbpedia.org/resource/Balbaroo_fangaroo, http://dbpedia.org/resource/Acrobatidae, http://dbpedia.org/resource/Brushtail_possum, http://dbpedia.org/resource/Common_ringtail_possum, http://dbpedia.org/resource/Feather-tailed_possum, http://dbpedia.org/resource/Feathertail_glider, http://dbpedia.org/resource/Honey_possum, http://dbpedia.org/resource/Hypsiprymnodontidae, http://dbpedia.org/resource/Mahogany_glider, http://dbpedia.org/resource/Musky_rat-kangaroo, http://dbpedia.org/resource/Petauridae, http://dbpedia.org/resource/Potoroo, http://dbpedia.org/resource/Pseudocheiridae, http://dbpedia.org/resource/Quokka, http://dbpedia.org/resource/Squirrel_glider, http://dbpedia.org/resource/Striped_possum, http://dbpedia.org/resource/Sugar_glider, http://dbpedia.org/resource/Yellow-bellied_glider, http://dbpedia.org/resource/Southern_hairy-nosed_wombat, http://dbpedia.org/resource/Alkwertatherium, http://dbpedia.org/resource/Bematherium, http://dbpedia.org/resource/Euowenia, http://dbpedia.org/resource/Euryzygoma, http://dbpedia.org/resource/Invictokoala, http://dbpedia.org/resource/Kolopsis, http://dbpedia.org/resource/Kolopsoides, http://dbpedia.org/resource/Kuterintja, http://dbpedia.org/resource/Maokopia, http://dbpedia.org/resource/Marada, http://dbpedia.org/resource/Meniscolophus, http://dbpedia.org/resource/Plaisiodon, http://dbpedia.org/resource/Priscileo, http://dbpedia.org/resource/Pyramios, http://dbpedia.org/resource/Raemeotherium, http://dbpedia.org/resource/Silvabestius, http://dbpedia.org/resource/Stenomerus, http://dbpedia.org/resource/Wakaleo, http://dbpedia.org/resource/Zygomaturinae, http://dbpedia.org/resource/Ektopodon, http://dbpedia.org/resource/Ganguroo, http://dbpedia.org/resource/Marlu, http://dbpedia.org/resource/Nambaroo, http://dbpedia.org/resource/Namilamadeta, http://dbpedia.org/resource/Wombat, http://dbpedia.org/resource/Tenkile, http://dbpedia.org/resource/Common_brushtail_possum, http://dbpedia.org/resource/Broad-faced_potoroo, http://dbpedia.org/resource/Phalangerida, http://dbpedia.org/resource/Gilbert's_potoroo, http://dbpedia.org/resource/Silvaroo, http://dbpedia.org/resource/Small_dorcopsis, http://dbpedia.org/resource/Sthenurinae, http://dbpedia.org/resource/Desert_rat-kangaroo, http://dbpedia.org/resource/Ekaltadeta, http://dbpedia.org/resource/Nullarbor_dwarf_bettong, http://dbpedia.org/resource/Nail-tail_wallaby, http://dbpedia.org/resource/Leadbeater's_possum, http://dbpedia.org/resource/Goodfellow's_tree-kangaroo, http://dbpedia.org/resource/Red-necked_wallaby, http://dbpedia.org/resource/Dorcopsis_(genus), http://dbpedia.org/resource/Red_kangaroo, http://dbpedia.org/resource/Swamp_wallaby, http://dbpedia.org/resource/Long-nosed_potoroo, http://dbpedia.org/resource/Wakaleo_alcootaensis, http://dbpedia.org/resource/Wakaleo_oldfieldi, http://dbpedia.org/resource/Wakaleo_vanderleueri, http://dbpedia.org/resource/Diprotodon, http://dbpedia.org/resource/Kangaroo, http://dbpedia.org/resource/Koala, http://dbpedia.org/resource/Macropodidae, http://dbpedia.org/resource/Marsupial_lion, http://dbpedia.org/resource/Nototherium, http://dbpedia.org/resource/Palorchestes, http://dbpedia.org/resource/Zygomaturus, http://dbpedia.org/resource/Thylacoleo, http://dbpedia.org/resource/Hulitherium, http://dbpedia.org/resource/Macropus_titan, http://dbpedia.org/resource/Phascolonus, http://dbpedia.org/resource/Procoptodon, http://dbpedia.org/resource/Protemnodon, http://dbpedia.org/resource/Simosthenurus, http://dbpedia.org/resource/Sthenurus, http://dbpedia.org/resource/Agile_wallaby, http://dbpedia.org/resource/Antilopine_kangaroo, http://dbpedia.org/resource/Balbaridae, http://dbpedia.org/resource/Eastern_grey_kangaroo, http://dbpedia.org/resource/Giant_koala, http://dbpedia.org/resource/Greater_glider, http://dbpedia.org/resource/Ilaria, http://dbpedia.org/resource/Litokoala, http://dbpedia.org/resource/Macropodiformes, http://dbpedia.org/resource/Macropodinae, http://dbpedia.org/resource/Macropus, http://dbpedia.org/resource/Muramura, http://dbpedia.org/resource/Ngapakaldia, http://dbpedia.org/resource/Perikoala, http://dbpedia.org/resource/Petauroidea, http://dbpedia.org/resource/Phalangeroidea, http://dbpedia.org/resource/Phascolarctidae, http://dbpedia.org/resource/Phascolarctos, http://dbpedia.org/resource/Potoroidae, http://dbpedia.org/resource/Riversleigh_rainforest_koala, http://dbpedia.org/resource/Tammar_wallaby, http://dbpedia.org/resource/Thylacoleonidae, http://dbpedia.org/resource/Wallaroo, http://dbpedia.org/resource/Western_grey_kangaroo, http://dbpedia.org/resource/Biak_glider, http://dbpedia.org/resource/Petaurus, http://dbpedia.org/resource/Admiralty_Island_cuscus, http://dbpedia.org/resource/Banggai_cuscus, http://dbpedia.org/resource/Bear_cuscus, http://dbpedia.org/resource/Black-spotted_cuscus, http://dbpedia.org/resource/Blue-eyed_cuscus, http://dbpedia.org/resource/Blue-eyed_spotted_cuscus, http://dbpedia.org/resource/Cinereus_ringtail_possum, http://dbpedia.org/resource/Common_spotted_cuscus, http://dbpedia.org/resource/Coppery_brushtail_possum, http://dbpedia.org/resource/Coppery_ringtail_possum, http://dbpedia.org/resource/D'Albertis'_ringtail_possum, http://dbpedia.org/resource/Dactylopsila, http://dbpedia.org/resource/Eastern_common_cuscus, http://dbpedia.org/resource/Gebe_cuscus, http://dbpedia.org/resource/Great-tailed_triok, http://dbpedia.org/resource/Green_ringtail_possum, http://dbpedia.org/resource/Ground_cuscus, http://dbpedia.org/resource/Herbert_River_ringtail_possum, http://dbpedia.org/resource/Lemur-like_ringtail_possum, http://dbpedia.org/resource/Long-fingered_triok, http://dbpedia.org/resource/Lowland_ringtail_possum, http://dbpedia.org/resource/Masked_ringtail_possum, http://dbpedia.org/resource/Mountain_brushtail_possum, http://dbpedia.org/resource/Mountain_cuscus, http://dbpedia.org/resource/Northern_brushtail_possum, http://dbpedia.org/resource/Northern_glider, http://dbpedia.org/resource/Ornate_cuscus, http://dbpedia.org/resource/Painted_ringtail_possum, http://dbpedia.org/resource/Phalanger, http://dbpedia.org/resource/Plush-coated_ringtail_possum, http://dbpedia.org/resource/Pseudocheirus, http://dbpedia.org/resource/Pseudochirops, http://dbpedia.org/resource/Pseudochirulus, http://dbpedia.org/resource/Pygmy_ringtail_possum, http://dbpedia.org/resource/Reclusive_ringtail_possum, http://dbpedia.org/resource/Rock-haunting_ringtail_possum, http://dbpedia.org/resource/Rothschild's_cuscus, http://dbpedia.org/resource/Scaly-tailed_possum, http://dbpedia.org/resource/Short-eared_possum, http://dbpedia.org/resource/Silky_cuscus, http://dbpedia.org/resource/Southern_common_cuscus, http://dbpedia.org/resource/Spilocuscus, http://dbpedia.org/resource/Stein's_cuscus, http://dbpedia.org/resource/Strigocuscus, http://dbpedia.org/resource/Sulawesi_bear_cuscus, http://dbpedia.org/resource/Sulawesi_dwarf_cuscus, http://dbpedia.org/resource/Talaud_bear_cuscus, http://dbpedia.org/resource/Tate's_triok, http://dbpedia.org/resource/Telefomin_cuscus, http://dbpedia.org/resource/Vogelkop_ringtail_possum, http://dbpedia.org/resource/Waigeou_cuscus, http://dbpedia.org/resource/Weyland_ringtail_possum, http://dbpedia.org/resource/Woodlark_cuscus, http://dbpedia.org/resource/Rufous_hare-wallaby, http://dbpedia.org/resource/Macropus_pearsoni, http://dbpedia.org/resource/Black_wallaroo, http://dbpedia.org/resource/Eastern_bettong, http://dbpedia.org/resource/Rufous_rat-kangaroo, http://dbpedia.org/resource/Boodie, http://dbpedia.org/resource/Mount_Claro_rock-wallaby, http://dbpedia.org/resource/Cookeroo, http://dbpedia.org/resource/Eastern_short-eared_rock-wallaby, http://dbpedia.org/resource/Common_wallaroo, http://dbpedia.org/resource/Long-footed_potoroo, http://dbpedia.org/resource/Lumholtz's_tree-kangaroo, http://dbpedia.org/resource/Northern_bettong, http://dbpedia.org/resource/Spectacled_hare-wallaby, http://dbpedia.org/resource/Western_ringtail_possum, http://dbpedia.org/resource/Woylie] [INFO] Predicted Answer: [http://dbpedia.org/resource/Propleopus, http://dbpedia.org/resource/Diprotodontia] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 615, P: 0.513, R: 0.562, macro F1: 0.518, macro F1*: 0.536 -[INFO] Current time: 5341 ms, average: 2731.22 ms/ques, entity detection: 423.67 ms/ques (linking tool: 330.89 ms/ques), relation detection: 900.73 ms/ques (similarity: 705.05 ms/ques), EDG generation: 49.46 ms/ques +[INFO] Cumulative metrics, sample: 615, P: 0.513, R: 0.562, macro F1: 0.536 +[INFO] Current question consumed time: 4869 ms, average: 2479.90 ms/ques, entity detection: 378.29 ms/ques (linking tool: 298.59 ms/ques), relation detection: 837.07 ms/ques (similarity: 693.82 ms/ques), EDG generation: 49.22 ms/ques [INFO] Entity Linking P: 0.7463414634146341, R: 0.6991869918699187, micro F1: 0.71409214092141, macro F1: 0.7219951164641005 [INFO] Relation Linking P: 0.5941734417344173, R: 0.540650406504065, micro F1: 0.5556097560975614, macro F1: 0.5661497391093313 [INFO] Type Linking P: 0.5430894308943089, R: 0.5430894308943089, micro F1: 0.5430894308943089, macro F1: 0.5430894308943089 @@ -8634,8 +8634,8 @@ Question 615: Count the key people of the Clinton Foundation? [INFO] Golden Answer: [8] [INFO] Predicted Answer: [8] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 616, P: 0.514, R: 0.563, macro F1: 0.519, macro F1*: 0.537 -[INFO] Current time: 1002 ms, average: 2728.41 ms/ques, entity detection: 423.57 ms/ques (linking tool: 330.80 ms/ques), relation detection: 899.57 ms/ques (similarity: 704.09 ms/ques), EDG generation: 49.40 ms/ques +[INFO] Cumulative metrics, sample: 616, P: 0.514, R: 0.563, macro F1: 0.537 +[INFO] Current question consumed time: 825 ms, average: 2477.21 ms/ques, entity detection: 378.14 ms/ques (linking tool: 298.47 ms/ques), relation detection: 835.98 ms/ques (similarity: 692.88 ms/ques), EDG generation: 49.16 ms/ques [INFO] Entity Linking P: 0.7467532467532467, R: 0.6996753246753247, micro F1: 0.7145562770562779, macro F1: 0.722448146690571 [INFO] Relation Linking P: 0.594832251082251, R: 0.5413961038961039, micro F1: 0.5563311688311692, macro F1: 0.5668576422981709 [INFO] Type Linking P: 0.5438311688311688, R: 0.5438311688311688, micro F1: 0.5438311688311688, macro F1: 0.5438311688311688 @@ -8648,8 +8648,8 @@ Question 616: Name the company founded in New Mexico and provides Outlook.com as [INFO] Golden Answer: [http://dbpedia.org/resource/Microsoft] [INFO] Predicted Answer: [http://dbpedia.org/resource/Cumbres_and_Toltec_Scenic_Railroad, http://dbpedia.org/resource/Santa_Fe_Southern_Railway, http://dbpedia.org/resource/KVIA-TV, http://dbpedia.org/resource/Los_Alamos_National_Laboratory, http://dbpedia.org/resource/Southern_Pacific_Transportation_Company, http://dbpedia.org/resource/Colorado_and_Southern_Railway, http://dbpedia.org/resource/Atlantic_and_Pacific_Railroad, http://dbpedia.org/resource/KOB, http://dbpedia.org/resource/Nusenda_Credit_Union, http://dbpedia.org/resource/New_Mexico_Rail_Runner_Express, http://dbpedia.org/resource/KFOX-TV, http://dbpedia.org/resource/Panhandle_and_Santa_Fe_Railway, http://dbpedia.org/resource/Bosque_Brewing_Company, http://dbpedia.org/resource/Canteen_Brewhouse, http://dbpedia.org/resource/Marble_Brewery_(Albuquerque,_New_Mexico), http://dbpedia.org/resource/Tractor_Brewing_Company, http://dbpedia.org/resource/Zody's, http://dbpedia.org/resource/Albuquerque_Public_Schools] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 617, P: 0.513, R: 0.562, macro F1: 0.518, macro F1*: 0.536 -[INFO] Current time: 8109 ms, average: 2737.13 ms/ques, entity detection: 425.06 ms/ques (linking tool: 332.33 ms/ques), relation detection: 902.18 ms/ques (similarity: 707.32 ms/ques), EDG generation: 49.36 ms/ques +[INFO] Cumulative metrics, sample: 617, P: 0.513, R: 0.562, macro F1: 0.536 +[INFO] Current question consumed time: 7244 ms, average: 2484.94 ms/ques, entity detection: 379.41 ms/ques (linking tool: 299.79 ms/ques), relation detection: 838.39 ms/ques (similarity: 696.19 ms/ques), EDG generation: 49.13 ms/ques [INFO] Entity Linking P: 0.747163695299838, R: 0.6993517017828201, micro F1: 0.7144786601836853, macro F1: 0.7224675283403474 [INFO] Relation Linking P: 0.5938681793625067, R: 0.5405186385737439, micro F1: 0.555429497568882, macro F1: 0.5659389103009291 [INFO] Type Linking P: 0.5429497568881686, R: 0.5429497568881686, micro F1: 0.5429497568881686, macro F1: 0.5429497568881686 @@ -8662,8 +8662,8 @@ Question 617: Which tv show was preceded by The spirit of Christmas and voice to [INFO] Golden Answer: [http://dbpedia.org/resource/South_Park] [INFO] Predicted Answer: [http://dbpedia.org/resource/Spider-Man_and_His_Amazing_Friends, Spider-Man (Toei)@en] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 618, P: 0.512, R: 0.561, macro F1: 0.517, macro F1*: 0.535 -[INFO] Current time: 2610 ms, average: 2736.93 ms/ques, entity detection: 424.66 ms/ques (linking tool: 331.99 ms/ques), relation detection: 902.02 ms/ques (similarity: 707.44 ms/ques), EDG generation: 49.42 ms/ques +[INFO] Cumulative metrics, sample: 618, P: 0.512, R: 0.561, macro F1: 0.535 +[INFO] Current question consumed time: 2249 ms, average: 2484.56 ms/ques, entity detection: 379.04 ms/ques (linking tool: 299.46 ms/ques), relation detection: 838.19 ms/ques (similarity: 696.33 ms/ques), EDG generation: 49.19 ms/ques [INFO] Entity Linking P: 0.7475728155339806, R: 0.6990291262135923, micro F1: 0.7144012944983827, macro F1: 0.7224864794422363 [INFO] Relation Linking P: 0.5929072276159654, R: 0.5396440129449838, micro F1: 0.5545307443365699, macro F1: 0.5650231515463968 [INFO] Type Linking P: 0.5420711974110033, R: 0.5420711974110033, micro F1: 0.5420711974110033, macro F1: 0.5420711974110033 @@ -8676,8 +8676,8 @@ Question 618: Under which common jurisdiction do Trinity School, Brentwood and B [INFO] Golden Answer: [http://dbpedia.org/resource/Essex] [INFO] Predicted Answer: [http://dbpedia.org/resource/Essex] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 619, P: 0.513, R: 0.562, macro F1: 0.518, macro F1*: 0.536 -[INFO] Current time: 8287 ms, average: 2745.90 ms/ques, entity detection: 424.80 ms/ques (linking tool: 331.97 ms/ques), relation detection: 905.89 ms/ques (similarity: 712.38 ms/ques), EDG generation: 49.40 ms/ques +[INFO] Cumulative metrics, sample: 619, P: 0.513, R: 0.562, macro F1: 0.536 +[INFO] Current question consumed time: 7377 ms, average: 2492.46 ms/ques, entity detection: 379.13 ms/ques (linking tool: 299.41 ms/ques), relation detection: 841.68 ms/ques (similarity: 701.27 ms/ques), EDG generation: 49.17 ms/ques [INFO] Entity Linking P: 0.7479806138933764, R: 0.6987075928917609, micro F1: 0.7143241787829843, macro F1: 0.7225050039282753 [INFO] Relation Linking P: 0.5935648896068928, R: 0.5403877221324718, micro F1: 0.5552504038772217, macro F1: 0.5657294234553171 [INFO] Type Linking P: 0.5428109854604201, R: 0.5428109854604201, micro F1: 0.5428109854604201, macro F1: 0.5428109854604201 @@ -8690,8 +8690,8 @@ Question 619: Music of how many things has been given by Akira Ifukube? [INFO] Golden Answer: [54] [INFO] Predicted Answer: [54] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 620, P: 0.514, R: 0.562, macro F1: 0.518, macro F1*: 0.537 -[INFO] Current time: 1078 ms, average: 2743.21 ms/ques, entity detection: 424.47 ms/ques (linking tool: 331.62 ms/ques), relation detection: 904.75 ms/ques (similarity: 711.55 ms/ques), EDG generation: 49.38 ms/ques +[INFO] Cumulative metrics, sample: 620, P: 0.514, R: 0.562, macro F1: 0.537 +[INFO] Current question consumed time: 939 ms, average: 2489.96 ms/ques, entity detection: 378.81 ms/ques (linking tool: 299.09 ms/ques), relation detection: 840.62 ms/ques (similarity: 700.45 ms/ques), EDG generation: 49.15 ms/ques [INFO] Entity Linking P: 0.7483870967741936, R: 0.6991935483870968, micro F1: 0.7147849462365601, macro F1: 0.7229544433462125 [INFO] Relation Linking P: 0.5942204301075268, R: 0.5411290322580645, micro F1: 0.5559677419354843, macro F1: 0.5664333968540081 [INFO] Type Linking P: 0.5435483870967742, R: 0.5435483870967742, micro F1: 0.5435483870967742, macro F1: 0.5435483870967742 @@ -8704,8 +8704,8 @@ Question 620: What made jo garcia and merritt cabal both famous? [INFO] Golden Answer: [http://dbpedia.org/resource/Playboy_Cyber_Club, http://dbpedia.org/resource/Playboy] [INFO] Predicted Answer: [http://dbpedia.org/resource/Playboy_Cyber_Club, http://dbpedia.org/resource/Playboy, appearance@en, ''@en] [INFO] P: 0.5, R: 1.0, F: 0.6666666666666666 -[INFO] Cumulative metrics, sample: 621, P: 0.514, R: 0.563, macro F1: 0.519, macro F1*: 0.537 -[INFO] Current time: 1291 ms, average: 2740.87 ms/ques, entity detection: 424.00 ms/ques (linking tool: 331.18 ms/ques), relation detection: 903.69 ms/ques (similarity: 710.64 ms/ques), EDG generation: 49.33 ms/ques +[INFO] Cumulative metrics, sample: 621, P: 0.514, R: 0.563, macro F1: 0.537 +[INFO] Current question consumed time: 1045 ms, average: 2487.63 ms/ques, entity detection: 378.36 ms/ques (linking tool: 298.67 ms/ques), relation detection: 839.64 ms/ques (similarity: 699.55 ms/ques), EDG generation: 49.10 ms/ques [INFO] Entity Linking P: 0.748792270531401, R: 0.6996779388083736, micro F1: 0.7152442297369843, macro F1: 0.7234024270058782 [INFO] Relation Linking P: 0.594873859366613, R: 0.5418679549114331, micro F1: 0.5566827697262483, macro F1: 0.5671350830179159 [INFO] Type Linking P: 0.5442834138486312, R: 0.5442834138486312, micro F1: 0.5442834138486312, macro F1: 0.5442834138486312 @@ -8718,8 +8718,8 @@ Question 621: What was the career of some people who have committed suicide? [INFO] Golden Answer: [http://dbpedia.org/resource/Tony_Hancock__1, http://dbpedia.org/resource/Business_magnate, http://dbpedia.org/resource/Businessperson, http://dbpedia.org/resource/Ivar_Kreuger__1, http://dbpedia.org/resource/Ivar_Kreuger__2, http://dbpedia.org/resource/Jim_Jones__1, http://dbpedia.org/resource/Scotty_Beckett__1, http://dbpedia.org/resource/Suicide_of_Tyler_Clementi__1, http://dbpedia.org/resource/COSIC, http://dbpedia.org/resource/Research, http://dbpedia.org/resource/Len_Sassaman__1, http://dbpedia.org/resource/Child_actor, http://dbpedia.org/resource/Engineering, http://dbpedia.org/resource/Kendall_McComas__1, http://dbpedia.org/resource/Kendall_McComas__2, http://dbpedia.org/resource/Actor, http://dbpedia.org/resource/Colin_Tarrant__1, http://dbpedia.org/resource/William_Liley__1, http://dbpedia.org/resource/Politician, http://dbpedia.org/resource/Albert_Vögler__1, http://dbpedia.org/resource/Maryse_Guy_Mitsouko__1, http://dbpedia.org/resource/Priesthood_(Catholic_Church), http://dbpedia.org/resource/Revolutionary, http://dbpedia.org/resource/Jacques_Roux__1, http://dbpedia.org/resource/John_Monk_Saunders__1, http://dbpedia.org/resource/James_Bullough_Lansing__1, http://dbpedia.org/resource/Voice_acting, http://dbpedia.org/resource/Character_actor, http://dbpedia.org/resource/Herman_Bing__1, http://dbpedia.org/resource/Jean-Marie_Roland,_vicomte_de_la_Platière__1, http://dbpedia.org/resource/Alexander_Gauge__1, http://dbpedia.org/resource/Costume_designer, http://dbpedia.org/resource/Paul_C._Barth__1, http://dbpedia.org/resource/Ethnology, http://dbpedia.org/resource/Buell_Quain__1, http://dbpedia.org/resource/Stephanie_Parker__1, http://dbpedia.org/resource/Film_director, http://dbpedia.org/resource/Chantal_Akerman__1, http://dbpedia.org/resource/Education, http://dbpedia.org/resource/Invention, http://dbpedia.org/resource/Journalist, http://dbpedia.org/resource/Geoffrey_Pyke__1, http://dbpedia.org/resource/Presenter, http://dbpedia.org/resource/Annie_St_John__1, http://dbpedia.org/resource/George_Sanders__1, http://dbpedia.org/resource/Theatre_director, http://dbpedia.org/resource/Songwriter, http://dbpedia.org/resource/Everett_Sloane__1, http://dbpedia.org/resource/Richard_Farnsworth__1, http://dbpedia.org/resource/Rusty_Hamer__1, http://dbpedia.org/resource/Psychoanalysis, http://dbpedia.org/resource/Psychologist, http://dbpedia.org/resource/Wilhelm_Stekel__1, http://dbpedia.org/resource/Wilhelm_Stekel__2, http://dbpedia.org/resource/Shinji_Maki__1, http://dbpedia.org/resource/Eve_Miller__1, http://dbpedia.org/resource/Brian_Keith__1, http://dbpedia.org/resource/Harold_Switzer__1, http://dbpedia.org/resource/Novelist, http://dbpedia.org/resource/Jerzy_Kosiński__1, http://dbpedia.org/resource/Joan_Gamper__1, http://dbpedia.org/resource/Sculpture, http://dbpedia.org/resource/Rembrandt_Bugatti__1, http://dbpedia.org/resource/Spalding_Gray__1, http://dbpedia.org/resource/Historian, http://dbpedia.org/resource/Ronald_Takaki__1, http://dbpedia.org/resource/Robin_Stille__1, http://dbpedia.org/resource/Pediatrics, http://dbpedia.org/resource/Real_estate_development, http://dbpedia.org/resource/Suicide_of_Chen_Gang__1, http://dbpedia.org/resource/Model_(people), http://dbpedia.org/resource/Maggie_McNamara__1, http://dbpedia.org/resource/Petroleum_geology, http://dbpedia.org/resource/George_de_Mohrenschildt__1, http://dbpedia.org/resource/Game_show, http://dbpedia.org/resource/Jay_Stewart__1, http://dbpedia.org/resource/Socialite, http://dbpedia.org/resource/Frances_Ford_Seymour__1, http://dbpedia.org/resource/Arabella_Arbenz__1, http://dbpedia.org/resource/Announcer, http://dbpedia.org/resource/Dick_Wesson_(announcer)__1, http://dbpedia.org/resource/Larry_Grey__1, http://dbpedia.org/resource/William_Byrd_III__1, http://dbpedia.org/resource/Television_presenter, http://dbpedia.org/resource/Eva_Mottley__1, http://dbpedia.org/resource/Charles_Gifkins__1, http://dbpedia.org/resource/Robert_Martinson__1, http://dbpedia.org/resource/Brewing, http://dbpedia.org/resource/Adolph_Coors__1, http://dbpedia.org/resource/Scott_Kalvert__1, http://dbpedia.org/resource/Gus_Meins__1, http://dbpedia.org/resource/Chess, http://dbpedia.org/resource/Karen_Grigorian__1, http://dbpedia.org/resource/Comedian, http://dbpedia.org/resource/Akhil_Mehta__1, http://dbpedia.org/resource/Photographer, http://dbpedia.org/resource/Ara_Gallant__1] [INFO] Predicted Answer: [] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 622, P: 0.513, R: 0.562, macro F1: 0.518, macro F1*: 0.536 -[INFO] Current time: 510 ms, average: 2737.28 ms/ques, entity detection: 423.32 ms/ques (linking tool: 330.65 ms/ques), relation detection: 902.74 ms/ques (similarity: 709.85 ms/ques), EDG generation: 49.29 ms/ques +[INFO] Cumulative metrics, sample: 622, P: 0.513, R: 0.562, macro F1: 0.536 +[INFO] Current question consumed time: 450 ms, average: 2484.35 ms/ques, entity detection: 377.75 ms/ques (linking tool: 298.20 ms/ques), relation detection: 838.74 ms/ques (similarity: 698.76 ms/ques), EDG generation: 49.06 ms/ques [INFO] Entity Linking P: 0.747588424437299, R: 0.6985530546623794, micro F1: 0.7140943193997866, macro F1: 0.7222394005959009 [INFO] Relation Linking P: 0.5939174705251875, R: 0.5409967845659164, micro F1: 0.5557877813504827, macro F1: 0.566223290279945 [INFO] Type Linking P: 0.545016077170418, R: 0.545016077170418, micro F1: 0.545016077170418, macro F1: 0.545016077170418 @@ -8732,8 +8732,8 @@ Question 622: In how many places are the tombs of people who died in Connecticut [INFO] Golden Answer: [11] [INFO] Predicted Answer: [9] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 623, P: 0.512, R: 0.561, macro F1: 0.517, macro F1*: 0.535 -[INFO] Current time: 4628 ms, average: 2740.32 ms/ques, entity detection: 423.74 ms/ques (linking tool: 331.07 ms/ques), relation detection: 902.97 ms/ques (similarity: 710.30 ms/ques), EDG generation: 49.25 ms/ques +[INFO] Cumulative metrics, sample: 623, P: 0.512, R: 0.561, macro F1: 0.535 +[INFO] Current question consumed time: 4058 ms, average: 2486.88 ms/ques, entity detection: 378.09 ms/ques (linking tool: 298.55 ms/ques), relation detection: 838.89 ms/ques (similarity: 699.22 ms/ques), EDG generation: 49.02 ms/ques [INFO] Entity Linking P: 0.7479935794542536, R: 0.6990369181380417, micro F1: 0.7145532370251481, macro F1: 0.7226870856402163 [INFO] Relation Linking P: 0.592964151952916, R: 0.5401284109149278, micro F1: 0.5548956661316216, macro F1: 0.5653144246454668 [INFO] Type Linking P: 0.5441412520064205, R: 0.5441412520064205, micro F1: 0.5441412520064205, macro F1: 0.5441412520064205 @@ -8752,8 +8752,8 @@ Question 623: List the producer of the television shows distributed by Fremantle * Serge Lobo * Fernando Sulichin@en, Claudia Lloyd@en, Tom Beattie@en, Karen Richardson@en, Boundless Productions@en, Basi Akpabio@en, John Moore@en, Gabe Landau@en, FreemantleMedia@en, Joerg Winger@en, Henriette Lippold@en, Nico Hoffman@en, David Frankel@en, George E. Crosby@en, Bruce Chevillat@en, Marc Scarpa@en] [INFO] P: 0.3236994219653179, R: 1.0, F: 0.4890829694323144 -[INFO] Cumulative metrics, sample: 624, P: 0.512, R: 0.562, macro F1: 0.517, macro F1*: 0.536 -[INFO] Current time: 2344 ms, average: 2739.68 ms/ques, entity detection: 423.23 ms/ques (linking tool: 330.68 ms/ques), relation detection: 903.38 ms/ques (similarity: 710.36 ms/ques), EDG generation: 49.20 ms/ques +[INFO] Cumulative metrics, sample: 624, P: 0.512, R: 0.562, macro F1: 0.536 +[INFO] Current question consumed time: 2096 ms, average: 2486.25 ms/ques, entity detection: 377.63 ms/ques (linking tool: 298.19 ms/ques), relation detection: 839.31 ms/ques (similarity: 699.51 ms/ques), EDG generation: 48.98 ms/ques [INFO] Entity Linking P: 0.7483974358974359, R: 0.6995192307692307, micro F1: 0.7150106837606847, macro F1: 0.7231333276574008 [INFO] Relation Linking P: 0.5936164529914529, R: 0.5408653846153846, micro F1: 0.5556089743589747, macro F1: 0.566014510798207 [INFO] Type Linking P: 0.5432692307692307, R: 0.5432692307692307, micro F1: 0.5432692307692307, macro F1: 0.5432692307692307 @@ -8766,8 +8766,8 @@ Question 624: Name the awards won by Elie Wiesel ? [INFO] Golden Answer: [http://dbpedia.org/resource/Becket_Fund_for_Religious_Liberty, http://dbpedia.org/resource/Nobel_Peace_Prize, http://dbpedia.org/resource/Presidential_Medal_of_Freedom, http://dbpedia.org/resource/Legion_of_Honour, Commander of the Order of the British Empire@en, Congressional Gold Medal@en, Grand Officer of the Order of the Star of Romania@en] [INFO] Predicted Answer: [http://dbpedia.org/resource/Becket_Fund_for_Religious_Liberty, http://dbpedia.org/resource/Congressional_Gold_Medal, http://dbpedia.org/resource/Nobel_Peace_Prize, http://dbpedia.org/resource/Order_of_the_British_Empire, http://dbpedia.org/resource/Presidential_Medal_of_Freedom, http://dbpedia.org/resource/Legion_of_Honour, http://dbpedia.org/resource/Order_of_the_Star_of_Romania] [INFO] P: 0.7142857142857143, R: 0.7142857142857143, F: 0.7142857142857143 -[INFO] Cumulative metrics, sample: 625, P: 0.512, R: 0.562, macro F1: 0.517, macro F1*: 0.536 -[INFO] Current time: 785 ms, average: 2736.56 ms/ques, entity detection: 422.84 ms/ques (linking tool: 330.34 ms/ques), relation detection: 902.28 ms/ques (similarity: 709.42 ms/ques), EDG generation: 49.14 ms/ques +[INFO] Cumulative metrics, sample: 625, P: 0.512, R: 0.562, macro F1: 0.536 +[INFO] Current question consumed time: 700 ms, average: 2483.40 ms/ques, entity detection: 377.27 ms/ques (linking tool: 297.89 ms/ques), relation detection: 838.31 ms/ques (similarity: 698.61 ms/ques), EDG generation: 48.92 ms/ques [INFO] Entity Linking P: 0.7488, R: 0.7, micro F1: 0.7154666666666676, macro F1: 0.72357813362783 [INFO] Relation Linking P: 0.5926666666666666, R: 0.54, micro F1: 0.5547200000000003, macro F1: 0.5651088875809299 [INFO] Type Linking P: 0.544, R: 0.544, micro F1: 0.544, macro F1: 0.544 @@ -8780,8 +8780,8 @@ Question 625: Who are the producers of Marie Sisters? [INFO] Golden Answer: [http://dbpedia.org/resource/Kenny_Gioia, http://dbpedia.org/resource/Richard_Marx, http://dbpedia.org/resource/Guy_Roche, http://dbpedia.org/resource/Max_T._Barnes, http://dbpedia.org/resource/Shep_Goodman] [INFO] Predicted Answer: [http://dbpedia.org/resource/Kenny_Gioia, http://dbpedia.org/resource/Richard_Marx, http://dbpedia.org/resource/Guy_Roche, http://dbpedia.org/resource/Max_T._Barnes, http://dbpedia.org/resource/Shep_Goodman] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 626, P: 0.513, R: 0.563, macro F1: 0.518, macro F1*: 0.537 -[INFO] Current time: 683 ms, average: 2733.28 ms/ques, entity detection: 422.48 ms/ques (linking tool: 329.96 ms/ques), relation detection: 901.11 ms/ques (similarity: 708.47 ms/ques), EDG generation: 49.08 ms/ques +[INFO] Cumulative metrics, sample: 626, P: 0.513, R: 0.563, macro F1: 0.537 +[INFO] Current question consumed time: 589 ms, average: 2480.37 ms/ques, entity detection: 376.93 ms/ques (linking tool: 297.53 ms/ques), relation detection: 837.23 ms/ques (similarity: 697.68 ms/ques), EDG generation: 48.85 ms/ques [INFO] Entity Linking P: 0.7492012779552716, R: 0.700479233226837, micro F1: 0.7159211927582544, macro F1: 0.7240215104868024 [INFO] Relation Linking P: 0.5933173588924388, R: 0.5407348242811502, micro F1: 0.5554313099041537, macro F1: 0.5658070458554023 [INFO] Type Linking P: 0.5447284345047924, R: 0.5447284345047924, micro F1: 0.5447284345047924, macro F1: 0.5447284345047924 @@ -8794,8 +8794,8 @@ Question 626: Who won the silver medal when the bronze was won by Yang Yilin? [INFO] Golden Answer: [http://dbpedia.org/resource/Shawn_Johnson, http://dbpedia.org/resource/Nastia_Liukin] [INFO] Predicted Answer: [http://dbpedia.org/resource/Nastia_Liukin, http://dbpedia.org/resource/He_Kexin] [INFO] P: 0.5, R: 0.5, F: 0.5 -[INFO] Cumulative metrics, sample: 627, P: 0.513, R: 0.563, macro F1: 0.518, macro F1*: 0.537 -[INFO] Current time: 1462 ms, average: 2731.25 ms/ques, entity detection: 422.08 ms/ques (linking tool: 329.59 ms/ques), relation detection: 900.44 ms/ques (similarity: 708.01 ms/ques), EDG generation: 49.15 ms/ques +[INFO] Cumulative metrics, sample: 627, P: 0.513, R: 0.563, macro F1: 0.537 +[INFO] Current question consumed time: 1306 ms, average: 2478.50 ms/ques, entity detection: 376.53 ms/ques (linking tool: 297.19 ms/ques), relation detection: 836.60 ms/ques (similarity: 697.25 ms/ques), EDG generation: 48.95 ms/ques [INFO] Entity Linking P: 0.7496012759170654, R: 0.7009569377990431, micro F1: 0.7163742690058489, macro F1: 0.7244634651249044 [INFO] Relation Linking P: 0.5923710792131844, R: 0.539872408293461, micro F1: 0.5545454545454549, macro F1: 0.5649046422734958 [INFO] Type Linking P: 0.5454545454545454, R: 0.5454545454545454, micro F1: 0.5454545454545454, macro F1: 0.5454545454545454 @@ -8808,8 +8808,8 @@ Question 627: List the successor of successor of Hayden, Stone & Co. [INFO] Golden Answer: [http://dbpedia.org/resource/American_Express, http://dbpedia.org/resource/Morgan_Stanley_Wealth_Management] [INFO] Predicted Answer: [http://dbpedia.org/resource/Cogan,_Berlind,_Weill_&_Levitt, http://dbpedia.org/resource/Shearson, Shearson Hayden Stone, Shearson/American Express@en] [INFO] P: 0.3333333333333333, R: 0.5, F: 0.4 -[INFO] Cumulative metrics, sample: 628, P: 0.512, R: 0.563, macro F1: 0.518, macro F1*: 0.536 -[INFO] Current time: 1435 ms, average: 2729.18 ms/ques, entity detection: 422.46 ms/ques (linking tool: 330.05 ms/ques), relation detection: 899.43 ms/ques (similarity: 707.30 ms/ques), EDG generation: 49.15 ms/ques +[INFO] Cumulative metrics, sample: 628, P: 0.512, R: 0.563, macro F1: 0.536 +[INFO] Current question consumed time: 1292 ms, average: 2476.61 ms/ques, entity detection: 376.86 ms/ques (linking tool: 297.58 ms/ques), relation detection: 835.65 ms/ques (similarity: 696.55 ms/ques), EDG generation: 48.96 ms/ques [INFO] Entity Linking P: 0.75, R: 0.7014331210191083, micro F1: 0.7168259023354574, macro F1: 0.7249040043883708 [INFO] Relation Linking P: 0.59302016985138, R: 0.5406050955414012, micro F1: 0.555254777070064, macro F1: 0.5656008830561915 [INFO] Type Linking P: 0.5461783439490446, R: 0.5461783439490446, micro F1: 0.5461783439490446, macro F1: 0.5461783439490446 @@ -8822,8 +8822,8 @@ Question 628: Count the number of battles fought by the military person involved [INFO] Golden Answer: [12] [INFO] Predicted Answer: [3] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 629, P: 0.512, R: 0.562, macro F1: 0.517, macro F1*: 0.535 -[INFO] Current time: 6177 ms, average: 2734.67 ms/ques, entity detection: 423.26 ms/ques (linking tool: 330.82 ms/ques), relation detection: 901.66 ms/ques (similarity: 709.78 ms/ques), EDG generation: 49.11 ms/ques +[INFO] Cumulative metrics, sample: 629, P: 0.512, R: 0.562, macro F1: 0.535 +[INFO] Current question consumed time: 5544 ms, average: 2481.48 ms/ques, entity detection: 377.57 ms/ques (linking tool: 298.30 ms/ques), relation detection: 837.66 ms/ques (similarity: 699.04 ms/ques), EDG generation: 48.93 ms/ques [INFO] Entity Linking P: 0.7503974562798092, R: 0.7019077901430842, micro F1: 0.7172760996290417, macro F1: 0.7253431350794434 [INFO] Relation Linking P: 0.5920773714891361, R: 0.5397456279809221, micro F1: 0.5543720190779018, macro F1: 0.5647016765648462 [INFO] Type Linking P: 0.5453100158982512, R: 0.5453100158982512, micro F1: 0.5453100158982512, macro F1: 0.5453100158982512 @@ -8836,8 +8836,8 @@ Question 629: What is the origin of Xynisteri? [INFO] Golden Answer: [http://dbpedia.org/resource/Cyprus] [INFO] Predicted Answer: [http://dbpedia.org/resource/Cyprus] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 630, P: 0.512, R: 0.562, macro F1: 0.518, macro F1*: 0.536 -[INFO] Current time: 242 ms, average: 2730.71 ms/ques, entity detection: 422.70 ms/ques (linking tool: 330.37 ms/ques), relation detection: 900.33 ms/ques (similarity: 708.72 ms/ques), EDG generation: 49.04 ms/ques +[INFO] Cumulative metrics, sample: 630, P: 0.512, R: 0.562, macro F1: 0.536 +[INFO] Current question consumed time: 205 ms, average: 2477.87 ms/ques, entity detection: 377.06 ms/ques (linking tool: 297.89 ms/ques), relation detection: 836.42 ms/ques (similarity: 698.00 ms/ques), EDG generation: 48.86 ms/ques [INFO] Entity Linking P: 0.7507936507936508, R: 0.7023809523809523, micro F1: 0.7177248677248687, macro F1: 0.7257808639567241 [INFO] Relation Linking P: 0.5927248677248677, R: 0.5404761904761904, micro F1: 0.5550793650793654, macro F1: 0.5653960101608053 [INFO] Type Linking P: 0.546031746031746, R: 0.546031746031746, micro F1: 0.546031746031746, macro F1: 0.546031746031746 @@ -8850,8 +8850,8 @@ Question 630: How many important works have been done by Russel Davies? [INFO] Golden Answer: [9] [INFO] Predicted Answer: [9] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 631, P: 0.513, R: 0.563, macro F1: 0.519, macro F1*: 0.537 -[INFO] Current time: 1387 ms, average: 2728.58 ms/ques, entity detection: 422.40 ms/ques (linking tool: 330.06 ms/ques), relation detection: 899.59 ms/ques (similarity: 708.27 ms/ques), EDG generation: 49.00 ms/ques +[INFO] Cumulative metrics, sample: 631, P: 0.513, R: 0.563, macro F1: 0.537 +[INFO] Current question consumed time: 1185 ms, average: 2475.82 ms/ques, entity detection: 376.76 ms/ques (linking tool: 297.59 ms/ques), relation detection: 835.71 ms/ques (similarity: 697.57 ms/ques), EDG generation: 48.81 ms/ques [INFO] Entity Linking P: 0.751188589540412, R: 0.7028526148969889, micro F1: 0.7181722134178562, macro F1: 0.7262171977355264 [INFO] Relation Linking P: 0.5933703116745905, R: 0.5412044374009509, micro F1: 0.5557844690966723, macro F1: 0.566088124139791 [INFO] Type Linking P: 0.5467511885895404, R: 0.5467511885895404, micro F1: 0.5467511885895404, macro F1: 0.5467511885895404 @@ -8864,8 +8864,8 @@ Question 631: Who are currently playing for Barcelona FC? [INFO] Golden Answer: [http://dbpedia.org/resource/Dani_Alves, http://dbpedia.org/resource/Marc-André_ter_Stegen, http://dbpedia.org/resource/Neymar, http://dbpedia.org/resource/Sergi_Roberto, http://dbpedia.org/resource/Jordi_Alba, http://dbpedia.org/resource/Rafinha_(footballer,_born_1993), http://dbpedia.org/resource/Claudio_Bravo_(footballer), http://dbpedia.org/resource/Luis_Enrique_(footballer), http://dbpedia.org/resource/Luis_Suárez, http://dbpedia.org/resource/Thomas_Vermaelen, http://dbpedia.org/resource/Andrés_Iniesta, http://dbpedia.org/resource/Lionel_Messi, http://dbpedia.org/resource/Ivan_Rakitić, http://dbpedia.org/resource/Adriano_(footballer,_born_1984), http://dbpedia.org/resource/Aleix_Vidal, http://dbpedia.org/resource/Douglas_(footballer,_born_1990), http://dbpedia.org/resource/Joan_Barbarà, http://dbpedia.org/resource/Jordi_Masip, http://dbpedia.org/resource/José_Ramón_de_la_Fuente, http://dbpedia.org/resource/Juan_Carlos_Unzué, http://dbpedia.org/resource/Munir_El_Haddadi, http://dbpedia.org/resource/Sandro_Ramírez, http://dbpedia.org/resource/Lee_Seung-woo, http://dbpedia.org/resource/Arda_Turan, http://dbpedia.org/resource/Javier_Mascherano, http://dbpedia.org/resource/Marc_Bartra, http://dbpedia.org/resource/Sergio_Busquets, http://dbpedia.org/resource/Gerard_Piqué, http://dbpedia.org/resource/Jérémy_Mathieu] [INFO] Predicted Answer: [http://dbpedia.org/resource/Albert_Jorquera, http://dbpedia.org/resource/Franck_Songo'o, http://dbpedia.org/resource/Jordi_Gómez, http://dbpedia.org/resource/Jesús_Olmo, http://dbpedia.org/resource/David_Casablanca, http://dbpedia.org/resource/Felipe_Sanchón, http://dbpedia.org/resource/Sergio_Rodríguez_García, http://dbpedia.org/resource/Toni_Moral, http://dbpedia.org/resource/Albert_Puigdollers, http://dbpedia.org/resource/Antonio_Hidalgo_(footballer,_born_1979), http://dbpedia.org/resource/Albert_Jorquera__3, http://dbpedia.org/resource/David_Bermudo__2, http://dbpedia.org/resource/Fernando_Navarro__2, http://dbpedia.org/resource/Jofre__2, http://dbpedia.org/resource/Luis_Cembranos__2, http://dbpedia.org/resource/Jordi_Vinyals__3, http://dbpedia.org/resource/Oriol_Riera__4, http://dbpedia.org/resource/Felip_Ortiz, http://dbpedia.org/resource/Jordi_Ferrón, http://dbpedia.org/resource/Luis_Cembranos, http://dbpedia.org/resource/Mohammed_El_Yaagoubi, http://dbpedia.org/resource/Roberto_Trashorras, http://dbpedia.org/resource/David_Bauzá, http://dbpedia.org/resource/David_Bermudo, http://dbpedia.org/resource/Nuno_Carvalho, http://dbpedia.org/resource/Ramón_Masó, http://dbpedia.org/resource/Víctor_Sánchez_(footballer,_born_1987), http://dbpedia.org/resource/Andreu_Guerao, http://dbpedia.org/resource/Carles_Puyol, http://dbpedia.org/resource/Pepe_Reina, http://dbpedia.org/resource/Pedro_(footballer,_born_July_1987), http://dbpedia.org/resource/Gerard_Autet, http://dbpedia.org/resource/Miguel_Ángel_Lozano, http://dbpedia.org/resource/Alfi_Conteh-Lacalle, http://dbpedia.org/resource/Francisco_Martos, http://dbpedia.org/resource/Jito_(footballer), http://dbpedia.org/resource/Federico_Bessone, http://dbpedia.org/resource/Genís_García_Junyent, http://dbpedia.org/resource/Ramón_Ros, http://dbpedia.org/resource/Xavi_Moro, http://dbpedia.org/resource/Sebastián_Herrera, http://dbpedia.org/resource/Francisco_Rufete, http://dbpedia.org/resource/Mikel_Arteta, http://dbpedia.org/resource/Lionel_Messi, http://dbpedia.org/resource/Jeffrén_Suárez, http://dbpedia.org/resource/Marc_Crosas, http://dbpedia.org/resource/Víctor_Valdés, http://dbpedia.org/resource/Toni_Jiménez, http://dbpedia.org/resource/Carlos_Peña_(footballer), http://dbpedia.org/resource/Joan_Verdú, http://dbpedia.org/resource/Jordi_López, http://dbpedia.org/resource/Lluís_Sastre, http://dbpedia.org/resource/Marc_Martínez_(footballer), http://dbpedia.org/resource/Jofre, http://dbpedia.org/resource/David_Sánchez_(footballer), http://dbpedia.org/resource/David_Pirri, http://dbpedia.org/resource/Óscar_Álvarez_(footballer), http://dbpedia.org/resource/Manolo_Pérez, http://dbpedia.org/resource/Francisco_Montañés, http://dbpedia.org/resource/José_Miguel_Bermúdez, http://dbpedia.org/resource/Rubén_Iván_Martínez, http://dbpedia.org/resource/Sergio_Busquets, http://dbpedia.org/resource/Dani_Fernández, http://dbpedia.org/resource/Albert_Dorca, http://dbpedia.org/resource/Javi_Moreno, http://dbpedia.org/resource/Manuel_Lanzarote, http://dbpedia.org/resource/Marc_Valiente, http://dbpedia.org/resource/Pitu, http://dbpedia.org/resource/Víctor_Vázquez, http://dbpedia.org/resource/Sito_Riera, http://dbpedia.org/resource/Jordi_Vinyals, http://dbpedia.org/resource/Nakor_Bueno, http://dbpedia.org/resource/Raúl_Vates, http://dbpedia.org/resource/Javi_Ruiz, http://dbpedia.org/resource/Dani_Tortolero, http://dbpedia.org/resource/Arnau_Riera, http://dbpedia.org/resource/Fernando_Navarro, http://dbpedia.org/resource/Víctor_Curto, http://dbpedia.org/resource/Alberto_Manga, http://dbpedia.org/resource/Luka_Savić, http://dbpedia.org/resource/Moisés_Pereiro, http://dbpedia.org/resource/David_García_(footballer,_born_1980), http://dbpedia.org/resource/Luis_Carrión, http://dbpedia.org/resource/Oriol_Riera, http://dbpedia.org/resource/Urko_Pardo, http://dbpedia.org/resource/Nano_(Spanish_footballer,_born_1982), http://dbpedia.org/resource/Pere_Gratacós, http://dbpedia.org/resource/Sergio_García_(footballer,_born_1983), http://dbpedia.org/resource/Sergio_Santamaría, http://dbpedia.org/resource/Toni_Calvo, http://dbpedia.org/resource/Juan_Carricondo, http://dbpedia.org/resource/Jorge_Perona, http://dbpedia.org/resource/Óscar_López_(footballer), http://dbpedia.org/resource/Javito, http://dbpedia.org/resource/José_Manuel_Casado, http://dbpedia.org/resource/Marc_Bernaus, http://dbpedia.org/resource/José_Antonio_Llamas, http://dbpedia.org/resource/Miguel_Martínez_(footballer), http://dbpedia.org/resource/Cristian_Hidalgo, http://dbpedia.org/resource/Fali_(footballer,_born_1986), http://dbpedia.org/resource/Ibán_Cuadrado, http://dbpedia.org/resource/Daniel_Mejías, http://dbpedia.org/resource/Jesús_Rodríguez_(footballer), http://dbpedia.org/resource/Francisco_Borrego, http://dbpedia.org/resource/Carles_Puyol__3, http://dbpedia.org/resource/Mikel_Arteta__3, http://dbpedia.org/resource/Franck_Songo'o__3, http://dbpedia.org/resource/Juan_Carricondo__3, http://dbpedia.org/resource/Francisco_Rufete__3, http://dbpedia.org/resource/Mohammed_El_Yaagoubi__3, http://dbpedia.org/resource/Rubén_Iván_Martínez__4, http://dbpedia.org/resource/Sergio_Rodríguez_García__2, http://dbpedia.org/resource/Felip_Ortiz__3, http://dbpedia.org/resource/Jordi_Ferrón__2, http://dbpedia.org/resource/Pitu__2, http://dbpedia.org/resource/Gil_Orriols_Jansana__2, http://dbpedia.org/resource/Ismael_Santiago, http://dbpedia.org/resource/Migue, http://dbpedia.org/resource/Mingo_(footballer), http://dbpedia.org/resource/Xavier_Roca, http://dbpedia.org/resource/Lionel_Messi__3, http://dbpedia.org/resource/Pepe_Reina__3, http://dbpedia.org/resource/Javi_Ruiz__2, http://dbpedia.org/resource/Víctor_Sánchez_(footballer,_born_1987)__3, http://dbpedia.org/resource/Sergio_Busquets__4, http://dbpedia.org/resource/Francisco_Martos__2, http://dbpedia.org/resource/Jeffrén_Suárez__3, http://dbpedia.org/resource/Marc_Crosas__2, http://dbpedia.org/resource/Marc_Valiente__2, http://dbpedia.org/resource/Sergio_García_(footballer,_born_1983)__4, http://dbpedia.org/resource/Lluís_Sastre__2, http://dbpedia.org/resource/Toni_Calvo__2, http://dbpedia.org/resource/Urko_Pardo__3, http://dbpedia.org/resource/Víctor_Vázquez__3, http://dbpedia.org/resource/Roberto_Trashorras__3, http://dbpedia.org/resource/Felipe_Sanchón__2, http://dbpedia.org/resource/Javito__3, http://dbpedia.org/resource/Jordi_Gómez__2, http://dbpedia.org/resource/Juan_Carlos_Rojo__12, http://dbpedia.org/resource/Xavi_Moro__2, http://dbpedia.org/resource/David_García_(footballer,_born_1980)__3, http://dbpedia.org/resource/Mingo_(footballer)__2, http://dbpedia.org/resource/Dani_Tortolero__3, http://dbpedia.org/resource/Francisco_Borrego__3, http://dbpedia.org/resource/Toni_Jiménez__4, http://dbpedia.org/resource/Alfi_Conteh-Lacalle__2, http://dbpedia.org/resource/Lluís_Pujol__13, http://dbpedia.org/resource/José_Antonio_Llamas__2, http://dbpedia.org/resource/Luka_Savić__3, http://dbpedia.org/resource/Sebastián_Herrera__2, http://dbpedia.org/resource/Quique_Costas__6, http://dbpedia.org/resource/Sergio_Santamaría__4, http://dbpedia.org/resource/Albert_Dorca__2, http://dbpedia.org/resource/Alberto_Manga__2, http://dbpedia.org/resource/Carlos_Peña_(footballer)__2, http://dbpedia.org/resource/David_Bauzá__2, http://dbpedia.org/resource/Francisco_Montañés__3, http://dbpedia.org/resource/Genís_García_Junyent__3, http://dbpedia.org/resource/Jito_(footballer)__2, http://dbpedia.org/resource/José_Manuel_Casado__4, http://dbpedia.org/resource/Luis_Carrión__4, http://dbpedia.org/resource/Migue__2, http://dbpedia.org/resource/Moisés_Pereiro__2, http://dbpedia.org/resource/Nano_(Spanish_footballer,_born_1982)__3, http://dbpedia.org/resource/Pere_Gratacós__3, http://dbpedia.org/resource/Albert_Puigdollers__3, http://dbpedia.org/resource/Pedro_(footballer,_born_July_1987)__3, http://dbpedia.org/resource/Arnau_Riera__6, http://dbpedia.org/resource/Cristian_Hidalgo__3, http://dbpedia.org/resource/Joan_Verdú__3, http://dbpedia.org/resource/Miguel_Ángel_Lozano__4, http://dbpedia.org/resource/Nakor_Bueno__4, http://dbpedia.org/resource/Esteban_Vigo__13, http://dbpedia.org/resource/FC_Barcelona_C__Alfredo_Zardoya__1, http://dbpedia.org/resource/FC_Barcelona_C__Blas_Vilchez__1, http://dbpedia.org/resource/FC_Barcelona_C__Daniel_Planchería__1, http://dbpedia.org/resource/FC_Barcelona_C__David_Cornellà__1, http://dbpedia.org/resource/FC_Barcelona_C__Eugenio_Plazuelo__1, http://dbpedia.org/resource/FC_Barcelona_C__Fali__1, http://dbpedia.org/resource/FC_Barcelona_C__Ismael_Remacha__1, http://dbpedia.org/resource/FC_Barcelona_C__Javi_Heredia__1, http://dbpedia.org/resource/FC_Barcelona_C__Jordi_Benet__1, http://dbpedia.org/resource/FC_Barcelona_C__Jordi_Folguera__1, http://dbpedia.org/resource/FC_Barcelona_C__Josué_Fernández__1, http://dbpedia.org/resource/FC_Barcelona_C__José_Manuel_Cabrera__1, http://dbpedia.org/resource/FC_Barcelona_C__José_Manuel__1, http://dbpedia.org/resource/FC_Barcelona_C__José_de_la_Puente__1, http://dbpedia.org/resource/FC_Barcelona_C__Miguel_Ángel__1, http://dbpedia.org/resource/FC_Barcelona_C__Nuno_Carvalho__1, http://dbpedia.org/resource/FC_Barcelona_C__Pau_Torres__1, http://dbpedia.org/resource/FC_Barcelona_C__Pedro_Rodríguez__1, http://dbpedia.org/resource/FC_Barcelona_C__Ramón_Masó__1, http://dbpedia.org/resource/FC_Barcelona_C__Sergio_Urbano__1, http://dbpedia.org/resource/FC_Barcelona_C__Stefan_Rodríguez__1, http://dbpedia.org/resource/FC_Barcelona_C__Víctor_Sánchez__1, http://dbpedia.org/resource/FC_Barcelona_C__arlos_Alonso__1, http://dbpedia.org/resource/FC_Barcelona_C__Álex_Sánchez__1, http://dbpedia.org/resource/Gerard_Autet__2, http://dbpedia.org/resource/Ismael_Santiago__3, http://dbpedia.org/resource/Javi_Moreno__3, http://dbpedia.org/resource/Jesús_Rodríguez_(footballer)__3, http://dbpedia.org/resource/Manuel_Lanzarote__2, http://dbpedia.org/resource/Ramón_Ros__3, http://dbpedia.org/resource/Toni_Moral__3, http://dbpedia.org/resource/Xavier_Roca__2, http://dbpedia.org/resource/Jordi_López__2, http://dbpedia.org/resource/Marc_Bernaus__3, http://dbpedia.org/resource/Óscar_López_(footballer)__2, http://dbpedia.org/resource/David_Casablanca__2, http://dbpedia.org/resource/Víctor_Valdés__4, http://dbpedia.org/resource/Daniel_Mejías__2, http://dbpedia.org/resource/Jorge_Perona__3, http://dbpedia.org/resource/José_Miguel_Bermúdez__3, http://dbpedia.org/resource/Ramón_Masó__3, http://dbpedia.org/resource/Raúl_Vates__7, http://dbpedia.org/resource/Víctor_Curto__4, http://dbpedia.org/resource/Jesús_Olmo__2, http://dbpedia.org/resource/Nuno_Carvalho__6, http://dbpedia.org/resource/Antonio_Hidalgo_(footballer,_born_1979)__3, http://dbpedia.org/resource/Ibán_Cuadrado__2, http://dbpedia.org/resource/Fali_(footballer,_born_1986)__4, http://dbpedia.org/resource/Marc_Martínez_(footballer)__2, http://dbpedia.org/resource/Sergio_Lobera__7, http://dbpedia.org/resource/Óscar_Álvarez_(footballer)__2, http://dbpedia.org/resource/David_Sánchez_(footballer)__3, http://dbpedia.org/resource/Manolo_Pérez__2, http://dbpedia.org/resource/Andreu_Guerao__2, http://dbpedia.org/resource/Federico_Bessone__2, http://dbpedia.org/resource/Miguel_Martínez_(footballer)__3, http://dbpedia.org/resource/Sito_Riera__2, http://dbpedia.org/resource/David_Pirri__2, http://dbpedia.org/resource/Dani_Fernández__2, http://dbpedia.org/resource/1981–82_Copa_del_Rey, http://dbpedia.org/resource/1982–83_Copa_del_Rey, http://dbpedia.org/resource/1984–85_Copa_del_Rey] [INFO] P: 0.00851063829787234, R: 0.06896551724137931, F: 0.015151515151515152 -[INFO] Cumulative metrics, sample: 632, P: 0.512, R: 0.562, macro F1: 0.518, macro F1*: 0.536 -[INFO] Current time: 5934 ms, average: 2733.65 ms/ques, entity detection: 422.85 ms/ques (linking tool: 330.58 ms/ques), relation detection: 903.02 ms/ques (similarity: 709.81 ms/ques), EDG generation: 48.93 ms/ques +[INFO] Cumulative metrics, sample: 632, P: 0.512, R: 0.562, macro F1: 0.536 +[INFO] Current question consumed time: 5151 ms, average: 2480.06 ms/ques, entity detection: 377.15 ms/ques (linking tool: 298.05 ms/ques), relation detection: 838.72 ms/ques (similarity: 699.06 ms/ques), EDG generation: 48.75 ms/ques [INFO] Entity Linking P: 0.75, R: 0.7017405063291139, micro F1: 0.7170358649789039, macro F1: 0.7250681198910083 [INFO] Relation Linking P: 0.592431434599156, R: 0.5403481012658228, micro F1: 0.5549050632911395, macro F1: 0.5651924150826078 [INFO] Type Linking P: 0.5474683544303798, R: 0.5474683544303798, micro F1: 0.5474683544303798, macro F1: 0.5474683544303798 @@ -8878,8 +8878,8 @@ Question 632: Is Andrew Lincoln the narrator of Million Dollar Traders? [INFO] Golden Answer: [true] [INFO] Predicted Answer: [true] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 633, P: 0.513, R: 0.563, macro F1: 0.519, macro F1*: 0.537 -[INFO] Current time: 1265 ms, average: 2731.33 ms/ques, entity detection: 422.99 ms/ques (linking tool: 330.54 ms/ques), relation detection: 901.99 ms/ques (similarity: 708.93 ms/ques), EDG generation: 48.88 ms/ques +[INFO] Cumulative metrics, sample: 633, P: 0.513, R: 0.563, macro F1: 0.537 +[INFO] Current question consumed time: 1086 ms, average: 2477.85 ms/ques, entity detection: 377.23 ms/ques (linking tool: 297.99 ms/ques), relation detection: 837.76 ms/ques (similarity: 698.20 ms/ques), EDG generation: 48.69 ms/ques [INFO] Entity Linking P: 0.7503949447077409, R: 0.7022116903633492, micro F1: 0.7174828857293322, macro F1: 0.7255041934150969 [INFO] Relation Linking P: 0.5930753027909426, R: 0.5410742496050553, micro F1: 0.5556082148499214, macro F1: 0.5658826452630932 [INFO] Type Linking P: 0.5481832543443917, R: 0.5481832543443917, micro F1: 0.5481832543443917, macro F1: 0.5481832543443917 @@ -8892,8 +8892,8 @@ Question 633: Who is the owner of Saumarez ? [INFO] Golden Answer: [http://dbpedia.org/resource/Wayne_Gretzky, http://dbpedia.org/resource/Bruce_McNall] [INFO] Predicted Answer: [http://dbpedia.org/resource/Wayne_Gretzky, http://dbpedia.org/resource/Bruce_McNall] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 634, P: 0.514, R: 0.564, macro F1: 0.519, macro F1*: 0.538 -[INFO] Current time: 806 ms, average: 2728.29 ms/ques, entity detection: 422.56 ms/ques (linking tool: 330.13 ms/ques), relation detection: 901.05 ms/ques (similarity: 708.10 ms/ques), EDG generation: 48.81 ms/ques +[INFO] Cumulative metrics, sample: 634, P: 0.514, R: 0.564, macro F1: 0.538 +[INFO] Current question consumed time: 696 ms, average: 2475.04 ms/ques, entity detection: 376.82 ms/ques (linking tool: 297.61 ms/ques), relation detection: 836.90 ms/ques (similarity: 697.39 ms/ques), EDG generation: 48.63 ms/ques [INFO] Entity Linking P: 0.750788643533123, R: 0.7026813880126183, micro F1: 0.7179284963196645, macro F1: 0.7259388837634428 [INFO] Relation Linking P: 0.5937171398527865, R: 0.5417981072555205, micro F1: 0.5563091482649846, macro F1: 0.5665706795863378 [INFO] Type Linking P: 0.5488958990536278, R: 0.5488958990536278, micro F1: 0.5488958990536278, macro F1: 0.5488958990536278 @@ -8906,8 +8906,8 @@ Question 634: What is telecasted on the network where Deborah Turness holds impo [INFO] Golden Answer: [http://dbpedia.org/resource/Cheers_(season_1), http://dbpedia.org/resource/Cheers_(season_2), http://dbpedia.org/resource/Cheers_(season_3), http://dbpedia.org/resource/Cheers_(season_4), http://dbpedia.org/resource/Cheers_(season_5), http://dbpedia.org/resource/Cheers_(season_6), http://dbpedia.org/resource/Cheers_(season_7), http://dbpedia.org/resource/Frasier_(season_9), http://dbpedia.org/resource/Friends, http://dbpedia.org/resource/It's_Punky_Brewster, http://dbpedia.org/resource/Scrubs_(TV_series), http://dbpedia.org/resource/St._Elsewhere, http://dbpedia.org/resource/The_Cosby_Show, http://dbpedia.org/resource/Alvin_and_the_Chipmunks_(1983_TV_series), http://dbpedia.org/resource/Dolly_Parton's_Coat_of_Many_Colors, http://dbpedia.org/resource/E-Ring, http://dbpedia.org/resource/Father_of_the_Pride, http://dbpedia.org/resource/High_School_U.S.A., http://dbpedia.org/resource/Information_Please, http://dbpedia.org/resource/My_Two_Dads, http://dbpedia.org/resource/Queen_for_a_Day, http://dbpedia.org/resource/Remington_Steele, http://dbpedia.org/resource/Rowan_&_Martin's_Laugh-In, http://dbpedia.org/resource/Saturday_Night_Live_(season_1), http://dbpedia.org/resource/The_Man_from_U.N.C.L.E., http://dbpedia.org/resource/The_Single_Guy, http://dbpedia.org/resource/The_Steve_Allen_Show, http://dbpedia.org/resource/The_Virginian_(TV_series), http://dbpedia.org/resource/Victoria_Beckham:_Coming_to_America, http://dbpedia.org/resource/Walt_Disney_anthology_television_series, http://dbpedia.org/resource/Banacek, http://dbpedia.org/resource/Buck_Rogers_in_the_25th_Century_(TV_series), http://dbpedia.org/resource/Guilty_or_Innocent:_The_Sam_Sheppard_Murder_Case, http://dbpedia.org/resource/Howdy_Doody, http://dbpedia.org/resource/I_Spy_(1965_TV_series), http://dbpedia.org/resource/Magnavox_Presents_Frank_Sinatra, http://dbpedia.org/resource/Suspicion_(TV_series), http://dbpedia.org/resource/The_Funny_Side, http://dbpedia.org/resource/The_Gary_Coleman_Show, http://dbpedia.org/resource/The_Ghost_&_Mrs._Muir_(TV_series), http://dbpedia.org/resource/The_High_Chaparral, http://dbpedia.org/resource/The_Kaiser_Aluminum_Hour, http://dbpedia.org/resource/The_New_Show, http://dbpedia.org/resource/The_Questor_Tapes, http://dbpedia.org/resource/To_Tell_the_Truth, http://dbpedia.org/resource/Exiled:_A_Law_&_Order_Movie, http://dbpedia.org/resource/Fame_(1982_TV_series), http://dbpedia.org/resource/Late_Night_with_Seth_Meyers, http://dbpedia.org/resource/Law_&_Order:_Special_Victims_Unit_(season_7), http://dbpedia.org/resource/On_the_Edge_of_Innocence, http://dbpedia.org/resource/Spider-Man_and_His_Amazing_Friends, http://dbpedia.org/resource/The_Jay_Leno_Show, http://dbpedia.org/resource/L.A._Takedown, http://dbpedia.org/resource/Rudolph_the_Red-Nosed_Reindeer_(TV_special), http://dbpedia.org/resource/The_Rocky_and_Bullwinkle_Show, http://dbpedia.org/resource/Quincy,_M.E., http://dbpedia.org/resource/The_First_Olympics:_Athens_1896, http://dbpedia.org/resource/The_Golden_Girls_(season_4), http://dbpedia.org/resource/A_Man_Called_Sloane, http://dbpedia.org/resource/Please_Don't_Eat_the_Daisies_(TV_series), http://dbpedia.org/resource/The_Parent_Trap_III, http://dbpedia.org/resource/The_Parent_Trap_IV:_Hawaiian_Honeymoon, http://dbpedia.org/resource/AFL_on_NBC, http://dbpedia.org/resource/The_Jensen_Project, http://dbpedia.org/resource/Three_Sisters_(TV_series), http://dbpedia.org/resource/50_Grand_Slam, http://dbpedia.org/resource/Against_the_Grain_(TV_series), http://dbpedia.org/resource/Eerie,_Indiana, http://dbpedia.org/resource/The_Dinah_Shore_Chevy_Show, http://dbpedia.org/resource/Halloween_with_the_New_Addams_Family, http://dbpedia.org/resource/It's_Worth_What%3F, http://dbpedia.org/resource/Lipshitz_Saves_the_World, http://dbpedia.org/resource/Little_Mo_(1978_film), http://dbpedia.org/resource/Saturday_Night_Live_(season_40), http://dbpedia.org/resource/The_Addams_Family_(1973_animated_series), http://dbpedia.org/resource/The_Bob_Cummings_Show, http://dbpedia.org/resource/The_Bold_Ones:_The_Protectors, http://dbpedia.org/resource/The_Cosby_Mysteries, http://dbpedia.org/resource/Whoopi, http://dbpedia.org/resource/Your_First_Impression, http://dbpedia.org/resource/Fire_on_the_Mountain_(1981_film), http://dbpedia.org/resource/Five_Fingers_(TV_series), http://dbpedia.org/resource/Liza_with_a_Z, http://dbpedia.org/resource/Rainbow_(1978_film), http://dbpedia.org/resource/Return_to_Mayberry, http://dbpedia.org/resource/The_Bold_Ones, http://dbpedia.org/resource/1_vs._100_(U.S._game_show), http://dbpedia.org/resource/Average_Joe_(TV_series), http://dbpedia.org/resource/Chain_Reaction_(game_show), http://dbpedia.org/resource/Jackpot_(game_show), http://dbpedia.org/resource/Time_Machine_(game_show), http://dbpedia.org/resource/Treasure_Hunt_(U.S._game_show), http://dbpedia.org/resource/Win,_Lose_or_Draw, http://dbpedia.org/resource/Banyon, http://dbpedia.org/resource/Café_Americain, http://dbpedia.org/resource/Camel_News_Caravan, http://dbpedia.org/resource/Celebrity_Family_Feud, http://dbpedia.org/resource/City_of_Angels_(1976_TV_series), http://dbpedia.org/resource/Dawn:_Portrait_of_a_Teenage_Runaway, http://dbpedia.org/resource/Fraggle_Rock:_The_Animated_Series, http://dbpedia.org/resource/I_Dream_of_Jeannie..._Fifteen_Years_Later, http://dbpedia.org/resource/Return_to_Peyton_Place_(TV_series), http://dbpedia.org/resource/Runaround_(game_show), http://dbpedia.org/resource/Surface_(TV_series), http://dbpedia.org/resource/Bonino_(TV_series), http://dbpedia.org/resource/Bret_Maverick, http://dbpedia.org/resource/Cursed_(TV_series), http://dbpedia.org/resource/LateLine, http://dbpedia.org/resource/Baggy_Pants_and_the_Nitwits, http://dbpedia.org/resource/Quantum_Leap_(season_2), http://dbpedia.org/resource/Quarterlife, http://dbpedia.org/resource/Something_for_a_Lonely_Man, http://dbpedia.org/resource/Then_Came_Bronson, http://dbpedia.org/resource/Wild_Kingdom, http://dbpedia.org/resource/Free_Agents_(U.S._TV_series), http://dbpedia.org/resource/Hitler's_SS:_Portrait_in_Evil, http://dbpedia.org/resource/Murder_in_Texas, http://dbpedia.org/resource/Revenge_of_the_Stepford_Wives, http://dbpedia.org/resource/The_Bob_Newhart_Show_(1961_TV_series), http://dbpedia.org/resource/Redigo_(TV_series), http://dbpedia.org/resource/Rockin'_Through_the_Decades, http://dbpedia.org/resource/Saturday_Night_Live_(season_39), http://dbpedia.org/resource/Ding_Dong_School, http://dbpedia.org/resource/M.Y.O.B._(TV_series), http://dbpedia.org/resource/Boone_(TV_series), http://dbpedia.org/resource/Kingpin_(TV_series), http://dbpedia.org/resource/Emergency_+4, http://dbpedia.org/resource/The_Snoop_Sisters, http://dbpedia.org/resource/30_Rock_(season_7), http://dbpedia.org/resource/33⅓_Revolutions_per_Monkee, http://dbpedia.org/resource/Battlestars_(game_show), http://dbpedia.org/resource/Joe_Forrester_(TV_series), http://dbpedia.org/resource/Law_&_Order_(season_9), http://dbpedia.org/resource/Little_Richard_(film), http://dbpedia.org/resource/Love,_Sidney, http://dbpedia.org/resource/Northwest_Passage_(TV_series), http://dbpedia.org/resource/The_More_You_Know, http://dbpedia.org/resource/Movin'_with_Nancy, http://dbpedia.org/resource/Scrubs_(season_4), http://dbpedia.org/resource/The_Spike_Jones_Show_(series), http://dbpedia.org/resource/Danger_Island_(film), http://dbpedia.org/resource/Dragnet_(1951_TV_series), http://dbpedia.org/resource/Fashion_Star_(season_1), http://dbpedia.org/resource/Living_Proof:_The_Hank_Williams,_Jr._Story, http://dbpedia.org/resource/Thank_God_You're_Here_(U.S._TV_series), http://dbpedia.org/resource/The_Real_Wedding_Crashers, http://dbpedia.org/resource/Fatal_Vision_(miniseries), http://dbpedia.org/resource/Call_My_Bluff_(U.S._game_show), http://dbpedia.org/resource/Trinity_(U.S._TV_series), http://dbpedia.org/resource/The_Biggest_Loser:_Families, http://dbpedia.org/resource/Saturday_Night_Live_(season_34), http://dbpedia.org/resource/The_Restaurant_(U.S._TV_series), http://dbpedia.org/resource/World's_Most_Amazing_Videos, http://dbpedia.org/resource/Your_Mother_Wears_Combat_Boots, http://dbpedia.org/resource/Parks_and_Recreation_(season_2), http://dbpedia.org/resource/Saturday_Night_Live_(season_10), http://dbpedia.org/resource/Saturday_Night_Live_(season_32), http://dbpedia.org/resource/Reach_for_the_Stars_(game_show), http://dbpedia.org/resource/The_Rockford_Files_(season_2), http://dbpedia.org/resource/Jigsaw_John_(TV_series), http://dbpedia.org/resource/Someone_Like_Me_(TV_series), http://dbpedia.org/resource/Winnetka_Road, http://dbpedia.org/resource/Combat_Academy, http://dbpedia.org/resource/Kate's_Secret, http://dbpedia.org/resource/Class_Cruise, http://dbpedia.org/resource/Science_of_Love, http://dbpedia.org/resource/Silent_Witness_(1985_film), http://dbpedia.org/resource/Moment_of_Truth:_Why_My_Daughter%3F, http://dbpedia.org/resource/Strong_(TV_series), http://dbpedia.org/resource/My_Name_Is_Earl_(season_4), http://dbpedia.org/resource/Chicago_Fire_(season_1), http://dbpedia.org/resource/Chicago_Fire_(season_2), http://dbpedia.org/resource/Chicago_Fire_(season_3), http://dbpedia.org/resource/Chicago_Fire_(season_4), http://dbpedia.org/resource/Chicago_P.D._(season_1), http://dbpedia.org/resource/Chicago_P.D._(season_2), http://dbpedia.org/resource/Chicago_P.D._(season_3), http://dbpedia.org/resource/Make_Your_Own_Kind_of_Music_(TV_series), http://dbpedia.org/resource/Knight_Rider_(season_4), http://dbpedia.org/resource/Fashion_Star_(season_2), http://dbpedia.org/resource/The_Six_of_Us, http://dbpedia.org/resource/The_Ted_Steele_Show, http://dbpedia.org/resource/Friday_Night_Lights_(season_2), http://dbpedia.org/resource/Take_a_Chance_(U.S._game_show), http://dbpedia.org/resource/Parenthood_(season_1), http://dbpedia.org/resource/Parenthood_(season_3), http://dbpedia.org/resource/Parenthood_(season_4), http://dbpedia.org/resource/Parenthood_(season_5), http://dbpedia.org/resource/Parenthood_(season_6), http://dbpedia.org/resource/Where_the_Girls_Are_(TV_program), http://dbpedia.org/resource/The_A-Team_(season_2), http://dbpedia.org/resource/The_A-Team_(season_3), http://dbpedia.org/resource/The_A-Team_(season_5), http://dbpedia.org/resource/Adam-12_(season_2), http://dbpedia.org/resource/Dragnet_(1967_series)_(season_1), http://dbpedia.org/resource/Dragnet_(1967_series)_(season_3), http://dbpedia.org/resource/Dragnet_(1967_series)_(season_4), http://dbpedia.org/resource/Ironside_(season_1), http://dbpedia.org/resource/Ironside_(season_2), http://dbpedia.org/resource/Ironside_(season_4), http://dbpedia.org/resource/Ironside_(season_5), http://dbpedia.org/resource/Ironside_(season_6), http://dbpedia.org/resource/Ironside_(season_8), http://dbpedia.org/resource/Quincy,_M.E._(season_1), http://dbpedia.org/resource/Quincy,_M.E._(season_2), http://dbpedia.org/resource/Quincy,_M.E._(season_3), http://dbpedia.org/resource/Quincy,_M.E._(season_4), http://dbpedia.org/resource/Quincy,_M.E._(season_5), http://dbpedia.org/resource/Quincy,_M.E._(season_6), http://dbpedia.org/resource/Quincy,_M.E._(season_7), http://dbpedia.org/resource/Quincy,_M.E._(season_8), http://dbpedia.org/resource/3rd_Rock_from_the_Sun, http://dbpedia.org/resource/Alfred_Hitchcock_Presents, http://dbpedia.org/resource/Alfred_Hitchcock_Presents_(1985_TV_series), http://dbpedia.org/resource/Jeopardy!, http://dbpedia.org/resource/Star_Trek:_The_Original_Series, http://dbpedia.org/resource/30_Rock_(season_3), http://dbpedia.org/resource/Crime_Story_(TV_series), http://dbpedia.org/resource/Frasier_(season_10), http://dbpedia.org/resource/Knight_Rider_(1982_TV_series), http://dbpedia.org/resource/Studio_60_on_the_Sunset_Strip, http://dbpedia.org/resource/The_Tonight_Show_Starring_Jimmy_Fallon, http://dbpedia.org/resource/Alice_in_Wonderland_(1999_film), http://dbpedia.org/resource/Caesar's_Hour, http://dbpedia.org/resource/College_Basketball_on_NBC, http://dbpedia.org/resource/Eischied, http://dbpedia.org/resource/Major_League_Baseball:_An_Inside_Look, http://dbpedia.org/resource/Thoroughbred_Racing_on_NBC, http://dbpedia.org/resource/WWE_Tribute_to_the_Troops, http://dbpedia.org/resource/American_Gladiators_(2008_TV_series), http://dbpedia.org/resource/Mann_&_Machine, http://dbpedia.org/resource/National_Heads-Up_Poker_Championship, http://dbpedia.org/resource/Ringo_(1978_film), http://dbpedia.org/resource/Space_Sentinels, http://dbpedia.org/resource/The_Golden_Girls_(season_1), http://dbpedia.org/resource/The_Rogues_(TV_series), http://dbpedia.org/resource/The_Yellow_Rose, http://dbpedia.org/resource/Suddenly_Susan, http://dbpedia.org/resource/A_Flintstone_Christmas, http://dbpedia.org/resource/Fred_and_Barney_Meet_The_Thing, http://dbpedia.org/resource/Fred_and_Barney_Meet_the_Shmoo, http://dbpedia.org/resource/Galaxy_Goof-Ups, http://dbpedia.org/resource/Serving_in_Silence:_The_Margarethe_Cammermeyer_Story, http://dbpedia.org/resource/The_Flintstone_Funnies, http://dbpedia.org/resource/The_New_Fred_and_Barney_Show, http://dbpedia.org/resource/You,_Me_and_the_Apocalypse, http://dbpedia.org/resource/Nearly_Departed, http://dbpedia.org/resource/Parks_and_Recreation_(season_7), http://dbpedia.org/resource/Sealab_2020, http://dbpedia.org/resource/The_Flintstones'_New_Neighbors, http://dbpedia.org/resource/The_Flintstones:_Fred's_Final_Fling, http://dbpedia.org/resource/The_Flintstones:_Jogging_Fever, http://dbpedia.org/resource/The_Flintstones:_Little_Big_League, http://dbpedia.org/resource/The_Flintstones:_Wind-Up_Wilma, http://dbpedia.org/resource/The_Flintstones_Meet_Rockula_and_Frankenstone, http://dbpedia.org/resource/Identity_(game_show), http://dbpedia.org/resource/On_Wings_of_Eagles_(miniseries), http://dbpedia.org/resource/Outlaws_(1960_TV_series), http://dbpedia.org/resource/Who_Said_That%3F, http://dbpedia.org/resource/The_Biggest_Loser:_No_Excuses, http://dbpedia.org/resource/The_Burning_Bed, http://dbpedia.org/resource/Awake_(TV_series), http://dbpedia.org/resource/The_Biggest_Loser_(season_3), http://dbpedia.org/resource/The_Book_of_Daniel_(TV_series), http://dbpedia.org/resource/Run,_Joe,_Run, http://dbpedia.org/resource/Sigmund_and_the_Sea_Monsters, http://dbpedia.org/resource/The_Bold_Ones:_The_Lawyers, http://dbpedia.org/resource/Alexander:_The_Other_Side_of_Dawn, http://dbpedia.org/resource/Law_&_Order:_Special_Victims_Unit_(season_10), http://dbpedia.org/resource/The_Gabby_Hayes_Show, http://dbpedia.org/resource/Deadline_(American_TV_series), http://dbpedia.org/resource/In_Love_and_War_(1987_film), http://dbpedia.org/resource/Saturday_Night_Live_(season_2), http://dbpedia.org/resource/She_Spies, http://dbpedia.org/resource/Adam_(1983_film), http://dbpedia.org/resource/The_Ruff_and_Reddy_Show, http://dbpedia.org/resource/McKeever_and_the_Colonel, http://dbpedia.org/resource/Sierra_(TV_series), http://dbpedia.org/resource/Columbo_(season_4), http://dbpedia.org/resource/Columbo_(season_5), http://dbpedia.org/resource/Jewels_(miniseries), http://dbpedia.org/resource/Kristin_(TV_series), http://dbpedia.org/resource/Quantum_Leap_(season_5), http://dbpedia.org/resource/The_Fresh_Prince_of_Bel-Air_(season_3), http://dbpedia.org/resource/The_Secret_Life_of_Kathy_McCormick, http://dbpedia.org/resource/Casper's_First_Christmas, http://dbpedia.org/resource/Doug_Henning's_World_of_Magic, http://dbpedia.org/resource/Most_Outrageous_Moments, http://dbpedia.org/resource/Betty_White's_Off_Their_Rockers, http://dbpedia.org/resource/Half_Nelson_(TV_series), http://dbpedia.org/resource/Jack_Frost_(TV_special), http://dbpedia.org/resource/Lookwell, http://dbpedia.org/resource/Saturday_Night_Live_(season_18), http://dbpedia.org/resource/Homeland_Security_(film), http://dbpedia.org/resource/Skunked_TV, http://dbpedia.org/resource/The_Man_and_the_Challenge, http://dbpedia.org/resource/Sweet_Justice, http://dbpedia.org/resource/A_Howling_in_the_Woods, http://dbpedia.org/resource/I_Still_Dream_of_Jeannie, http://dbpedia.org/resource/McDuff,_the_Talking_Dog, http://dbpedia.org/resource/In_the_Line_of_Duty:_The_F.B.I._Murders, http://dbpedia.org/resource/Legmen, http://dbpedia.org/resource/Revelations_(miniseries), http://dbpedia.org/resource/The_Innocent_(1994_film), http://dbpedia.org/resource/A_Chipmunk_Reunion, http://dbpedia.org/resource/Opposites_Attract_(1990_film), http://dbpedia.org/resource/Those_She_Left_Behind, http://dbpedia.org/resource/Popeye_Doyle_(film), http://dbpedia.org/resource/A_Child's_Cry_for_Help, http://dbpedia.org/resource/Frasier_(season_4), http://dbpedia.org/resource/Miracle_Child_(1993_film), http://dbpedia.org/resource/Snowbeast, http://dbpedia.org/resource/Grimm_(season_1), http://dbpedia.org/resource/Dragnet_(1967_series)_(season_2), http://dbpedia.org/resource/Come_to_Papa_(TV_series), http://dbpedia.org/resource/Lazer_Tag_Academy, http://dbpedia.org/resource/My_Son_Jeep, http://dbpedia.org/resource/Community_(season_2), http://dbpedia.org/resource/The_Jane_Whitney_Show, http://dbpedia.org/resource/The_Kids_From_C.A.P.E.R., http://dbpedia.org/resource/The_World's_Most_Dangerous_Magic, http://dbpedia.org/resource/My_Name_Is_Earl_(season_2), http://dbpedia.org/resource/The_Office_(U.S._season_2), http://dbpedia.org/resource/Amnesia_(game_show), http://dbpedia.org/resource/3rd_Rock_from_the_Sun_(season_2), http://dbpedia.org/resource/3rd_Rock_from_the_Sun_(season_6), http://dbpedia.org/resource/List_of_The_Tonight_Show_with_Jay_Leno_episodes_(2000–09), http://dbpedia.org/resource/The_Jack_Benny_Program_(season_15), http://dbpedia.org/resource/List_of_Late_Night_with_Jimmy_Fallon_episodes_(2010), http://dbpedia.org/resource/Harry's_Law_(season_1), http://dbpedia.org/resource/The_Fresh_Prince_of_Bel-Air_(season_5), http://dbpedia.org/resource/List_of_The_Tonight_Show_Starring_Johnny_Carson_episodes_(1974), http://dbpedia.org/resource/List_of_The_Tonight_Show_Starring_Johnny_Carson_episodes_(1975), http://dbpedia.org/resource/List_of_The_Tonight_Show_Starring_Johnny_Carson_episodes_(1976), http://dbpedia.org/resource/List_of_The_Tonight_Show_Starring_Johnny_Carson_episodes_(1978), http://dbpedia.org/resource/List_of_The_Tonight_Show_Starring_Johnny_Carson_episodes_(1979), http://dbpedia.org/resource/List_of_The_Tonight_Show_Starring_Johnny_Carson_episodes_(1980), http://dbpedia.org/resource/List_of_The_Tonight_Show_Starring_Johnny_Carson_episodes_(1981), http://dbpedia.org/resource/List_of_The_Tonight_Show_Starring_Johnny_Carson_episodes_(1982), http://dbpedia.org/resource/List_of_The_Tonight_Show_Starring_Johnny_Carson_episodes_(1983), http://dbpedia.org/resource/List_of_The_Tonight_Show_Starring_Johnny_Carson_episodes_(1984), http://dbpedia.org/resource/List_of_The_Tonight_Show_Starring_Johnny_Carson_episodes_(1985), http://dbpedia.org/resource/List_of_The_Tonight_Show_Starring_Johnny_Carson_episodes_(1986), http://dbpedia.org/resource/List_of_The_Tonight_Show_Starring_Johnny_Carson_episodes_(1987), http://dbpedia.org/resource/List_of_The_Tonight_Show_Starring_Johnny_Carson_episodes_(1988), http://dbpedia.org/resource/List_of_The_Tonight_Show_Starring_Johnny_Carson_episodes_(1989), http://dbpedia.org/resource/List_of_The_Tonight_Show_Starring_Johnny_Carson_episodes_(1990), http://dbpedia.org/resource/List_of_The_Tonight_Show_Starring_Johnny_Carson_episodes_(1991), http://dbpedia.org/resource/List_of_The_Tonight_Show_Starring_Johnny_Carson_episodes_(1992), http://dbpedia.org/resource/List_of_Late_Night_with_Jimmy_Fallon_episodes_(2009), http://dbpedia.org/resource/List_of_Late_Night_with_Jimmy_Fallon_episodes_(2011), http://dbpedia.org/resource/List_of_The_Tonight_Show_Starring_Jimmy_Fallon_episodes_(2014), http://dbpedia.org/resource/List_of_The_Tonight_Show_Starring_Jimmy_Fallon_episodes_(2016), http://dbpedia.org/resource/Dateline_NBC, http://dbpedia.org/resource/I_Dream_of_Jeannie, http://dbpedia.org/resource/Kings_(U.S._TV_series), http://dbpedia.org/resource/Medium_(TV_series), http://dbpedia.org/resource/Miami_Vice, http://dbpedia.org/resource/SeaQuest_DSV, http://dbpedia.org/resource/The_Celebrity_Apprentice, http://dbpedia.org/resource/The_Incredible_Hulk_(1978_TV_series), http://dbpedia.org/resource/The_Tonight_Show, http://dbpedia.org/resource/The_Tonight_Show_with_Jay_Leno, http://dbpedia.org/resource/Wheel_of_Fortune_(U.S._game_show), http://dbpedia.org/resource/Bionic_Woman_(2007_TV_series), http://dbpedia.org/resource/Homicide:_Life_on_the_Street_(season_6), http://dbpedia.org/resource/L.A._Law, http://dbpedia.org/resource/Law_&_Order:_Special_Victims_Unit, http://dbpedia.org/resource/Law_&_Order:_Special_Victims_Unit_(season_8), http://dbpedia.org/resource/Love,_Lies_and_Murder, http://dbpedia.org/resource/Misfits_of_Science, http://dbpedia.org/resource/Taxi_(TV_series), http://dbpedia.org/resource/The_Bionic_Woman, http://dbpedia.org/resource/The_Year_Without_a_Santa_Claus_(2006_film), http://dbpedia.org/resource/Earth_2_(TV_series), http://dbpedia.org/resource/Riverboat_(TV_series), http://dbpedia.org/resource/Underdog_(TV_series), http://dbpedia.org/resource/The_Apprentice_(TV_series), http://dbpedia.org/resource/Windfall_(TV_series), http://dbpedia.org/resource/Ellery_Queen_(TV_series), http://dbpedia.org/resource/Seinfeld_(season_1), http://dbpedia.org/resource/Seinfeld_(season_2), http://dbpedia.org/resource/Seinfeld_(season_3), http://dbpedia.org/resource/Seinfeld_(season_4), http://dbpedia.org/resource/Seinfeld_(season_5), http://dbpedia.org/resource/Seinfeld_(season_6), http://dbpedia.org/resource/Seinfeld_(season_7), http://dbpedia.org/resource/Seinfeld_(season_8), http://dbpedia.org/resource/Seinfeld_(season_9), http://dbpedia.org/resource/The_John_Larroquette_Show, http://dbpedia.org/resource/The_Michael_Richards_Show, http://dbpedia.org/resource/Watching_Ellie, http://dbpedia.org/resource/Backstairs_at_the_White_House, http://dbpedia.org/resource/Love_in_the_Wild, http://dbpedia.org/resource/The_Contender_(TV_series), http://dbpedia.org/resource/Choose_Up_Sides, http://dbpedia.org/resource/Dough_Re_Mi, http://dbpedia.org/resource/Make_the_Connection, http://dbpedia.org/resource/Sale_of_the_Century_(U.S._game_show), http://dbpedia.org/resource/The_Two_Mrs._Grenvilles, http://dbpedia.org/resource/3rd_Rock_from_the_Sun_(season_1), http://dbpedia.org/resource/3rd_Rock_from_the_Sun_(season_5), http://dbpedia.org/resource/America's_Most_Talented_Kid, http://dbpedia.org/resource/From_Here_to_Eternity_(miniseries), http://dbpedia.org/resource/I'll_Fly_Away_(TV_series), http://dbpedia.org/resource/I_Married_Joan, http://dbpedia.org/resource/Something_So_Right_(TV_series), http://dbpedia.org/resource/Bare_Essence, http://dbpedia.org/resource/C.P.O._Sharkey, http://dbpedia.org/resource/Freaks_and_Geeks, http://dbpedia.org/resource/Harper_Valley_PTA_(TV_series), http://dbpedia.org/resource/Heist_(TV_series), http://dbpedia.org/resource/Kath_&_Kim_(U.S._TV_series), http://dbpedia.org/resource/Lovers_and_Friends, http://dbpedia.org/resource/My_Hero_(U.S._TV_series), http://dbpedia.org/resource/Needles_and_Pins_(TV_series), http://dbpedia.org/resource/Rescue_from_Gilligan's_Island, http://dbpedia.org/resource/Saturday_Night_Live_(season_21), http://dbpedia.org/resource/Saturday_Night_Live_(season_36), http://dbpedia.org/resource/Saturday_Night_Live_(season_6), http://dbpedia.org/resource/Sirota's_Court, http://dbpedia.org/resource/Teachers_(2006_U.S._TV_series), http://dbpedia.org/resource/The_Bill_Cosby_Show, http://dbpedia.org/resource/The_Castaways_on_Gilligan's_Island, http://dbpedia.org/resource/The_Duck_Factory, http://dbpedia.org/resource/The_Others_(TV_series), http://dbpedia.org/resource/The_Pursuit_of_Happiness_(TV_series), http://dbpedia.org/resource/Twenty_Good_Years, http://dbpedia.org/resource/My_Name_Is_Earl_(season_1), http://dbpedia.org/resource/Raines, http://dbpedia.org/resource/Journeyman_(TV_series), http://dbpedia.org/resource/List_of_The_Tonight_Show_Starring_Johnny_Carson_episodes_(1977), http://dbpedia.org/resource/Born_Innocent_(film), http://dbpedia.org/resource/Polly_(1989_film), http://dbpedia.org/resource/How_to_Break_Up_a_Happy_Divorce, http://dbpedia.org/resource/Leave_It_to_the_Girls, http://dbpedia.org/resource/The_Greatest_Gift_(film), http://dbpedia.org/resource/America's_Next_Great_Restaurant, http://dbpedia.org/resource/A_Cry_for_Help:_The_Tracey_Thurman_Story, http://dbpedia.org/resource/Columbo_(season_1), http://dbpedia.org/resource/Columbo_(season_6), http://dbpedia.org/resource/Columbo_(season_7), http://dbpedia.org/resource/The_Hero_(1966_TV_series), http://dbpedia.org/resource/100_Questions, http://dbpedia.org/resource/Face_the_Ace, http://dbpedia.org/resource/Hank_(1965_TV_series), http://dbpedia.org/resource/Hawaii_(TV_series), http://dbpedia.org/resource/Inconceivable, http://dbpedia.org/resource/Karen_(1964_TV_series), http://dbpedia.org/resource/Meet_My_Folks, http://dbpedia.org/resource/Mr._Rhodes, http://dbpedia.org/resource/Pride_&_Joy_(TV_series), http://dbpedia.org/resource/The_Great_American_Road_Trip, http://dbpedia.org/resource/The_Night_They_Took_Miss_Beautiful, http://dbpedia.org/resource/Punch_and_Jody, http://dbpedia.org/resource/Somerset_Maugham_TV_Theatre, http://dbpedia.org/resource/Undercovers_(TV_series), http://dbpedia.org/resource/Growing_Up_Brady_(film), http://dbpedia.org/resource/How_to_Eat_Like_a_Child, http://dbpedia.org/resource/In_the_Beginning_(miniseries), http://dbpedia.org/resource/Homicide:_Life_on_the_Street_(season_1), http://dbpedia.org/resource/Homicide:_Life_on_the_Street_(season_2), http://dbpedia.org/resource/Homicide:_Life_on_the_Street_(season_3), http://dbpedia.org/resource/Homicide:_Life_on_the_Street_(season_4), http://dbpedia.org/resource/Mindreaders, http://dbpedia.org/resource/Crash:_The_Mystery_of_Flight_1501, http://dbpedia.org/resource/My_Dad_Is_Better_Than_Your_Dad, http://dbpedia.org/resource/Doomsday_Virus, http://dbpedia.org/resource/Law_&_Order:_Criminal_Intent_(season_5), http://dbpedia.org/resource/Sportsworld_(U.S._TV_series), http://dbpedia.org/resource/The_Home_Court, http://dbpedia.org/resource/Parks_and_Recreation_(season_4), http://dbpedia.org/resource/Parks_and_Recreation_(season_6), http://dbpedia.org/resource/In_the_Loop_with_iVillage, http://dbpedia.org/resource/Law_&_Order:_Criminal_Intent_(season_1), http://dbpedia.org/resource/Law_&_Order:_Criminal_Intent_(season_2), http://dbpedia.org/resource/Law_&_Order:_Criminal_Intent_(season_4), http://dbpedia.org/resource/Law_&_Order:_Criminal_Intent_(season_6), http://dbpedia.org/resource/Saturday_Night_Live_(season_14), http://dbpedia.org/resource/Saturday_Night_Live_(season_15), http://dbpedia.org/resource/Saturday_Night_Live_(season_17), http://dbpedia.org/resource/Saturday_Night_Live_(season_22), http://dbpedia.org/resource/Saturday_Night_Live_(season_25), http://dbpedia.org/resource/Saturday_Night_Live_(season_33), http://dbpedia.org/resource/Saturday_Night_Live_(season_41), http://dbpedia.org/resource/Saturday_Night_Live_(season_8), http://dbpedia.org/resource/Community_(season_1), http://dbpedia.org/resource/Eyes_of_Terror, http://dbpedia.org/resource/Stonestreet:_Who_Killed_the_Centerfold_Model%3F, http://dbpedia.org/resource/The_West_Wing_(season_3), http://dbpedia.org/resource/Money_Talks_News, http://dbpedia.org/resource/Lost_(game_show), http://dbpedia.org/resource/The_Biggest_Loser:_Battle_of_the_Ages, http://dbpedia.org/resource/The_Biggest_Loser:_Challenge_America, http://dbpedia.org/resource/The_Biggest_Loser:_Pay_It_Forward, http://dbpedia.org/resource/The_Biggest_Loser:_Second_Chances, http://dbpedia.org/resource/The_Biggest_Loser_(season_1), http://dbpedia.org/resource/The_Biggest_Loser_(season_2), http://dbpedia.org/resource/The_Biggest_Loser_(season_4), http://dbpedia.org/resource/Harry's_Law_(season_2), http://dbpedia.org/resource/Grimm_(season_3), http://dbpedia.org/resource/Community_(season_4), http://dbpedia.org/resource/Boss_Lady, http://dbpedia.org/resource/Frasier_(season_7), http://dbpedia.org/resource/Casper's_Halloween_Special, http://dbpedia.org/resource/Son-Rise:_A_Miracle_of_Love, http://dbpedia.org/resource/The_Golden_Girls_(season_7), http://dbpedia.org/resource/Quantum_Leap_(season_1), http://dbpedia.org/resource/Quantum_Leap_(season_4), http://dbpedia.org/resource/Three_Wishes_(U.S._TV_series), http://dbpedia.org/resource/List_of_Late_Night_with_Jimmy_Fallon_episodes_(2013–14), http://dbpedia.org/resource/List_of_The_Tonight_Show_Starring_Jimmy_Fallon_episodes_(2015), http://dbpedia.org/resource/List_of_Late_Night_with_Seth_Meyers_episodes_(2014), http://dbpedia.org/resource/List_of_Late_Night_with_Seth_Meyers_episodes_(2016), http://dbpedia.org/resource/30_Rock, http://dbpedia.org/resource/A.D._The_Bible_Continues, http://dbpedia.org/resource/ALF:_The_Animated_Series, http://dbpedia.org/resource/Another_World_(TV_series), http://dbpedia.org/resource/Candid_Camera, http://dbpedia.org/resource/Days_of_Our_Lives, http://dbpedia.org/resource/Hallmark_Hall_of_Fame, http://dbpedia.org/resource/Heidi_(1968_film), http://dbpedia.org/resource/Law_&_Order, http://dbpedia.org/resource/Law_&_Order_(season_5), http://dbpedia.org/resource/Meet_the_Press, http://dbpedia.org/resource/Seinfeld, http://dbpedia.org/resource/Treasure_Hunters_(TV_series), http://dbpedia.org/resource/Wild_Texas_Wind, http://dbpedia.org/resource/Amazing_Stories_(TV_series), http://dbpedia.org/resource/Dr._Kildare_(TV_series), http://dbpedia.org/resource/J.J._Starbuck, http://dbpedia.org/resource/Land_of_the_Lost_(1974_TV_series), http://dbpedia.org/resource/Late_Night_with_Jimmy_Fallon, http://dbpedia.org/resource/Man_of_the_People_(TV_series), http://dbpedia.org/resource/Nashville_Star, http://dbpedia.org/resource/The_Danny_Thomas_Hour, http://dbpedia.org/resource/The_Death_of_the_Incredible_Hulk, http://dbpedia.org/resource/The_Incredible_Hulk_Returns, http://dbpedia.org/resource/The_Lieutenant, http://dbpedia.org/resource/The_Magician_(U.S._TV_series), http://dbpedia.org/resource/The_Oregon_Trail_(TV_series), http://dbpedia.org/resource/The_Trial_of_the_Incredible_Hulk, http://dbpedia.org/resource/You_Bet_Your_Life, http://dbpedia.org/resource/A_Killer_Among_Us, http://dbpedia.org/resource/Bitter_Harvest_(1981_film), http://dbpedia.org/resource/CHiPs, http://dbpedia.org/resource/Centennial_(miniseries), http://dbpedia.org/resource/Ed_(TV_series), http://dbpedia.org/resource/Friday_Night_Lights_(TV_series), http://dbpedia.org/resource/Hollywood_Squares, http://dbpedia.org/resource/Kidnapped_(TV_series), http://dbpedia.org/resource/Late_Night_(NBC), http://dbpedia.org/resource/NASCAR_on_NBC, http://dbpedia.org/resource/Police_Story_(TV_series), http://dbpedia.org/resource/The_A-Team, http://dbpedia.org/resource/The_David_Letterman_Show, http://dbpedia.org/resource/The_Dick_Powell_Show, http://dbpedia.org/resource/The_Gong_Show, http://dbpedia.org/resource/The_Tonight_Show_with_Conan_O'Brien, http://dbpedia.org/resource/Babes_in_Toyland_(1986_film), http://dbpedia.org/resource/Caroline_in_the_City, http://dbpedia.org/resource/Cocaine:_One_Man's_Seduction, http://dbpedia.org/resource/David_Copperfield_(1993_film), http://dbpedia.org/resource/Elf:_Buddy's_Musical_Christmas, http://dbpedia.org/resource/Friday_Night_Videos, http://dbpedia.org/resource/Hollywood_Game_Night, http://dbpedia.org/resource/Homicide:_Life_on_the_Street_(season_5), http://dbpedia.org/resource/How_Murray_Saved_Christmas, http://dbpedia.org/resource/Light's_Diamond_Jubilee, http://dbpedia.org/resource/McMillan_&_Wife, http://dbpedia.org/resource/Mr._Krueger's_Christmas, http://dbpedia.org/resource/My_World_and_Welcome_to_It, http://dbpedia.org/resource/Passions, http://dbpedia.org/resource/Phenomenon_(TV_series), http://dbpedia.org/resource/Poker_After_Dark, http://dbpedia.org/resource/Providence_(TV_series), http://dbpedia.org/resource/Saturday_Night_Live_(season_7), http://dbpedia.org/resource/Saved_by_the_Bell, http://dbpedia.org/resource/Shannon's_Deal, http://dbpedia.org/resource/The_Family_Tree, http://dbpedia.org/resource/The_Jimmy_Stewart_Show, http://dbpedia.org/resource/The_Magic_Clown, http://dbpedia.org/resource/The_New_Fantastic_Four, http://dbpedia.org/resource/The_Office_(U.S._TV_series), http://dbpedia.org/resource/The_Office_(U.S._season_7), http://dbpedia.org/resource/Thundarr_the_Barbarian, http://dbpedia.org/resource/Tonight_Starring_Jack_Paar, http://dbpedia.org/resource/Veronica's_Closet, http://dbpedia.org/resource/Death_of_a_Centerfold:_The_Dorothy_Stratten_Story, http://dbpedia.org/resource/Ensign_O'Toole, http://dbpedia.org/resource/Grandpa_Goes_to_Washington, http://dbpedia.org/resource/Poison_Ivy_(1985_film), http://dbpedia.org/resource/The_Harlem_Globetrotters_on_Gilligan's_Island, http://dbpedia.org/resource/The_Restless_Gun, http://dbpedia.org/resource/The_Black_Donnellys, http://dbpedia.org/resource/Law_&_Order:_Special_Victims_Unit_(season_14), http://dbpedia.org/resource/Undateable, http://dbpedia.org/resource/Fame_Is_the_Name_of_the_Game, http://dbpedia.org/resource/Saturday_Night_Live_(season_28), http://dbpedia.org/resource/Saturday_Night_Live_(season_29), http://dbpedia.org/resource/The_Incredible_Hulk_(1982_TV_series), http://dbpedia.org/resource/The_Odyssey_(miniseries), http://dbpedia.org/resource/Brand_New_Life_(TV_series), http://dbpedia.org/resource/Everything's_Relative_(1999_TV_series), http://dbpedia.org/resource/30_Rock_(season_4), http://dbpedia.org/resource/Snap_Judgment_(game_show), http://dbpedia.org/resource/Will_&_Grace_(season_1), http://dbpedia.org/resource/Will_&_Grace_(season_2), http://dbpedia.org/resource/Will_&_Grace_(season_3), http://dbpedia.org/resource/Will_&_Grace_(season_4), http://dbpedia.org/resource/Will_&_Grace_(season_5), http://dbpedia.org/resource/Will_&_Grace_(season_6), http://dbpedia.org/resource/Will_&_Grace_(season_7), http://dbpedia.org/resource/Will_&_Grace_(season_8), http://dbpedia.org/resource/Child_Bride_of_Short_Creek, http://dbpedia.org/resource/Sarah_T._–_Portrait_of_a_Teenage_Alcoholic, http://dbpedia.org/resource/The_Rockford_Files_(season_3), http://dbpedia.org/resource/Star_Trek:_The_Original_Series_(season_1), http://dbpedia.org/resource/Star_Trek:_The_Original_Series_(season_2), http://dbpedia.org/resource/The_Kid_with_the_200_I.Q., http://dbpedia.org/resource/The_Possessed_(1977_film), http://dbpedia.org/resource/One_Happy_Family, http://dbpedia.org/resource/Say_When!!, http://dbpedia.org/resource/Space_Cats, http://dbpedia.org/resource/The_Red_Pony_(1973_film), http://dbpedia.org/resource/Mama's_Boy_(TV_series), http://dbpedia.org/resource/Crusoe_(TV_series), http://dbpedia.org/resource/She_Cried_No, http://dbpedia.org/resource/30_Rock_(season_6), http://dbpedia.org/resource/The_Little_Rascals_Christmas_Special, http://dbpedia.org/resource/The_Rockford_Files_(season_1), http://dbpedia.org/resource/Meteor_(miniseries), http://dbpedia.org/resource/Rosetti_and_Ryan, http://dbpedia.org/resource/Who's_Watching_the_Kids%3F, http://dbpedia.org/resource/Diana:_Her_True_Story, http://dbpedia.org/resource/Great_Ghost_Tales, http://dbpedia.org/resource/The_Rockford_Files_(season_4), http://dbpedia.org/resource/The_A-Team_(season_4), http://dbpedia.org/resource/Norby_(TV_series), http://dbpedia.org/resource/The_Armed_Forces_Hour, http://dbpedia.org/resource/Friday_Night_Lights_(season_4), http://dbpedia.org/resource/My_Name_Is_Earl_(season_3), http://dbpedia.org/resource/List_of_The_Tonight_Show_with_Jay_Leno_episodes_(1992–99), http://dbpedia.org/resource/List_of_The_Tonight_Show_with_Jay_Leno_episodes_(2010), http://dbpedia.org/resource/The_Winds_of_Whoopie, http://dbpedia.org/resource/Cheers_(season_10), http://dbpedia.org/resource/The_Biggest_Loser_(Dutch_TV_series), http://dbpedia.org/resource/Baywatch, http://dbpedia.org/resource/Cheers, http://dbpedia.org/resource/Fear_Factor, http://dbpedia.org/resource/Gambit_(game_show), http://dbpedia.org/resource/Major_League_Baseball_on_NBC, http://dbpedia.org/resource/NBC_Nightly_News, http://dbpedia.org/resource/The_Baseball_Network, http://dbpedia.org/resource/The_Fresh_Prince_of_Bel-Air, http://dbpedia.org/resource/The_Revenge_of_Al_Capone, http://dbpedia.org/resource/The_Tonight_Show_Starring_Johnny_Carson, http://dbpedia.org/resource/The_West_Wing, http://dbpedia.org/resource/This_Is_Your_Life, http://dbpedia.org/resource/Today_(U.S._TV_program), http://dbpedia.org/resource/Will_&_Grace, http://dbpedia.org/resource/Your_Show_of_Shows, http://dbpedia.org/resource/A_Different_World, http://dbpedia.org/resource/Hannibal_(TV_series), http://dbpedia.org/resource/Law_&_Order:_Special_Victims_Unit_(season_13), http://dbpedia.org/resource/Let's_Make_a_Deal, http://dbpedia.org/resource/My_Name_Is_Earl, http://dbpedia.org/resource/Night_Court, http://dbpedia.org/resource/Star_Trek:_The_Animated_Series, http://dbpedia.org/resource/The_Price_Is_Right_(1956_U.S._game_show), http://dbpedia.org/resource/This_Is_Show_Business, http://dbpedia.org/resource/Truth_or_Consequences, http://dbpedia.org/resource/Twenty_One_(game_show), http://dbpedia.org/resource/Unsolved_Mysteries, http://dbpedia.org/resource/The_Hobbit_(1977_film), http://dbpedia.org/resource/The_Murder_of_Mary_Phagan, http://dbpedia.org/resource/Bates_Motel_(film), http://dbpedia.org/resource/Dick_Clark's_New_Year's_Rockin'_Eve, http://dbpedia.org/resource/Law_of_the_Plainsman, http://dbpedia.org/resource/Lipstick_Jungle_(TV_series), http://dbpedia.org/resource/Scared_Shrekless, http://dbpedia.org/resource/Gavilan_(TV_series), http://dbpedia.org/resource/Name_That_Tune, http://dbpedia.org/resource/Profiler_(TV_series), http://dbpedia.org/resource/American_Dreams, http://dbpedia.org/resource/Crusader_Rabbit, http://dbpedia.org/resource/Dante_(TV_series), http://dbpedia.org/resource/King_Leonardo_and_His_Short_Subjects, http://dbpedia.org/resource/Meet_McGraw, http://dbpedia.org/resource/The_Loneliest_Runner, http://dbpedia.org/resource/Laura_Lansing_Slept_Here, http://dbpedia.org/resource/One_Christmas_(film), http://dbpedia.org/resource/Parenthood_(1990_TV_series), http://dbpedia.org/resource/The_Rebels_(TV_series), http://dbpedia.org/resource/The_Troubleshooters_(1959_TV_series), http://dbpedia.org/resource/Law_&_Order:_Special_Victims_Unit_(season_1), http://dbpedia.org/resource/Law_&_Order:_Special_Victims_Unit_(season_2), http://dbpedia.org/resource/Sunday_Night_(American_TV_program), http://dbpedia.org/resource/CB_Bears, http://dbpedia.org/resource/Casper_and_the_Angels, http://dbpedia.org/resource/Foofur, http://dbpedia.org/resource/Going_Bananas_(U.S._series), http://dbpedia.org/resource/Hey,_Landlord, http://dbpedia.org/resource/Jokebook, http://dbpedia.org/resource/Match_Game-Hollywood_Squares_Hour, http://dbpedia.org/resource/Shirley_(TV_series), http://dbpedia.org/resource/Shirt_Tales, http://dbpedia.org/resource/Snorks, http://dbpedia.org/resource/Space_Stars, http://dbpedia.org/resource/The_Space_Kidettes, http://dbpedia.org/resource/Wheelie_and_the_Chopper_Bunch, http://dbpedia.org/resource/Yo_Yogi!, http://dbpedia.org/resource/Yogi's_Space_Race, http://dbpedia.org/resource/Danielle_Steel's_Star, http://dbpedia.org/resource/Family_Album_(miniseries), http://dbpedia.org/resource/Mister_Sterling, http://dbpedia.org/resource/Perry_Mason_(TV_movies), http://dbpedia.org/resource/Saturday_Night's_Main_Event, http://dbpedia.org/resource/Saturday_Night_Live_(season_9), http://dbpedia.org/resource/Tate_(TV_series), http://dbpedia.org/resource/The_Firm_(2012_TV_series), http://dbpedia.org/resource/Michael_Shayne, http://dbpedia.org/resource/My_Sweet_Charlie, http://dbpedia.org/resource/The_Californians_(TV_series), http://dbpedia.org/resource/For_Your_Love_(TV_series), http://dbpedia.org/resource/Tennis_on_NBC, http://dbpedia.org/resource/The_Gangster_Chronicles, http://dbpedia.org/resource/The_Great_Escape_II:_The_Untold_Story, http://dbpedia.org/resource/The_Temptations_(miniseries), http://dbpedia.org/resource/A_Year_in_the_Life, http://dbpedia.org/resource/Blind_Date_(1950s_game_show), http://dbpedia.org/resource/Brotherly_Love_(1995_TV_series), http://dbpedia.org/resource/Fall_into_Darkness, http://dbpedia.org/resource/Hidden_Faces, http://dbpedia.org/resource/How_to_Survive_a_Marriage, http://dbpedia.org/resource/My_Friend_Tony, http://dbpedia.org/resource/Our_Five_Daughters, http://dbpedia.org/resource/Saber_of_London, http://dbpedia.org/resource/Sister,_Sister_(1982_film), http://dbpedia.org/resource/The_Bold_Ones:_The_Senator, http://dbpedia.org/resource/The_Brighter_Day, http://dbpedia.org/resource/The_John_Forsythe_Show, http://dbpedia.org/resource/The_Round_Table_(TV_series), http://dbpedia.org/resource/The_Sophisticated_Gents, http://dbpedia.org/resource/A_Reason_to_Live_(1985_film), http://dbpedia.org/resource/Little_House_on_the_Prairie_(film), http://dbpedia.org/resource/Law_&_Order:_Special_Victims_Unit_(season_11), http://dbpedia.org/resource/Law_&_Order:_Special_Victims_Unit_(season_12), http://dbpedia.org/resource/Law_&_Order:_Special_Victims_Unit_(season_15), http://dbpedia.org/resource/Law_&_Order:_Special_Victims_Unit_(season_16), http://dbpedia.org/resource/Law_&_Order:_Special_Victims_Unit_(season_17), http://dbpedia.org/resource/Law_&_Order:_Special_Victims_Unit_(season_3), http://dbpedia.org/resource/Law_&_Order:_Special_Victims_Unit_(season_4), http://dbpedia.org/resource/Law_&_Order:_Special_Victims_Unit_(season_5), http://dbpedia.org/resource/Law_&_Order:_Special_Victims_Unit_(season_6), http://dbpedia.org/resource/Andy_Barker,_P.I., http://dbpedia.org/resource/Saturday_Night_Live_(season_26), http://dbpedia.org/resource/Saturday_Night_Live_(season_27), http://dbpedia.org/resource/Saturday_Night_Live_(season_30), http://dbpedia.org/resource/Scrubs_(season_1), http://dbpedia.org/resource/Scrubs_(season_2), http://dbpedia.org/resource/Scrubs_(season_3), http://dbpedia.org/resource/Scrubs_(season_5), http://dbpedia.org/resource/Scrubs_(season_6), http://dbpedia.org/resource/Scrubs_(season_7), http://dbpedia.org/resource/The_Golden_Girls_(season_2), http://dbpedia.org/resource/Saturday_Night_Live_(season_38), http://dbpedia.org/resource/The_Berenstain_Bears_Meet_Bigpaw, http://dbpedia.org/resource/ALF_Tales, http://dbpedia.org/resource/Private_Eye_(TV_series), http://dbpedia.org/resource/Age_of_Love_(TV_series), http://dbpedia.org/resource/Clash_of_the_Choirs, http://dbpedia.org/resource/A.U.S.A., http://dbpedia.org/resource/Frasier_(season_5), http://dbpedia.org/resource/The_Gambler_(film_series), http://dbpedia.org/resource/The_Jake_Effect, http://dbpedia.org/resource/The_Second_Half, http://dbpedia.org/resource/Crisis_Center, http://dbpedia.org/resource/PDQ_(game_show), http://dbpedia.org/resource/What's_This_Song%3F, http://dbpedia.org/resource/The_Stepford_Children, http://dbpedia.org/resource/Built_to_Last_(TV_series), http://dbpedia.org/resource/House_Rules_(1998_TV_series), http://dbpedia.org/resource/Mr._and_Mrs._North, http://dbpedia.org/resource/Black_Tie_Affair_(TV_series), http://dbpedia.org/resource/Chicago_Sons, http://dbpedia.org/resource/The_World_of_Mr._Sweeney, http://dbpedia.org/resource/Friday_Night_Lights_(season_1), http://dbpedia.org/resource/Snip_(TV_series), http://dbpedia.org/resource/Her_Hidden_Truth, http://dbpedia.org/resource/If_Someone_Had_Known, http://dbpedia.org/resource/Police_Story:_Confessions_of_a_Lady_Cop, http://dbpedia.org/resource/Parks_and_Recreation_(season_1), http://dbpedia.org/resource/Parks_and_Recreation_(season_3), http://dbpedia.org/resource/Parks_and_Recreation_(season_5), http://dbpedia.org/resource/The_West_Wing_(season_6), http://dbpedia.org/resource/The_Biggest_Loser:_Glory_Days, http://dbpedia.org/resource/Valley_Forge_(film), http://dbpedia.org/resource/The_Fresh_Prince_of_Bel-Air_(season_4), http://dbpedia.org/resource/Rewind_(2013_film), http://dbpedia.org/resource/Community_(season_5), http://dbpedia.org/resource/A_Burning_Passion:_The_Margaret_Mitchell_Story, http://dbpedia.org/resource/The_Crisis_(TV_series), http://dbpedia.org/resource/In_Wine_Country, http://dbpedia.org/resource/Friday_Night_Lights_(season_3), http://dbpedia.org/resource/People_Will_Talk_(game_show), http://dbpedia.org/resource/The_Office:_The_Accountants, http://dbpedia.org/resource/The_Swift_Show, http://dbpedia.org/resource/Frasier_(season_11), http://dbpedia.org/resource/The_Blacklist_(season_3), http://dbpedia.org/resource/Chain_Letter_(U.S._game_show), http://dbpedia.org/resource/3rd_Rock_from_the_Sun_(season_3), http://dbpedia.org/resource/Just_Shoot_Me!_(season_1), http://dbpedia.org/resource/Just_Shoot_Me!_(season_2), http://dbpedia.org/resource/Just_Shoot_Me!_(season_3), http://dbpedia.org/resource/Just_Shoot_Me!_(season_4), http://dbpedia.org/resource/Just_Shoot_Me!_(season_5), http://dbpedia.org/resource/Just_Shoot_Me!_(season_6), http://dbpedia.org/resource/Just_Shoot_Me!_(season_7), http://dbpedia.org/resource/The_Rockford_Files_(season_5), http://dbpedia.org/resource/The_Rockford_Files_(season_6), http://dbpedia.org/resource/All_Star_Secrets, http://dbpedia.org/resource/Columbo, http://dbpedia.org/resource/Crossing_Jordan, http://dbpedia.org/resource/Cruel_Doubt, http://dbpedia.org/resource/ER_(TV_series), http://dbpedia.org/resource/Empire_(1962_TV_series), http://dbpedia.org/resource/Football_Night_in_America, http://dbpedia.org/resource/Frasier, http://dbpedia.org/resource/Generations_(U.S._TV_series), http://dbpedia.org/resource/Golf_Channel_on_NBC, http://dbpedia.org/resource/Hill_Street_Blues, http://dbpedia.org/resource/Homicide:_Life_on_the_Street, http://dbpedia.org/resource/Las_Vegas_(TV_series), http://dbpedia.org/resource/Late_Night_with_Conan_O'Brien, http://dbpedia.org/resource/Late_Night_with_David_Letterman, http://dbpedia.org/resource/Later_(talk_show), http://dbpedia.org/resource/Matlock_(TV_series), http://dbpedia.org/resource/NBA_on_NBC, http://dbpedia.org/resource/NBC_Sunday_Night_Football, http://dbpedia.org/resource/NFL_on_NBC, http://dbpedia.org/resource/NHL_on_NBC, http://dbpedia.org/resource/NewsRadio, http://dbpedia.org/resource/Olympics_on_NBC, http://dbpedia.org/resource/Quantum_Leap, http://dbpedia.org/resource/Rock_Center_with_Brian_Williams, http://dbpedia.org/resource/Second_City_Television, http://dbpedia.org/resource/Sunshine_(U.S._TV_series), http://dbpedia.org/resource/Tattingers, http://dbpedia.org/resource/The_Rockford_Files, http://dbpedia.org/resource/Thursday_Night_Football, http://dbpedia.org/resource/Up_All_Night_(TV_series), http://dbpedia.org/resource/A_Man_and_His_Music_+_Ella_+_Jobim, http://dbpedia.org/resource/Baa_Baa_Black_Sheep_(TV_series), http://dbpedia.org/resource/Frank_Sinatra:_A_Man_and_His_Music, http://dbpedia.org/resource/Friends_(season_1), http://dbpedia.org/resource/Friends_(season_10), http://dbpedia.org/resource/Friends_(season_2), http://dbpedia.org/resource/Friends_(season_3), http://dbpedia.org/resource/Friends_(season_4), http://dbpedia.org/resource/Friends_(season_5), http://dbpedia.org/resource/Friends_(season_6), http://dbpedia.org/resource/Friends_(season_7), http://dbpedia.org/resource/Friends_(season_8), http://dbpedia.org/resource/Friends_(season_9), http://dbpedia.org/resource/Get_Smart, http://dbpedia.org/resource/Grimm_(season_4), http://dbpedia.org/resource/Joey_(TV_series), http://dbpedia.org/resource/Mad_About_You, http://dbpedia.org/resource/Wichita_Town, http://dbpedia.org/resource/Wings_(1990_TV_series), http://dbpedia.org/resource/Cartoon_All-Stars_to_the_Rescue, http://dbpedia.org/resource/Just_Shoot_Me!, http://dbpedia.org/resource/Law_&_Order:_Criminal_Intent, http://dbpedia.org/resource/The_New_Adventures_of_Huckleberry_Finn, http://dbpedia.org/resource/A_Muppets_Christmas:_Letters_to_Santa, http://dbpedia.org/resource/America's_Got_Talent, http://dbpedia.org/resource/Captain_N:_The_Game_Master, http://dbpedia.org/resource/Chuck_(TV_series), http://dbpedia.org/resource/ER_(season_3), http://dbpedia.org/resource/Emergency!, http://dbpedia.org/resource/Fired_Up_(TV_series), http://dbpedia.org/resource/Hunter_(U.S._TV_series), http://dbpedia.org/resource/It's_a_Very_Merry_Muppet_Christmas_Movie, http://dbpedia.org/resource/Lucas_Tanner, http://dbpedia.org/resource/Macy's_Thanksgiving_Day_Parade, http://dbpedia.org/resource/Man_from_Atlantis, http://dbpedia.org/resource/Mickey's_60th_Birthday, http://dbpedia.org/resource/Police_Woman_(TV_series), http://dbpedia.org/resource/Sisters_(TV_series), http://dbpedia.org/resource/Stark_Raving_Mad_(TV_series), http://dbpedia.org/resource/The_Bold_Ones:_The_New_Doctors, http://dbpedia.org/resource/The_Highwayman_(TV_series), http://dbpedia.org/resource/Third_Watch, http://dbpedia.org/resource/Westwind_(TV_series), http://dbpedia.org/resource/1st_Look, http://dbpedia.org/resource/3-2-1_Penguins!, http://dbpedia.org/resource/A_Man_and_His_Music_–_Part_II, http://dbpedia.org/resource/Adam-12, http://dbpedia.org/resource/Buffalo_Bill_(TV_series), http://dbpedia.org/resource/California_Dreams, http://dbpedia.org/resource/Card_Sharks, http://dbpedia.org/resource/City_Guys, http://dbpedia.org/resource/Concentration_(game_show), http://dbpedia.org/resource/Deal_or_No_Deal_(U.S._game_show), http://dbpedia.org/resource/Early_Today, http://dbpedia.org/resource/Endurance_(TV_series), http://dbpedia.org/resource/Got_Talent, http://dbpedia.org/resource/H.R._Pufnstuf, http://dbpedia.org/resource/Hang_Time_(TV_series), http://dbpedia.org/resource/High_Rollers, http://dbpedia.org/resource/Highway_to_Heaven, http://dbpedia.org/resource/Huntley-Brinkley_Report, http://dbpedia.org/resource/In_the_Heat_of_the_Night_(TV_series), http://dbpedia.org/resource/Knight_Rider_(2008_TV_series), http://dbpedia.org/resource/Leeza, http://dbpedia.org/resource/Minute_to_Win_It, http://dbpedia.org/resource/NBC_News_at_Sunrise, http://dbpedia.org/resource/NBC_Nightside, http://dbpedia.org/resource/Nero_Wolfe_(1981_TV_series), http://dbpedia.org/resource/Notre_Dame_Football_on_NBC, http://dbpedia.org/resource/One_World_(TV_series), http://dbpedia.org/resource/One_to_Grow_On, http://dbpedia.org/resource/Password_Plus_and_Super_Password, http://dbpedia.org/resource/Santa_Barbara_(TV_series), http://dbpedia.org/resource/Saved_by_the_Bell:_The_New_Class, http://dbpedia.org/resource/Scout's_Safari, http://dbpedia.org/resource/Somerset_(TV_series), http://dbpedia.org/resource/Sunset_Beach_(TV_series), http://dbpedia.org/resource/The_Apprentice_(U.S._TV_series), http://dbpedia.org/resource/The_Banana_Splits, http://dbpedia.org/resource/The_Biggest_Loser_(U.S._TV_series), http://dbpedia.org/resource/The_Bugaloos, http://dbpedia.org/resource/The_Flip_Wilson_Show, http://dbpedia.org/resource/The_Hogan_Family, http://dbpedia.org/resource/The_Krofft_Superstar_Hour, http://dbpedia.org/resource/The_Marriage_Ref_(U.S._TV_series), http://dbpedia.org/resource/The_Smurfs_(TV_series), http://dbpedia.org/resource/The_Waverly_Wonders, http://dbpedia.org/resource/Time_Warp_Trio, http://dbpedia.org/resource/Tomorrow_(TV_series), http://dbpedia.org/resource/Tonight_Starring_Steve_Allen, http://dbpedia.org/resource/Weekend_Today, http://dbpedia.org/resource/Fabulous_Funnies, http://dbpedia.org/resource/God,_the_Devil_and_Bob, http://dbpedia.org/resource/Star_Trek:_The_Original_Series_(season_3), http://dbpedia.org/resource/The_Kid_Super_Power_Hour_with_Shazam!, http://dbpedia.org/resource/The_New_Adventures_of_Flash_Gordon, http://dbpedia.org/resource/The_Secret_Lives_of_Waldo_Kitty, http://dbpedia.org/resource/A_Tour_of_the_White_House_with_Mrs._John_F._Kennedy, http://dbpedia.org/resource/ProStars, http://dbpedia.org/resource/Boomtown_(2002_TV_series), http://dbpedia.org/resource/Frasier_(season_6), http://dbpedia.org/resource/Knight_Rider_2000, http://dbpedia.org/resource/Team_Knight_Rider, http://dbpedia.org/resource/Evil_Roy_Slade, http://dbpedia.org/resource/Jackpot_Bowling, http://dbpedia.org/resource/San_Francisco_International_Airport_(TV_series), http://dbpedia.org/resource/The_Paul_Winchell_Show, http://dbpedia.org/resource/The_Pink_Panther_Show, http://dbpedia.org/resource/An_Evening_with_Fred_Astaire, http://dbpedia.org/resource/Choices_of_the_Heart, http://dbpedia.org/resource/Down_You_Go, http://dbpedia.org/resource/Father_Dowling_Mysteries, http://dbpedia.org/resource/Fear_Itself_(TV_series), http://dbpedia.org/resource/Godzilla_(animated_series), http://dbpedia.org/resource/Hec_Ramsey, http://dbpedia.org/resource/Monster_Squad, http://dbpedia.org/resource/Poor_Little_Rich_Girl:_The_Barbara_Hutton_Story, http://dbpedia.org/resource/Prime_Suspect_(U.S._TV_series), http://dbpedia.org/resource/The_Psychiatrist_(TV_series), http://dbpedia.org/resource/Law_&_Order:_LA, http://dbpedia.org/resource/Law_&_Order:_Trial_by_Jury, http://dbpedia.org/resource/Witness_to_the_Mob, http://dbpedia.org/resource/Camp_Candy, http://dbpedia.org/resource/Saturday_Night_Live_(season_13), http://dbpedia.org/resource/Witch's_Night_Out, http://dbpedia.org/resource/Columbo_(season_3), http://dbpedia.org/resource/Committed_(2005_TV_series), http://dbpedia.org/resource/Legends_of_the_Superheroes, http://dbpedia.org/resource/List_of_The_Alfred_Hitchcock_Hour_episodes, http://dbpedia.org/resource/Saturday_Night_Live_(season_20), http://dbpedia.org/resource/Stars_Earn_Stripes, http://dbpedia.org/resource/The_Court_of_Last_Resort, http://dbpedia.org/resource/$25_Million_Dollar_Hoax, http://dbpedia.org/resource/Cold_Feet_(U.S._TV_series), http://dbpedia.org/resource/Coupling_(U.S._TV_series), http://dbpedia.org/resource/I'm_a_Celebrity...Get_Me_Out_of_Here!_(U.S._TV_series), http://dbpedia.org/resource/Men_Behaving_Badly_(U.S._TV_series), http://dbpedia.org/resource/Return_to_the_Planet_of_the_Apes, http://dbpedia.org/resource/America's_Toughest_Jobs, http://dbpedia.org/resource/Amy_Prentiss, http://dbpedia.org/resource/Are_You_There,_Chelsea%3F, http://dbpedia.org/resource/Baby_Boom_(U.S._TV_series), http://dbpedia.org/resource/Battery_Park_(TV_series), http://dbpedia.org/resource/Blank_Check_(game_show), http://dbpedia.org/resource/Buford_and_the_Galloping_Ghost, http://dbpedia.org/resource/C._Everett_Koop,_M.D., http://dbpedia.org/resource/Campus_Hoopla, http://dbpedia.org/resource/Chicago_Story]... [INFO] Predicted Answer: [http://dbpedia.org/resource/Cheers_(season_1), http://dbpedia.org/resource/Cheers_(season_2), http://dbpedia.org/resource/Cheers_(season_3), http://dbpedia.org/resource/Cheers_(season_4), http://dbpedia.org/resource/Cheers_(season_5), http://dbpedia.org/resource/Cheers_(season_6), http://dbpedia.org/resource/Cheers_(season_7), http://dbpedia.org/resource/Frasier_(season_9), http://dbpedia.org/resource/Friends, http://dbpedia.org/resource/It's_Punky_Brewster, http://dbpedia.org/resource/Scrubs_(TV_series), http://dbpedia.org/resource/St._Elsewhere, http://dbpedia.org/resource/The_Cosby_Show, http://dbpedia.org/resource/Alvin_and_the_Chipmunks_(1983_TV_series), http://dbpedia.org/resource/Dolly_Parton's_Coat_of_Many_Colors, http://dbpedia.org/resource/E-Ring, http://dbpedia.org/resource/Father_of_the_Pride, http://dbpedia.org/resource/High_School_U.S.A., http://dbpedia.org/resource/Information_Please, http://dbpedia.org/resource/My_Two_Dads, http://dbpedia.org/resource/Queen_for_a_Day, http://dbpedia.org/resource/Remington_Steele, http://dbpedia.org/resource/Rowan_&_Martin's_Laugh-In, http://dbpedia.org/resource/Saturday_Night_Live_(season_1), http://dbpedia.org/resource/The_Man_from_U.N.C.L.E., http://dbpedia.org/resource/The_Single_Guy, http://dbpedia.org/resource/The_Steve_Allen_Show, http://dbpedia.org/resource/The_Virginian_(TV_series), http://dbpedia.org/resource/Victoria_Beckham:_Coming_to_America, http://dbpedia.org/resource/Walt_Disney_anthology_television_series, http://dbpedia.org/resource/Banacek, http://dbpedia.org/resource/Buck_Rogers_in_the_25th_Century_(TV_series), http://dbpedia.org/resource/Guilty_or_Innocent:_The_Sam_Sheppard_Murder_Case, http://dbpedia.org/resource/Howdy_Doody, http://dbpedia.org/resource/I_Spy_(1965_TV_series), http://dbpedia.org/resource/Magnavox_Presents_Frank_Sinatra, http://dbpedia.org/resource/Suspicion_(TV_series), http://dbpedia.org/resource/The_Funny_Side, http://dbpedia.org/resource/The_Gary_Coleman_Show, http://dbpedia.org/resource/The_Ghost_&_Mrs._Muir_(TV_series), http://dbpedia.org/resource/The_High_Chaparral, http://dbpedia.org/resource/The_Kaiser_Aluminum_Hour, http://dbpedia.org/resource/The_New_Show, http://dbpedia.org/resource/The_Questor_Tapes, http://dbpedia.org/resource/To_Tell_the_Truth, http://dbpedia.org/resource/Exiled:_A_Law_&_Order_Movie, http://dbpedia.org/resource/Fame_(1982_TV_series), http://dbpedia.org/resource/Late_Night_with_Seth_Meyers, http://dbpedia.org/resource/Law_&_Order:_Special_Victims_Unit_(season_7), http://dbpedia.org/resource/On_the_Edge_of_Innocence, http://dbpedia.org/resource/Spider-Man_and_His_Amazing_Friends, http://dbpedia.org/resource/The_Jay_Leno_Show, http://dbpedia.org/resource/L.A._Takedown, http://dbpedia.org/resource/Rudolph_the_Red-Nosed_Reindeer_(TV_special), http://dbpedia.org/resource/The_Rocky_and_Bullwinkle_Show, http://dbpedia.org/resource/Quincy,_M.E., http://dbpedia.org/resource/The_First_Olympics:_Athens_1896, http://dbpedia.org/resource/The_Golden_Girls_(season_4), http://dbpedia.org/resource/A_Man_Called_Sloane, http://dbpedia.org/resource/Please_Don't_Eat_the_Daisies_(TV_series), http://dbpedia.org/resource/The_Parent_Trap_III, http://dbpedia.org/resource/The_Parent_Trap_IV:_Hawaiian_Honeymoon, http://dbpedia.org/resource/AFL_on_NBC, http://dbpedia.org/resource/The_Jensen_Project, http://dbpedia.org/resource/Three_Sisters_(TV_series), http://dbpedia.org/resource/50_Grand_Slam, http://dbpedia.org/resource/Against_the_Grain_(TV_series), http://dbpedia.org/resource/Eerie,_Indiana, http://dbpedia.org/resource/The_Dinah_Shore_Chevy_Show, http://dbpedia.org/resource/Halloween_with_the_New_Addams_Family, http://dbpedia.org/resource/It's_Worth_What%3F, http://dbpedia.org/resource/Lipshitz_Saves_the_World, http://dbpedia.org/resource/Little_Mo_(1978_film), http://dbpedia.org/resource/Saturday_Night_Live_(season_40), http://dbpedia.org/resource/The_Addams_Family_(1973_animated_series), http://dbpedia.org/resource/The_Bob_Cummings_Show, http://dbpedia.org/resource/The_Bold_Ones:_The_Protectors, http://dbpedia.org/resource/The_Cosby_Mysteries, http://dbpedia.org/resource/Whoopi, http://dbpedia.org/resource/Your_First_Impression, http://dbpedia.org/resource/Fire_on_the_Mountain_(1981_film), http://dbpedia.org/resource/Five_Fingers_(TV_series), http://dbpedia.org/resource/Liza_with_a_Z, http://dbpedia.org/resource/Rainbow_(1978_film), http://dbpedia.org/resource/Return_to_Mayberry, http://dbpedia.org/resource/The_Bold_Ones, http://dbpedia.org/resource/1_vs._100_(U.S._game_show), http://dbpedia.org/resource/Average_Joe_(TV_series), http://dbpedia.org/resource/Chain_Reaction_(game_show), http://dbpedia.org/resource/Jackpot_(game_show), http://dbpedia.org/resource/Time_Machine_(game_show), http://dbpedia.org/resource/Treasure_Hunt_(U.S._game_show), http://dbpedia.org/resource/Win,_Lose_or_Draw, http://dbpedia.org/resource/Banyon, http://dbpedia.org/resource/Café_Americain, http://dbpedia.org/resource/Camel_News_Caravan, http://dbpedia.org/resource/Celebrity_Family_Feud, http://dbpedia.org/resource/City_of_Angels_(1976_TV_series), http://dbpedia.org/resource/Dawn:_Portrait_of_a_Teenage_Runaway, http://dbpedia.org/resource/Fraggle_Rock:_The_Animated_Series, http://dbpedia.org/resource/I_Dream_of_Jeannie..._Fifteen_Years_Later, http://dbpedia.org/resource/Return_to_Peyton_Place_(TV_series), http://dbpedia.org/resource/Runaround_(game_show), http://dbpedia.org/resource/Surface_(TV_series), http://dbpedia.org/resource/Bonino_(TV_series), http://dbpedia.org/resource/Bret_Maverick, http://dbpedia.org/resource/Cursed_(TV_series), http://dbpedia.org/resource/LateLine, http://dbpedia.org/resource/Baggy_Pants_and_the_Nitwits, http://dbpedia.org/resource/Quantum_Leap_(season_2), http://dbpedia.org/resource/Quarterlife, http://dbpedia.org/resource/Something_for_a_Lonely_Man, http://dbpedia.org/resource/Then_Came_Bronson, http://dbpedia.org/resource/Wild_Kingdom, http://dbpedia.org/resource/Free_Agents_(U.S._TV_series), http://dbpedia.org/resource/Hitler's_SS:_Portrait_in_Evil, http://dbpedia.org/resource/Murder_in_Texas, http://dbpedia.org/resource/Revenge_of_the_Stepford_Wives, http://dbpedia.org/resource/The_Bob_Newhart_Show_(1961_TV_series), http://dbpedia.org/resource/Redigo_(TV_series), http://dbpedia.org/resource/Rockin'_Through_the_Decades, http://dbpedia.org/resource/Saturday_Night_Live_(season_39), http://dbpedia.org/resource/Ding_Dong_School, http://dbpedia.org/resource/M.Y.O.B._(TV_series), http://dbpedia.org/resource/Boone_(TV_series), http://dbpedia.org/resource/Kingpin_(TV_series), http://dbpedia.org/resource/Emergency_+4, http://dbpedia.org/resource/The_Snoop_Sisters, http://dbpedia.org/resource/30_Rock_(season_7), http://dbpedia.org/resource/33⅓_Revolutions_per_Monkee, http://dbpedia.org/resource/Battlestars_(game_show), http://dbpedia.org/resource/Joe_Forrester_(TV_series), http://dbpedia.org/resource/Law_&_Order_(season_9), http://dbpedia.org/resource/Little_Richard_(film), http://dbpedia.org/resource/Love,_Sidney, http://dbpedia.org/resource/Northwest_Passage_(TV_series), http://dbpedia.org/resource/The_More_You_Know, http://dbpedia.org/resource/Movin'_with_Nancy, http://dbpedia.org/resource/Scrubs_(season_4), http://dbpedia.org/resource/The_Spike_Jones_Show_(series), http://dbpedia.org/resource/Danger_Island_(film), http://dbpedia.org/resource/Dragnet_(1951_TV_series), http://dbpedia.org/resource/Fashion_Star_(season_1), http://dbpedia.org/resource/Living_Proof:_The_Hank_Williams,_Jr._Story, http://dbpedia.org/resource/Thank_God_You're_Here_(U.S._TV_series), http://dbpedia.org/resource/The_Real_Wedding_Crashers, http://dbpedia.org/resource/Fatal_Vision_(miniseries), http://dbpedia.org/resource/Call_My_Bluff_(U.S._game_show), http://dbpedia.org/resource/Trinity_(U.S._TV_series), http://dbpedia.org/resource/The_Biggest_Loser:_Families, http://dbpedia.org/resource/Saturday_Night_Live_(season_34), http://dbpedia.org/resource/The_Restaurant_(U.S._TV_series), http://dbpedia.org/resource/World's_Most_Amazing_Videos, http://dbpedia.org/resource/Your_Mother_Wears_Combat_Boots, http://dbpedia.org/resource/Parks_and_Recreation_(season_2), http://dbpedia.org/resource/Saturday_Night_Live_(season_10), http://dbpedia.org/resource/Saturday_Night_Live_(season_32), http://dbpedia.org/resource/Reach_for_the_Stars_(game_show), http://dbpedia.org/resource/The_Rockford_Files_(season_2), http://dbpedia.org/resource/Jigsaw_John_(TV_series), http://dbpedia.org/resource/Someone_Like_Me_(TV_series), http://dbpedia.org/resource/Winnetka_Road, http://dbpedia.org/resource/Combat_Academy, http://dbpedia.org/resource/Kate's_Secret, http://dbpedia.org/resource/Class_Cruise, http://dbpedia.org/resource/Science_of_Love, http://dbpedia.org/resource/Silent_Witness_(1985_film), http://dbpedia.org/resource/Moment_of_Truth:_Why_My_Daughter%3F, http://dbpedia.org/resource/Strong_(TV_series), http://dbpedia.org/resource/My_Name_Is_Earl_(season_4), http://dbpedia.org/resource/Chicago_Fire_(season_1), http://dbpedia.org/resource/Chicago_Fire_(season_2), http://dbpedia.org/resource/Chicago_Fire_(season_3), http://dbpedia.org/resource/Chicago_Fire_(season_4), http://dbpedia.org/resource/Chicago_P.D._(season_1), http://dbpedia.org/resource/Chicago_P.D._(season_2), http://dbpedia.org/resource/Chicago_P.D._(season_3), http://dbpedia.org/resource/Make_Your_Own_Kind_of_Music_(TV_series), http://dbpedia.org/resource/Knight_Rider_(season_4), http://dbpedia.org/resource/Fashion_Star_(season_2), http://dbpedia.org/resource/The_Six_of_Us, http://dbpedia.org/resource/The_Ted_Steele_Show, http://dbpedia.org/resource/Friday_Night_Lights_(season_2), http://dbpedia.org/resource/Take_a_Chance_(U.S._game_show), http://dbpedia.org/resource/Parenthood_(season_1), http://dbpedia.org/resource/Parenthood_(season_3), http://dbpedia.org/resource/Parenthood_(season_4), http://dbpedia.org/resource/Parenthood_(season_5), http://dbpedia.org/resource/Parenthood_(season_6), http://dbpedia.org/resource/Where_the_Girls_Are_(TV_program), http://dbpedia.org/resource/The_A-Team_(season_2), http://dbpedia.org/resource/The_A-Team_(season_3), http://dbpedia.org/resource/The_A-Team_(season_5), http://dbpedia.org/resource/Adam-12_(season_2), http://dbpedia.org/resource/Dragnet_(1967_series)_(season_1), http://dbpedia.org/resource/Dragnet_(1967_series)_(season_3), http://dbpedia.org/resource/Dragnet_(1967_series)_(season_4), http://dbpedia.org/resource/Ironside_(season_1), http://dbpedia.org/resource/Ironside_(season_2), http://dbpedia.org/resource/Ironside_(season_4), http://dbpedia.org/resource/Ironside_(season_5), http://dbpedia.org/resource/Ironside_(season_6), http://dbpedia.org/resource/Ironside_(season_8), http://dbpedia.org/resource/Quincy,_M.E._(season_1), http://dbpedia.org/resource/Quincy,_M.E._(season_2), http://dbpedia.org/resource/Quincy,_M.E._(season_3), http://dbpedia.org/resource/Quincy,_M.E._(season_4), http://dbpedia.org/resource/Quincy,_M.E._(season_5), http://dbpedia.org/resource/Quincy,_M.E._(season_6), http://dbpedia.org/resource/Quincy,_M.E._(season_7), http://dbpedia.org/resource/Quincy,_M.E._(season_8), http://dbpedia.org/resource/3rd_Rock_from_the_Sun, http://dbpedia.org/resource/Alfred_Hitchcock_Presents, http://dbpedia.org/resource/Alfred_Hitchcock_Presents_(1985_TV_series), http://dbpedia.org/resource/Jeopardy!, http://dbpedia.org/resource/Star_Trek:_The_Original_Series, http://dbpedia.org/resource/30_Rock_(season_3), http://dbpedia.org/resource/Crime_Story_(TV_series), http://dbpedia.org/resource/Frasier_(season_10), http://dbpedia.org/resource/Knight_Rider_(1982_TV_series), http://dbpedia.org/resource/Studio_60_on_the_Sunset_Strip, http://dbpedia.org/resource/The_Tonight_Show_Starring_Jimmy_Fallon, http://dbpedia.org/resource/Alice_in_Wonderland_(1999_film), http://dbpedia.org/resource/Caesar's_Hour, http://dbpedia.org/resource/College_Basketball_on_NBC, http://dbpedia.org/resource/Eischied, http://dbpedia.org/resource/Major_League_Baseball:_An_Inside_Look, http://dbpedia.org/resource/Thoroughbred_Racing_on_NBC, http://dbpedia.org/resource/WWE_Tribute_to_the_Troops, http://dbpedia.org/resource/American_Gladiators_(2008_TV_series), http://dbpedia.org/resource/Mann_&_Machine, http://dbpedia.org/resource/National_Heads-Up_Poker_Championship, http://dbpedia.org/resource/Ringo_(1978_film), http://dbpedia.org/resource/Space_Sentinels, http://dbpedia.org/resource/The_Golden_Girls_(season_1), http://dbpedia.org/resource/The_Rogues_(TV_series), http://dbpedia.org/resource/The_Yellow_Rose, http://dbpedia.org/resource/Suddenly_Susan, http://dbpedia.org/resource/A_Flintstone_Christmas, http://dbpedia.org/resource/Fred_and_Barney_Meet_The_Thing, http://dbpedia.org/resource/Fred_and_Barney_Meet_the_Shmoo, http://dbpedia.org/resource/Galaxy_Goof-Ups, http://dbpedia.org/resource/Serving_in_Silence:_The_Margarethe_Cammermeyer_Story, http://dbpedia.org/resource/The_Flintstone_Funnies, http://dbpedia.org/resource/The_New_Fred_and_Barney_Show, http://dbpedia.org/resource/You,_Me_and_the_Apocalypse, http://dbpedia.org/resource/Nearly_Departed, http://dbpedia.org/resource/Parks_and_Recreation_(season_7), http://dbpedia.org/resource/Sealab_2020, http://dbpedia.org/resource/The_Flintstones'_New_Neighbors, http://dbpedia.org/resource/The_Flintstones:_Fred's_Final_Fling, http://dbpedia.org/resource/The_Flintstones:_Jogging_Fever, http://dbpedia.org/resource/The_Flintstones:_Little_Big_League, http://dbpedia.org/resource/The_Flintstones:_Wind-Up_Wilma, http://dbpedia.org/resource/The_Flintstones_Meet_Rockula_and_Frankenstone, http://dbpedia.org/resource/Identity_(game_show), http://dbpedia.org/resource/On_Wings_of_Eagles_(miniseries), http://dbpedia.org/resource/Outlaws_(1960_TV_series), http://dbpedia.org/resource/Who_Said_That%3F, http://dbpedia.org/resource/The_Biggest_Loser:_No_Excuses, http://dbpedia.org/resource/The_Burning_Bed, http://dbpedia.org/resource/Awake_(TV_series), http://dbpedia.org/resource/The_Biggest_Loser_(season_3), http://dbpedia.org/resource/The_Book_of_Daniel_(TV_series), http://dbpedia.org/resource/Run,_Joe,_Run, http://dbpedia.org/resource/Sigmund_and_the_Sea_Monsters, http://dbpedia.org/resource/The_Bold_Ones:_The_Lawyers, http://dbpedia.org/resource/Alexander:_The_Other_Side_of_Dawn, http://dbpedia.org/resource/Law_&_Order:_Special_Victims_Unit_(season_10), http://dbpedia.org/resource/The_Gabby_Hayes_Show, http://dbpedia.org/resource/Deadline_(American_TV_series), http://dbpedia.org/resource/In_Love_and_War_(1987_film), http://dbpedia.org/resource/Saturday_Night_Live_(season_2), http://dbpedia.org/resource/She_Spies, http://dbpedia.org/resource/Adam_(1983_film), http://dbpedia.org/resource/The_Ruff_and_Reddy_Show, http://dbpedia.org/resource/McKeever_and_the_Colonel, http://dbpedia.org/resource/Sierra_(TV_series), http://dbpedia.org/resource/Columbo_(season_4), http://dbpedia.org/resource/Columbo_(season_5), http://dbpedia.org/resource/Jewels_(miniseries), http://dbpedia.org/resource/Kristin_(TV_series), http://dbpedia.org/resource/Quantum_Leap_(season_5), http://dbpedia.org/resource/The_Fresh_Prince_of_Bel-Air_(season_3), http://dbpedia.org/resource/The_Secret_Life_of_Kathy_McCormick, http://dbpedia.org/resource/Casper's_First_Christmas, http://dbpedia.org/resource/Doug_Henning's_World_of_Magic, http://dbpedia.org/resource/Most_Outrageous_Moments, http://dbpedia.org/resource/Betty_White's_Off_Their_Rockers, http://dbpedia.org/resource/Half_Nelson_(TV_series), http://dbpedia.org/resource/Jack_Frost_(TV_special), http://dbpedia.org/resource/Lookwell, http://dbpedia.org/resource/Saturday_Night_Live_(season_18), http://dbpedia.org/resource/Homeland_Security_(film), http://dbpedia.org/resource/Skunked_TV, http://dbpedia.org/resource/The_Man_and_the_Challenge, http://dbpedia.org/resource/Sweet_Justice, http://dbpedia.org/resource/A_Howling_in_the_Woods, http://dbpedia.org/resource/I_Still_Dream_of_Jeannie, http://dbpedia.org/resource/McDuff,_the_Talking_Dog, http://dbpedia.org/resource/In_the_Line_of_Duty:_The_F.B.I._Murders, http://dbpedia.org/resource/Legmen, http://dbpedia.org/resource/Revelations_(miniseries), http://dbpedia.org/resource/The_Innocent_(1994_film), http://dbpedia.org/resource/A_Chipmunk_Reunion, http://dbpedia.org/resource/Opposites_Attract_(1990_film), http://dbpedia.org/resource/Those_She_Left_Behind, http://dbpedia.org/resource/Popeye_Doyle_(film), http://dbpedia.org/resource/A_Child's_Cry_for_Help, http://dbpedia.org/resource/Frasier_(season_4), http://dbpedia.org/resource/Miracle_Child_(1993_film), http://dbpedia.org/resource/Snowbeast, http://dbpedia.org/resource/Grimm_(season_1), http://dbpedia.org/resource/Dragnet_(1967_series)_(season_2), http://dbpedia.org/resource/Come_to_Papa_(TV_series), http://dbpedia.org/resource/Lazer_Tag_Academy, http://dbpedia.org/resource/My_Son_Jeep, http://dbpedia.org/resource/Community_(season_2), http://dbpedia.org/resource/The_Jane_Whitney_Show, http://dbpedia.org/resource/The_Kids_From_C.A.P.E.R., http://dbpedia.org/resource/The_World's_Most_Dangerous_Magic, http://dbpedia.org/resource/My_Name_Is_Earl_(season_2), http://dbpedia.org/resource/The_Office_(U.S._season_2), http://dbpedia.org/resource/Amnesia_(game_show), http://dbpedia.org/resource/3rd_Rock_from_the_Sun_(season_2), http://dbpedia.org/resource/3rd_Rock_from_the_Sun_(season_6), http://dbpedia.org/resource/List_of_The_Tonight_Show_with_Jay_Leno_episodes_(2000–09), http://dbpedia.org/resource/The_Jack_Benny_Program_(season_15), http://dbpedia.org/resource/List_of_Late_Night_with_Jimmy_Fallon_episodes_(2010), http://dbpedia.org/resource/Harry's_Law_(season_1), http://dbpedia.org/resource/The_Fresh_Prince_of_Bel-Air_(season_5), http://dbpedia.org/resource/List_of_The_Tonight_Show_Starring_Johnny_Carson_episodes_(1974), http://dbpedia.org/resource/List_of_The_Tonight_Show_Starring_Johnny_Carson_episodes_(1975), http://dbpedia.org/resource/List_of_The_Tonight_Show_Starring_Johnny_Carson_episodes_(1976), http://dbpedia.org/resource/List_of_The_Tonight_Show_Starring_Johnny_Carson_episodes_(1978), http://dbpedia.org/resource/List_of_The_Tonight_Show_Starring_Johnny_Carson_episodes_(1979), http://dbpedia.org/resource/List_of_The_Tonight_Show_Starring_Johnny_Carson_episodes_(1980), http://dbpedia.org/resource/List_of_The_Tonight_Show_Starring_Johnny_Carson_episodes_(1981), http://dbpedia.org/resource/List_of_The_Tonight_Show_Starring_Johnny_Carson_episodes_(1982), http://dbpedia.org/resource/List_of_The_Tonight_Show_Starring_Johnny_Carson_episodes_(1983), http://dbpedia.org/resource/List_of_The_Tonight_Show_Starring_Johnny_Carson_episodes_(1984), http://dbpedia.org/resource/List_of_The_Tonight_Show_Starring_Johnny_Carson_episodes_(1985), http://dbpedia.org/resource/List_of_The_Tonight_Show_Starring_Johnny_Carson_episodes_(1986), http://dbpedia.org/resource/List_of_The_Tonight_Show_Starring_Johnny_Carson_episodes_(1987), http://dbpedia.org/resource/List_of_The_Tonight_Show_Starring_Johnny_Carson_episodes_(1988), http://dbpedia.org/resource/List_of_The_Tonight_Show_Starring_Johnny_Carson_episodes_(1989), http://dbpedia.org/resource/List_of_The_Tonight_Show_Starring_Johnny_Carson_episodes_(1990), http://dbpedia.org/resource/List_of_The_Tonight_Show_Starring_Johnny_Carson_episodes_(1991), http://dbpedia.org/resource/List_of_The_Tonight_Show_Starring_Johnny_Carson_episodes_(1992), http://dbpedia.org/resource/List_of_Late_Night_with_Jimmy_Fallon_episodes_(2009), http://dbpedia.org/resource/List_of_Late_Night_with_Jimmy_Fallon_episodes_(2011), http://dbpedia.org/resource/List_of_The_Tonight_Show_Starring_Jimmy_Fallon_episodes_(2014), http://dbpedia.org/resource/List_of_The_Tonight_Show_Starring_Jimmy_Fallon_episodes_(2016), http://dbpedia.org/resource/Dateline_NBC, http://dbpedia.org/resource/I_Dream_of_Jeannie, http://dbpedia.org/resource/Kings_(U.S._TV_series), http://dbpedia.org/resource/Medium_(TV_series), http://dbpedia.org/resource/Miami_Vice, http://dbpedia.org/resource/SeaQuest_DSV, http://dbpedia.org/resource/The_Celebrity_Apprentice, http://dbpedia.org/resource/The_Incredible_Hulk_(1978_TV_series), http://dbpedia.org/resource/The_Tonight_Show, http://dbpedia.org/resource/The_Tonight_Show_with_Jay_Leno, http://dbpedia.org/resource/Wheel_of_Fortune_(U.S._game_show), http://dbpedia.org/resource/Bionic_Woman_(2007_TV_series), http://dbpedia.org/resource/Homicide:_Life_on_the_Street_(season_6), http://dbpedia.org/resource/L.A._Law, http://dbpedia.org/resource/Law_&_Order:_Special_Victims_Unit, http://dbpedia.org/resource/Law_&_Order:_Special_Victims_Unit_(season_8), http://dbpedia.org/resource/Love,_Lies_and_Murder, http://dbpedia.org/resource/Misfits_of_Science, http://dbpedia.org/resource/Taxi_(TV_series), http://dbpedia.org/resource/The_Bionic_Woman, http://dbpedia.org/resource/The_Year_Without_a_Santa_Claus_(2006_film), http://dbpedia.org/resource/Earth_2_(TV_series), http://dbpedia.org/resource/Riverboat_(TV_series), http://dbpedia.org/resource/Underdog_(TV_series), http://dbpedia.org/resource/The_Apprentice_(TV_series), http://dbpedia.org/resource/Windfall_(TV_series), http://dbpedia.org/resource/Ellery_Queen_(TV_series), http://dbpedia.org/resource/Seinfeld_(season_1), http://dbpedia.org/resource/Seinfeld_(season_2), http://dbpedia.org/resource/Seinfeld_(season_3), http://dbpedia.org/resource/Seinfeld_(season_4), http://dbpedia.org/resource/Seinfeld_(season_5), http://dbpedia.org/resource/Seinfeld_(season_6), http://dbpedia.org/resource/Seinfeld_(season_7), http://dbpedia.org/resource/Seinfeld_(season_8), http://dbpedia.org/resource/Seinfeld_(season_9), http://dbpedia.org/resource/The_John_Larroquette_Show, http://dbpedia.org/resource/The_Michael_Richards_Show, http://dbpedia.org/resource/Watching_Ellie, http://dbpedia.org/resource/Backstairs_at_the_White_House, http://dbpedia.org/resource/Love_in_the_Wild, http://dbpedia.org/resource/The_Contender_(TV_series), http://dbpedia.org/resource/Choose_Up_Sides, http://dbpedia.org/resource/Dough_Re_Mi, http://dbpedia.org/resource/Make_the_Connection, http://dbpedia.org/resource/Sale_of_the_Century_(U.S._game_show), http://dbpedia.org/resource/The_Two_Mrs._Grenvilles, http://dbpedia.org/resource/3rd_Rock_from_the_Sun_(season_1), http://dbpedia.org/resource/3rd_Rock_from_the_Sun_(season_5), http://dbpedia.org/resource/America's_Most_Talented_Kid, http://dbpedia.org/resource/From_Here_to_Eternity_(miniseries), http://dbpedia.org/resource/I'll_Fly_Away_(TV_series), http://dbpedia.org/resource/I_Married_Joan, http://dbpedia.org/resource/Something_So_Right_(TV_series), http://dbpedia.org/resource/Bare_Essence, http://dbpedia.org/resource/C.P.O._Sharkey, http://dbpedia.org/resource/Freaks_and_Geeks, http://dbpedia.org/resource/Harper_Valley_PTA_(TV_series), http://dbpedia.org/resource/Heist_(TV_series), http://dbpedia.org/resource/Kath_&_Kim_(U.S._TV_series), http://dbpedia.org/resource/Lovers_and_Friends, http://dbpedia.org/resource/My_Hero_(U.S._TV_series), http://dbpedia.org/resource/Needles_and_Pins_(TV_series), http://dbpedia.org/resource/Rescue_from_Gilligan's_Island, http://dbpedia.org/resource/Saturday_Night_Live_(season_21), http://dbpedia.org/resource/Saturday_Night_Live_(season_36), http://dbpedia.org/resource/Saturday_Night_Live_(season_6), http://dbpedia.org/resource/Sirota's_Court, http://dbpedia.org/resource/Teachers_(2006_U.S._TV_series), http://dbpedia.org/resource/The_Bill_Cosby_Show, http://dbpedia.org/resource/The_Castaways_on_Gilligan's_Island, http://dbpedia.org/resource/The_Duck_Factory, http://dbpedia.org/resource/The_Others_(TV_series), http://dbpedia.org/resource/The_Pursuit_of_Happiness_(TV_series), http://dbpedia.org/resource/Twenty_Good_Years, http://dbpedia.org/resource/My_Name_Is_Earl_(season_1), http://dbpedia.org/resource/Raines, http://dbpedia.org/resource/Journeyman_(TV_series), http://dbpedia.org/resource/List_of_The_Tonight_Show_Starring_Johnny_Carson_episodes_(1977), http://dbpedia.org/resource/Born_Innocent_(film), http://dbpedia.org/resource/Polly_(1989_film), http://dbpedia.org/resource/How_to_Break_Up_a_Happy_Divorce, http://dbpedia.org/resource/Leave_It_to_the_Girls, http://dbpedia.org/resource/The_Greatest_Gift_(film), http://dbpedia.org/resource/America's_Next_Great_Restaurant, http://dbpedia.org/resource/A_Cry_for_Help:_The_Tracey_Thurman_Story, http://dbpedia.org/resource/Columbo_(season_1), http://dbpedia.org/resource/Columbo_(season_6), http://dbpedia.org/resource/Columbo_(season_7), http://dbpedia.org/resource/The_Hero_(1966_TV_series), http://dbpedia.org/resource/100_Questions, http://dbpedia.org/resource/Face_the_Ace, http://dbpedia.org/resource/Hank_(1965_TV_series), http://dbpedia.org/resource/Hawaii_(TV_series), http://dbpedia.org/resource/Inconceivable, http://dbpedia.org/resource/Karen_(1964_TV_series), http://dbpedia.org/resource/Meet_My_Folks, http://dbpedia.org/resource/Mr._Rhodes, http://dbpedia.org/resource/Pride_&_Joy_(TV_series), http://dbpedia.org/resource/The_Great_American_Road_Trip, http://dbpedia.org/resource/The_Night_They_Took_Miss_Beautiful, http://dbpedia.org/resource/Punch_and_Jody, http://dbpedia.org/resource/Somerset_Maugham_TV_Theatre, http://dbpedia.org/resource/Undercovers_(TV_series), http://dbpedia.org/resource/Growing_Up_Brady_(film), http://dbpedia.org/resource/How_to_Eat_Like_a_Child, http://dbpedia.org/resource/In_the_Beginning_(miniseries), http://dbpedia.org/resource/Homicide:_Life_on_the_Street_(season_1), http://dbpedia.org/resource/Homicide:_Life_on_the_Street_(season_2), http://dbpedia.org/resource/Homicide:_Life_on_the_Street_(season_3), http://dbpedia.org/resource/Homicide:_Life_on_the_Street_(season_4), http://dbpedia.org/resource/Mindreaders, http://dbpedia.org/resource/Crash:_The_Mystery_of_Flight_1501, http://dbpedia.org/resource/My_Dad_Is_Better_Than_Your_Dad, http://dbpedia.org/resource/Doomsday_Virus, http://dbpedia.org/resource/Law_&_Order:_Criminal_Intent_(season_5), http://dbpedia.org/resource/Sportsworld_(U.S._TV_series), http://dbpedia.org/resource/The_Home_Court, http://dbpedia.org/resource/Parks_and_Recreation_(season_4), http://dbpedia.org/resource/Parks_and_Recreation_(season_6), http://dbpedia.org/resource/In_the_Loop_with_iVillage, http://dbpedia.org/resource/Law_&_Order:_Criminal_Intent_(season_1), http://dbpedia.org/resource/Law_&_Order:_Criminal_Intent_(season_2), http://dbpedia.org/resource/Law_&_Order:_Criminal_Intent_(season_4), http://dbpedia.org/resource/Law_&_Order:_Criminal_Intent_(season_6), http://dbpedia.org/resource/Saturday_Night_Live_(season_14), http://dbpedia.org/resource/Saturday_Night_Live_(season_15), http://dbpedia.org/resource/Saturday_Night_Live_(season_17), http://dbpedia.org/resource/Saturday_Night_Live_(season_22), http://dbpedia.org/resource/Saturday_Night_Live_(season_25), http://dbpedia.org/resource/Saturday_Night_Live_(season_33), http://dbpedia.org/resource/Saturday_Night_Live_(season_41), http://dbpedia.org/resource/Saturday_Night_Live_(season_8), http://dbpedia.org/resource/Community_(season_1), http://dbpedia.org/resource/Eyes_of_Terror, http://dbpedia.org/resource/Stonestreet:_Who_Killed_the_Centerfold_Model%3F, http://dbpedia.org/resource/The_West_Wing_(season_3), http://dbpedia.org/resource/Money_Talks_News, http://dbpedia.org/resource/Lost_(game_show), http://dbpedia.org/resource/The_Biggest_Loser:_Battle_of_the_Ages, http://dbpedia.org/resource/The_Biggest_Loser:_Challenge_America, http://dbpedia.org/resource/The_Biggest_Loser:_Pay_It_Forward, http://dbpedia.org/resource/The_Biggest_Loser:_Second_Chances, http://dbpedia.org/resource/The_Biggest_Loser_(season_1), http://dbpedia.org/resource/The_Biggest_Loser_(season_2), http://dbpedia.org/resource/The_Biggest_Loser_(season_4), http://dbpedia.org/resource/Harry's_Law_(season_2), http://dbpedia.org/resource/Grimm_(season_3), http://dbpedia.org/resource/Community_(season_4), http://dbpedia.org/resource/Boss_Lady, http://dbpedia.org/resource/Frasier_(season_7), http://dbpedia.org/resource/Casper's_Halloween_Special, http://dbpedia.org/resource/Son-Rise:_A_Miracle_of_Love, http://dbpedia.org/resource/The_Golden_Girls_(season_7), http://dbpedia.org/resource/Quantum_Leap_(season_1), http://dbpedia.org/resource/Quantum_Leap_(season_4), http://dbpedia.org/resource/Three_Wishes_(U.S._TV_series), http://dbpedia.org/resource/List_of_Late_Night_with_Jimmy_Fallon_episodes_(2013–14), http://dbpedia.org/resource/List_of_The_Tonight_Show_Starring_Jimmy_Fallon_episodes_(2015), http://dbpedia.org/resource/List_of_Late_Night_with_Seth_Meyers_episodes_(2014), http://dbpedia.org/resource/List_of_Late_Night_with_Seth_Meyers_episodes_(2016), http://dbpedia.org/resource/30_Rock, http://dbpedia.org/resource/A.D._The_Bible_Continues, http://dbpedia.org/resource/ALF:_The_Animated_Series, http://dbpedia.org/resource/Another_World_(TV_series), http://dbpedia.org/resource/Candid_Camera, http://dbpedia.org/resource/Days_of_Our_Lives, http://dbpedia.org/resource/Hallmark_Hall_of_Fame, http://dbpedia.org/resource/Heidi_(1968_film), http://dbpedia.org/resource/Law_&_Order, http://dbpedia.org/resource/Law_&_Order_(season_5), http://dbpedia.org/resource/Meet_the_Press, http://dbpedia.org/resource/Seinfeld, http://dbpedia.org/resource/Treasure_Hunters_(TV_series), http://dbpedia.org/resource/Wild_Texas_Wind, http://dbpedia.org/resource/Amazing_Stories_(TV_series), http://dbpedia.org/resource/Dr._Kildare_(TV_series), http://dbpedia.org/resource/J.J._Starbuck, http://dbpedia.org/resource/Land_of_the_Lost_(1974_TV_series), http://dbpedia.org/resource/Late_Night_with_Jimmy_Fallon, http://dbpedia.org/resource/Man_of_the_People_(TV_series), http://dbpedia.org/resource/Nashville_Star, http://dbpedia.org/resource/The_Danny_Thomas_Hour, http://dbpedia.org/resource/The_Death_of_the_Incredible_Hulk, http://dbpedia.org/resource/The_Incredible_Hulk_Returns, http://dbpedia.org/resource/The_Lieutenant, http://dbpedia.org/resource/The_Magician_(U.S._TV_series), http://dbpedia.org/resource/The_Oregon_Trail_(TV_series), http://dbpedia.org/resource/The_Trial_of_the_Incredible_Hulk, http://dbpedia.org/resource/You_Bet_Your_Life, http://dbpedia.org/resource/A_Killer_Among_Us, http://dbpedia.org/resource/Bitter_Harvest_(1981_film), http://dbpedia.org/resource/CHiPs, http://dbpedia.org/resource/Centennial_(miniseries), http://dbpedia.org/resource/Ed_(TV_series), http://dbpedia.org/resource/Friday_Night_Lights_(TV_series), http://dbpedia.org/resource/Hollywood_Squares, http://dbpedia.org/resource/Kidnapped_(TV_series), http://dbpedia.org/resource/Late_Night_(NBC), http://dbpedia.org/resource/NASCAR_on_NBC, http://dbpedia.org/resource/Police_Story_(TV_series), http://dbpedia.org/resource/The_A-Team, http://dbpedia.org/resource/The_David_Letterman_Show, http://dbpedia.org/resource/The_Dick_Powell_Show, http://dbpedia.org/resource/The_Gong_Show, http://dbpedia.org/resource/The_Tonight_Show_with_Conan_O'Brien, http://dbpedia.org/resource/Babes_in_Toyland_(1986_film), http://dbpedia.org/resource/Caroline_in_the_City, http://dbpedia.org/resource/Cocaine:_One_Man's_Seduction, http://dbpedia.org/resource/David_Copperfield_(1993_film), http://dbpedia.org/resource/Elf:_Buddy's_Musical_Christmas, http://dbpedia.org/resource/Friday_Night_Videos, http://dbpedia.org/resource/Hollywood_Game_Night, http://dbpedia.org/resource/Homicide:_Life_on_the_Street_(season_5), http://dbpedia.org/resource/How_Murray_Saved_Christmas, http://dbpedia.org/resource/Light's_Diamond_Jubilee, http://dbpedia.org/resource/McMillan_&_Wife, http://dbpedia.org/resource/Mr._Krueger's_Christmas, http://dbpedia.org/resource/My_World_and_Welcome_to_It, http://dbpedia.org/resource/Passions, http://dbpedia.org/resource/Phenomenon_(TV_series), http://dbpedia.org/resource/Poker_After_Dark, http://dbpedia.org/resource/Providence_(TV_series), http://dbpedia.org/resource/Saturday_Night_Live_(season_7), http://dbpedia.org/resource/Saved_by_the_Bell, http://dbpedia.org/resource/Shannon's_Deal, http://dbpedia.org/resource/The_Family_Tree, http://dbpedia.org/resource/The_Jimmy_Stewart_Show, http://dbpedia.org/resource/The_Magic_Clown, http://dbpedia.org/resource/The_New_Fantastic_Four, http://dbpedia.org/resource/The_Office_(U.S._TV_series), http://dbpedia.org/resource/The_Office_(U.S._season_7), http://dbpedia.org/resource/Thundarr_the_Barbarian, http://dbpedia.org/resource/Tonight_Starring_Jack_Paar, http://dbpedia.org/resource/Veronica's_Closet, http://dbpedia.org/resource/Death_of_a_Centerfold:_The_Dorothy_Stratten_Story, http://dbpedia.org/resource/Ensign_O'Toole, http://dbpedia.org/resource/Grandpa_Goes_to_Washington, http://dbpedia.org/resource/Poison_Ivy_(1985_film), http://dbpedia.org/resource/The_Harlem_Globetrotters_on_Gilligan's_Island, http://dbpedia.org/resource/The_Restless_Gun, http://dbpedia.org/resource/The_Black_Donnellys, http://dbpedia.org/resource/Law_&_Order:_Special_Victims_Unit_(season_14), http://dbpedia.org/resource/Undateable, http://dbpedia.org/resource/Fame_Is_the_Name_of_the_Game, http://dbpedia.org/resource/Saturday_Night_Live_(season_28), http://dbpedia.org/resource/Saturday_Night_Live_(season_29), http://dbpedia.org/resource/The_Incredible_Hulk_(1982_TV_series), http://dbpedia.org/resource/The_Odyssey_(miniseries), http://dbpedia.org/resource/Brand_New_Life_(TV_series), http://dbpedia.org/resource/Everything's_Relative_(1999_TV_series), http://dbpedia.org/resource/30_Rock_(season_4), http://dbpedia.org/resource/Snap_Judgment_(game_show), http://dbpedia.org/resource/Will_&_Grace_(season_1), http://dbpedia.org/resource/Will_&_Grace_(season_2), http://dbpedia.org/resource/Will_&_Grace_(season_3), http://dbpedia.org/resource/Will_&_Grace_(season_4), http://dbpedia.org/resource/Will_&_Grace_(season_5), http://dbpedia.org/resource/Will_&_Grace_(season_6), http://dbpedia.org/resource/Will_&_Grace_(season_7), http://dbpedia.org/resource/Will_&_Grace_(season_8), http://dbpedia.org/resource/Child_Bride_of_Short_Creek, http://dbpedia.org/resource/Sarah_T._–_Portrait_of_a_Teenage_Alcoholic, http://dbpedia.org/resource/The_Rockford_Files_(season_3), http://dbpedia.org/resource/Star_Trek:_The_Original_Series_(season_1), http://dbpedia.org/resource/Star_Trek:_The_Original_Series_(season_2), http://dbpedia.org/resource/The_Kid_with_the_200_I.Q., http://dbpedia.org/resource/The_Possessed_(1977_film), http://dbpedia.org/resource/One_Happy_Family, http://dbpedia.org/resource/Say_When!!, http://dbpedia.org/resource/Space_Cats, http://dbpedia.org/resource/The_Red_Pony_(1973_film), http://dbpedia.org/resource/Mama's_Boy_(TV_series), http://dbpedia.org/resource/Crusoe_(TV_series), http://dbpedia.org/resource/She_Cried_No, http://dbpedia.org/resource/30_Rock_(season_6), http://dbpedia.org/resource/The_Little_Rascals_Christmas_Special, http://dbpedia.org/resource/The_Rockford_Files_(season_1), http://dbpedia.org/resource/Meteor_(miniseries), http://dbpedia.org/resource/Rosetti_and_Ryan, http://dbpedia.org/resource/Who's_Watching_the_Kids%3F, http://dbpedia.org/resource/Diana:_Her_True_Story, http://dbpedia.org/resource/Great_Ghost_Tales, http://dbpedia.org/resource/The_Rockford_Files_(season_4), http://dbpedia.org/resource/The_A-Team_(season_4), http://dbpedia.org/resource/Norby_(TV_series), http://dbpedia.org/resource/The_Armed_Forces_Hour, http://dbpedia.org/resource/Friday_Night_Lights_(season_4), http://dbpedia.org/resource/My_Name_Is_Earl_(season_3), http://dbpedia.org/resource/List_of_The_Tonight_Show_with_Jay_Leno_episodes_(1992–99), http://dbpedia.org/resource/List_of_The_Tonight_Show_with_Jay_Leno_episodes_(2010), http://dbpedia.org/resource/The_Winds_of_Whoopie, http://dbpedia.org/resource/Cheers_(season_10), http://dbpedia.org/resource/The_Biggest_Loser_(Dutch_TV_series), http://dbpedia.org/resource/Baywatch, http://dbpedia.org/resource/Cheers, http://dbpedia.org/resource/Fear_Factor, http://dbpedia.org/resource/Gambit_(game_show), http://dbpedia.org/resource/Major_League_Baseball_on_NBC, http://dbpedia.org/resource/NBC_Nightly_News, http://dbpedia.org/resource/The_Baseball_Network, http://dbpedia.org/resource/The_Fresh_Prince_of_Bel-Air, http://dbpedia.org/resource/The_Revenge_of_Al_Capone, http://dbpedia.org/resource/The_Tonight_Show_Starring_Johnny_Carson, http://dbpedia.org/resource/The_West_Wing, http://dbpedia.org/resource/This_Is_Your_Life, http://dbpedia.org/resource/Today_(U.S._TV_program), http://dbpedia.org/resource/Will_&_Grace, http://dbpedia.org/resource/Your_Show_of_Shows, http://dbpedia.org/resource/A_Different_World, http://dbpedia.org/resource/Hannibal_(TV_series), http://dbpedia.org/resource/Law_&_Order:_Special_Victims_Unit_(season_13), http://dbpedia.org/resource/Let's_Make_a_Deal, http://dbpedia.org/resource/My_Name_Is_Earl, http://dbpedia.org/resource/Night_Court, http://dbpedia.org/resource/Star_Trek:_The_Animated_Series, http://dbpedia.org/resource/The_Price_Is_Right_(1956_U.S._game_show), http://dbpedia.org/resource/This_Is_Show_Business, http://dbpedia.org/resource/Truth_or_Consequences, http://dbpedia.org/resource/Twenty_One_(game_show), http://dbpedia.org/resource/Unsolved_Mysteries, http://dbpedia.org/resource/The_Hobbit_(1977_film), http://dbpedia.org/resource/The_Murder_of_Mary_Phagan, http://dbpedia.org/resource/Bates_Motel_(film), http://dbpedia.org/resource/Dick_Clark's_New_Year's_Rockin'_Eve, http://dbpedia.org/resource/Law_of_the_Plainsman, http://dbpedia.org/resource/Lipstick_Jungle_(TV_series), http://dbpedia.org/resource/Scared_Shrekless, http://dbpedia.org/resource/Gavilan_(TV_series), http://dbpedia.org/resource/Name_That_Tune, http://dbpedia.org/resource/Profiler_(TV_series), http://dbpedia.org/resource/American_Dreams, http://dbpedia.org/resource/Crusader_Rabbit, http://dbpedia.org/resource/Dante_(TV_series), http://dbpedia.org/resource/King_Leonardo_and_His_Short_Subjects, http://dbpedia.org/resource/Meet_McGraw, http://dbpedia.org/resource/The_Loneliest_Runner, http://dbpedia.org/resource/Laura_Lansing_Slept_Here, http://dbpedia.org/resource/One_Christmas_(film), http://dbpedia.org/resource/Parenthood_(1990_TV_series), http://dbpedia.org/resource/The_Rebels_(TV_series), http://dbpedia.org/resource/The_Troubleshooters_(1959_TV_series), http://dbpedia.org/resource/Law_&_Order:_Special_Victims_Unit_(season_1), http://dbpedia.org/resource/Law_&_Order:_Special_Victims_Unit_(season_2), http://dbpedia.org/resource/Sunday_Night_(American_TV_program), http://dbpedia.org/resource/CB_Bears, http://dbpedia.org/resource/Casper_and_the_Angels, http://dbpedia.org/resource/Foofur, http://dbpedia.org/resource/Going_Bananas_(U.S._series), http://dbpedia.org/resource/Hey,_Landlord, http://dbpedia.org/resource/Jokebook, http://dbpedia.org/resource/Match_Game-Hollywood_Squares_Hour, http://dbpedia.org/resource/Shirley_(TV_series), http://dbpedia.org/resource/Shirt_Tales, http://dbpedia.org/resource/Snorks, http://dbpedia.org/resource/Space_Stars, http://dbpedia.org/resource/The_Space_Kidettes, http://dbpedia.org/resource/Wheelie_and_the_Chopper_Bunch, http://dbpedia.org/resource/Yo_Yogi!, http://dbpedia.org/resource/Yogi's_Space_Race, http://dbpedia.org/resource/Danielle_Steel's_Star, http://dbpedia.org/resource/Family_Album_(miniseries), http://dbpedia.org/resource/Mister_Sterling, http://dbpedia.org/resource/Perry_Mason_(TV_movies), http://dbpedia.org/resource/Saturday_Night's_Main_Event, http://dbpedia.org/resource/Saturday_Night_Live_(season_9), http://dbpedia.org/resource/Tate_(TV_series), http://dbpedia.org/resource/The_Firm_(2012_TV_series), http://dbpedia.org/resource/Michael_Shayne, http://dbpedia.org/resource/My_Sweet_Charlie, http://dbpedia.org/resource/The_Californians_(TV_series), http://dbpedia.org/resource/For_Your_Love_(TV_series), http://dbpedia.org/resource/Tennis_on_NBC, http://dbpedia.org/resource/The_Gangster_Chronicles, http://dbpedia.org/resource/The_Great_Escape_II:_The_Untold_Story, http://dbpedia.org/resource/The_Temptations_(miniseries), http://dbpedia.org/resource/A_Year_in_the_Life, http://dbpedia.org/resource/Blind_Date_(1950s_game_show), http://dbpedia.org/resource/Brotherly_Love_(1995_TV_series), http://dbpedia.org/resource/Fall_into_Darkness, http://dbpedia.org/resource/Hidden_Faces, http://dbpedia.org/resource/How_to_Survive_a_Marriage, http://dbpedia.org/resource/My_Friend_Tony, http://dbpedia.org/resource/Our_Five_Daughters, http://dbpedia.org/resource/Saber_of_London, http://dbpedia.org/resource/Sister,_Sister_(1982_film), http://dbpedia.org/resource/The_Bold_Ones:_The_Senator, http://dbpedia.org/resource/The_Brighter_Day, http://dbpedia.org/resource/The_John_Forsythe_Show, http://dbpedia.org/resource/The_Round_Table_(TV_series), http://dbpedia.org/resource/The_Sophisticated_Gents, http://dbpedia.org/resource/A_Reason_to_Live_(1985_film), http://dbpedia.org/resource/Little_House_on_the_Prairie_(film), http://dbpedia.org/resource/Law_&_Order:_Special_Victims_Unit_(season_11), http://dbpedia.org/resource/Law_&_Order:_Special_Victims_Unit_(season_12), http://dbpedia.org/resource/Law_&_Order:_Special_Victims_Unit_(season_15), http://dbpedia.org/resource/Law_&_Order:_Special_Victims_Unit_(season_16), http://dbpedia.org/resource/Law_&_Order:_Special_Victims_Unit_(season_17), http://dbpedia.org/resource/Law_&_Order:_Special_Victims_Unit_(season_3), http://dbpedia.org/resource/Law_&_Order:_Special_Victims_Unit_(season_4), http://dbpedia.org/resource/Law_&_Order:_Special_Victims_Unit_(season_5), http://dbpedia.org/resource/Law_&_Order:_Special_Victims_Unit_(season_6), http://dbpedia.org/resource/Andy_Barker,_P.I., http://dbpedia.org/resource/Saturday_Night_Live_(season_26), http://dbpedia.org/resource/Saturday_Night_Live_(season_27), http://dbpedia.org/resource/Saturday_Night_Live_(season_30), http://dbpedia.org/resource/Scrubs_(season_1), http://dbpedia.org/resource/Scrubs_(season_2), http://dbpedia.org/resource/Scrubs_(season_3), http://dbpedia.org/resource/Scrubs_(season_5), http://dbpedia.org/resource/Scrubs_(season_6), http://dbpedia.org/resource/Scrubs_(season_7), http://dbpedia.org/resource/The_Golden_Girls_(season_2), http://dbpedia.org/resource/Saturday_Night_Live_(season_38), http://dbpedia.org/resource/The_Berenstain_Bears_Meet_Bigpaw, http://dbpedia.org/resource/ALF_Tales, http://dbpedia.org/resource/Private_Eye_(TV_series), http://dbpedia.org/resource/Age_of_Love_(TV_series), http://dbpedia.org/resource/Clash_of_the_Choirs, http://dbpedia.org/resource/A.U.S.A., http://dbpedia.org/resource/Frasier_(season_5), http://dbpedia.org/resource/The_Gambler_(film_series), http://dbpedia.org/resource/The_Jake_Effect, http://dbpedia.org/resource/The_Second_Half, http://dbpedia.org/resource/Crisis_Center, http://dbpedia.org/resource/PDQ_(game_show), http://dbpedia.org/resource/What's_This_Song%3F, http://dbpedia.org/resource/The_Stepford_Children, http://dbpedia.org/resource/Built_to_Last_(TV_series), http://dbpedia.org/resource/House_Rules_(1998_TV_series), http://dbpedia.org/resource/Mr._and_Mrs._North, http://dbpedia.org/resource/Black_Tie_Affair_(TV_series), http://dbpedia.org/resource/Chicago_Sons, http://dbpedia.org/resource/The_World_of_Mr._Sweeney, http://dbpedia.org/resource/Friday_Night_Lights_(season_1), http://dbpedia.org/resource/Snip_(TV_series), http://dbpedia.org/resource/Her_Hidden_Truth, http://dbpedia.org/resource/If_Someone_Had_Known, http://dbpedia.org/resource/Police_Story:_Confessions_of_a_Lady_Cop, http://dbpedia.org/resource/Parks_and_Recreation_(season_1), http://dbpedia.org/resource/Parks_and_Recreation_(season_3), http://dbpedia.org/resource/Parks_and_Recreation_(season_5), http://dbpedia.org/resource/The_West_Wing_(season_6), http://dbpedia.org/resource/The_Biggest_Loser:_Glory_Days, http://dbpedia.org/resource/Valley_Forge_(film), http://dbpedia.org/resource/The_Fresh_Prince_of_Bel-Air_(season_4), http://dbpedia.org/resource/Rewind_(2013_film), http://dbpedia.org/resource/Community_(season_5), http://dbpedia.org/resource/A_Burning_Passion:_The_Margaret_Mitchell_Story, http://dbpedia.org/resource/The_Crisis_(TV_series), http://dbpedia.org/resource/In_Wine_Country, http://dbpedia.org/resource/Friday_Night_Lights_(season_3), http://dbpedia.org/resource/People_Will_Talk_(game_show), http://dbpedia.org/resource/The_Office:_The_Accountants, http://dbpedia.org/resource/The_Swift_Show, http://dbpedia.org/resource/Frasier_(season_11), http://dbpedia.org/resource/The_Blacklist_(season_3), http://dbpedia.org/resource/Chain_Letter_(U.S._game_show), http://dbpedia.org/resource/3rd_Rock_from_the_Sun_(season_3), http://dbpedia.org/resource/Just_Shoot_Me!_(season_1), http://dbpedia.org/resource/Just_Shoot_Me!_(season_2), http://dbpedia.org/resource/Just_Shoot_Me!_(season_3), http://dbpedia.org/resource/Just_Shoot_Me!_(season_4), http://dbpedia.org/resource/Just_Shoot_Me!_(season_5), http://dbpedia.org/resource/Just_Shoot_Me!_(season_6), http://dbpedia.org/resource/Just_Shoot_Me!_(season_7), http://dbpedia.org/resource/The_Rockford_Files_(season_5), http://dbpedia.org/resource/The_Rockford_Files_(season_6), http://dbpedia.org/resource/All_Star_Secrets, http://dbpedia.org/resource/Columbo, http://dbpedia.org/resource/Crossing_Jordan, http://dbpedia.org/resource/Cruel_Doubt, http://dbpedia.org/resource/ER_(TV_series), http://dbpedia.org/resource/Empire_(1962_TV_series), http://dbpedia.org/resource/Football_Night_in_America, http://dbpedia.org/resource/Frasier, http://dbpedia.org/resource/Generations_(U.S._TV_series), http://dbpedia.org/resource/Golf_Channel_on_NBC, http://dbpedia.org/resource/Hill_Street_Blues, http://dbpedia.org/resource/Homicide:_Life_on_the_Street, http://dbpedia.org/resource/Las_Vegas_(TV_series), http://dbpedia.org/resource/Late_Night_with_Conan_O'Brien, http://dbpedia.org/resource/Late_Night_with_David_Letterman, http://dbpedia.org/resource/Later_(talk_show), http://dbpedia.org/resource/Matlock_(TV_series), http://dbpedia.org/resource/NBA_on_NBC, http://dbpedia.org/resource/NBC_Sunday_Night_Football, http://dbpedia.org/resource/NFL_on_NBC, http://dbpedia.org/resource/NHL_on_NBC, http://dbpedia.org/resource/NewsRadio, http://dbpedia.org/resource/Olympics_on_NBC, http://dbpedia.org/resource/Quantum_Leap, http://dbpedia.org/resource/Rock_Center_with_Brian_Williams, http://dbpedia.org/resource/Second_City_Television, http://dbpedia.org/resource/Sunshine_(U.S._TV_series), http://dbpedia.org/resource/Tattingers, http://dbpedia.org/resource/The_Rockford_Files, http://dbpedia.org/resource/Thursday_Night_Football, http://dbpedia.org/resource/Up_All_Night_(TV_series), http://dbpedia.org/resource/A_Man_and_His_Music_+_Ella_+_Jobim, http://dbpedia.org/resource/Baa_Baa_Black_Sheep_(TV_series), http://dbpedia.org/resource/Frank_Sinatra:_A_Man_and_His_Music, http://dbpedia.org/resource/Friends_(season_1), http://dbpedia.org/resource/Friends_(season_10), http://dbpedia.org/resource/Friends_(season_2), http://dbpedia.org/resource/Friends_(season_3), http://dbpedia.org/resource/Friends_(season_4), http://dbpedia.org/resource/Friends_(season_5), http://dbpedia.org/resource/Friends_(season_6), http://dbpedia.org/resource/Friends_(season_7), http://dbpedia.org/resource/Friends_(season_8), http://dbpedia.org/resource/Friends_(season_9), http://dbpedia.org/resource/Get_Smart, http://dbpedia.org/resource/Grimm_(season_4), http://dbpedia.org/resource/Joey_(TV_series), http://dbpedia.org/resource/Mad_About_You, http://dbpedia.org/resource/Wichita_Town, http://dbpedia.org/resource/Wings_(1990_TV_series), http://dbpedia.org/resource/Cartoon_All-Stars_to_the_Rescue, http://dbpedia.org/resource/Just_Shoot_Me!, http://dbpedia.org/resource/Law_&_Order:_Criminal_Intent, http://dbpedia.org/resource/The_New_Adventures_of_Huckleberry_Finn, http://dbpedia.org/resource/A_Muppets_Christmas:_Letters_to_Santa, http://dbpedia.org/resource/America's_Got_Talent, http://dbpedia.org/resource/Captain_N:_The_Game_Master, http://dbpedia.org/resource/Chuck_(TV_series), http://dbpedia.org/resource/ER_(season_3), http://dbpedia.org/resource/Emergency!, http://dbpedia.org/resource/Fired_Up_(TV_series), http://dbpedia.org/resource/Hunter_(U.S._TV_series), http://dbpedia.org/resource/It's_a_Very_Merry_Muppet_Christmas_Movie, http://dbpedia.org/resource/Lucas_Tanner, http://dbpedia.org/resource/Macy's_Thanksgiving_Day_Parade, http://dbpedia.org/resource/Man_from_Atlantis, http://dbpedia.org/resource/Mickey's_60th_Birthday, http://dbpedia.org/resource/Police_Woman_(TV_series), http://dbpedia.org/resource/Sisters_(TV_series), http://dbpedia.org/resource/Stark_Raving_Mad_(TV_series), http://dbpedia.org/resource/The_Bold_Ones:_The_New_Doctors, http://dbpedia.org/resource/The_Highwayman_(TV_series), http://dbpedia.org/resource/Third_Watch, http://dbpedia.org/resource/Westwind_(TV_series), http://dbpedia.org/resource/1st_Look, http://dbpedia.org/resource/3-2-1_Penguins!, http://dbpedia.org/resource/A_Man_and_His_Music_–_Part_II, http://dbpedia.org/resource/Adam-12, http://dbpedia.org/resource/Buffalo_Bill_(TV_series), http://dbpedia.org/resource/California_Dreams, http://dbpedia.org/resource/Card_Sharks, http://dbpedia.org/resource/City_Guys, http://dbpedia.org/resource/Concentration_(game_show), http://dbpedia.org/resource/Deal_or_No_Deal_(U.S._game_show), http://dbpedia.org/resource/Early_Today, http://dbpedia.org/resource/Endurance_(TV_series), http://dbpedia.org/resource/Got_Talent, http://dbpedia.org/resource/H.R._Pufnstuf, http://dbpedia.org/resource/Hang_Time_(TV_series), http://dbpedia.org/resource/High_Rollers, http://dbpedia.org/resource/Highway_to_Heaven, http://dbpedia.org/resource/Huntley-Brinkley_Report, http://dbpedia.org/resource/In_the_Heat_of_the_Night_(TV_series), http://dbpedia.org/resource/Knight_Rider_(2008_TV_series), http://dbpedia.org/resource/Leeza, http://dbpedia.org/resource/Minute_to_Win_It, http://dbpedia.org/resource/NBC_News_at_Sunrise, http://dbpedia.org/resource/NBC_Nightside, http://dbpedia.org/resource/Nero_Wolfe_(1981_TV_series), http://dbpedia.org/resource/Notre_Dame_Football_on_NBC, http://dbpedia.org/resource/One_World_(TV_series), http://dbpedia.org/resource/One_to_Grow_On, http://dbpedia.org/resource/Password_Plus_and_Super_Password, http://dbpedia.org/resource/Santa_Barbara_(TV_series), http://dbpedia.org/resource/Saved_by_the_Bell:_The_New_Class, http://dbpedia.org/resource/Scout's_Safari, http://dbpedia.org/resource/Somerset_(TV_series), http://dbpedia.org/resource/Sunset_Beach_(TV_series), http://dbpedia.org/resource/The_Apprentice_(U.S._TV_series), http://dbpedia.org/resource/The_Banana_Splits, http://dbpedia.org/resource/The_Biggest_Loser_(U.S._TV_series), http://dbpedia.org/resource/The_Bugaloos, http://dbpedia.org/resource/The_Flip_Wilson_Show, http://dbpedia.org/resource/The_Hogan_Family, http://dbpedia.org/resource/The_Krofft_Superstar_Hour, http://dbpedia.org/resource/The_Marriage_Ref_(U.S._TV_series), http://dbpedia.org/resource/The_Smurfs_(TV_series), http://dbpedia.org/resource/The_Waverly_Wonders, http://dbpedia.org/resource/Time_Warp_Trio, http://dbpedia.org/resource/Tomorrow_(TV_series), http://dbpedia.org/resource/Tonight_Starring_Steve_Allen, http://dbpedia.org/resource/Weekend_Today, http://dbpedia.org/resource/Fabulous_Funnies, http://dbpedia.org/resource/God,_the_Devil_and_Bob, http://dbpedia.org/resource/Star_Trek:_The_Original_Series_(season_3), http://dbpedia.org/resource/The_Kid_Super_Power_Hour_with_Shazam!, http://dbpedia.org/resource/The_New_Adventures_of_Flash_Gordon, http://dbpedia.org/resource/The_Secret_Lives_of_Waldo_Kitty, http://dbpedia.org/resource/A_Tour_of_the_White_House_with_Mrs._John_F._Kennedy, http://dbpedia.org/resource/ProStars, http://dbpedia.org/resource/Boomtown_(2002_TV_series), http://dbpedia.org/resource/Frasier_(season_6), http://dbpedia.org/resource/Knight_Rider_2000, http://dbpedia.org/resource/Team_Knight_Rider, http://dbpedia.org/resource/Evil_Roy_Slade, http://dbpedia.org/resource/Jackpot_Bowling, http://dbpedia.org/resource/San_Francisco_International_Airport_(TV_series), http://dbpedia.org/resource/The_Paul_Winchell_Show, http://dbpedia.org/resource/The_Pink_Panther_Show, http://dbpedia.org/resource/An_Evening_with_Fred_Astaire, http://dbpedia.org/resource/Choices_of_the_Heart, http://dbpedia.org/resource/Down_You_Go, http://dbpedia.org/resource/Father_Dowling_Mysteries, http://dbpedia.org/resource/Fear_Itself_(TV_series), http://dbpedia.org/resource/Godzilla_(animated_series), http://dbpedia.org/resource/Hec_Ramsey, http://dbpedia.org/resource/Monster_Squad, http://dbpedia.org/resource/Poor_Little_Rich_Girl:_The_Barbara_Hutton_Story, http://dbpedia.org/resource/Prime_Suspect_(U.S._TV_series), http://dbpedia.org/resource/The_Psychiatrist_(TV_series), http://dbpedia.org/resource/Law_&_Order:_LA, http://dbpedia.org/resource/Law_&_Order:_Trial_by_Jury, http://dbpedia.org/resource/Witness_to_the_Mob, http://dbpedia.org/resource/Camp_Candy, http://dbpedia.org/resource/Saturday_Night_Live_(season_13), http://dbpedia.org/resource/Witch's_Night_Out, http://dbpedia.org/resource/Columbo_(season_3), http://dbpedia.org/resource/Committed_(2005_TV_series), http://dbpedia.org/resource/Legends_of_the_Superheroes, http://dbpedia.org/resource/List_of_The_Alfred_Hitchcock_Hour_episodes, http://dbpedia.org/resource/Saturday_Night_Live_(season_20), http://dbpedia.org/resource/Stars_Earn_Stripes, http://dbpedia.org/resource/The_Court_of_Last_Resort, http://dbpedia.org/resource/$25_Million_Dollar_Hoax, http://dbpedia.org/resource/Cold_Feet_(U.S._TV_series), http://dbpedia.org/resource/Coupling_(U.S._TV_series), http://dbpedia.org/resource/I'm_a_Celebrity...Get_Me_Out_of_Here!_(U.S._TV_series), http://dbpedia.org/resource/Men_Behaving_Badly_(U.S._TV_series), http://dbpedia.org/resource/Return_to_the_Planet_of_the_Apes, http://dbpedia.org/resource/America's_Toughest_Jobs, http://dbpedia.org/resource/Amy_Prentiss, http://dbpedia.org/resource/Are_You_There,_Chelsea%3F, http://dbpedia.org/resource/Baby_Boom_(U.S._TV_series), http://dbpedia.org/resource/Battery_Park_(TV_series), http://dbpedia.org/resource/Blank_Check_(game_show), http://dbpedia.org/resource/Buford_and_the_Galloping_Ghost, http://dbpedia.org/resource/C._Everett_Koop,_M.D., http://dbpedia.org/resource/Campus_Hoopla, http://dbpedia.org/resource/Chicago_Story]... [INFO] P: 0.9200792602377807, R: 1.0, F: 0.9583763329893361 -[INFO] Cumulative metrics, sample: 635, P: 0.514, R: 0.564, macro F1: 0.520, macro F1*: 0.538 -[INFO] Current time: 2412 ms, average: 2727.80 ms/ques, entity detection: 422.20 ms/ques (linking tool: 329.77 ms/ques), relation detection: 901.20 ms/ques (similarity: 707.71 ms/ques), EDG generation: 48.78 ms/ques +[INFO] Cumulative metrics, sample: 635, P: 0.514, R: 0.564, macro F1: 0.538 +[INFO] Current question consumed time: 2166 ms, average: 2474.56 ms/ques, entity detection: 376.47 ms/ques (linking tool: 297.27 ms/ques), relation detection: 837.03 ms/ques (similarity: 697.01 ms/ques), EDG generation: 48.60 ms/ques [INFO] Entity Linking P: 0.7511811023622047, R: 0.7031496062992126, micro F1: 0.7183727034120744, macro F1: 0.7263721975197064 [INFO] Relation Linking P: 0.5943569553805774, R: 0.5425196850393701, micro F1: 0.5570078740157484, macro F1: 0.5672565285797777 [INFO] Type Linking P: 0.5496062992125984, R: 0.5496062992125984, micro F1: 0.5496062992125984, macro F1: 0.5496062992125984 @@ -8920,8 +8920,8 @@ Question 635: Who is the fictional character whose relative is Vestigial Peter? [INFO] Golden Answer: [http://dbpedia.org/resource/Peter_Griffin] [INFO] Predicted Answer: [http://dbpedia.org/resource/Peter_Griffin] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 636, P: 0.515, R: 0.565, macro F1: 0.521, macro F1*: 0.539 -[INFO] Current time: 1060 ms, average: 2725.17 ms/ques, entity detection: 421.78 ms/ques (linking tool: 329.40 ms/ques), relation detection: 900.25 ms/ques (similarity: 707.07 ms/ques), EDG generation: 48.73 ms/ques +[INFO] Cumulative metrics, sample: 636, P: 0.515, R: 0.565, macro F1: 0.539 +[INFO] Current question consumed time: 946 ms, average: 2472.15 ms/ques, entity detection: 376.08 ms/ques (linking tool: 296.92 ms/ques), relation detection: 836.14 ms/ques (similarity: 696.39 ms/ques), EDG generation: 48.56 ms/ques [INFO] Entity Linking P: 0.7515723270440252, R: 0.7036163522012578, micro F1: 0.7188155136268353, macro F1: 0.7268041412257186 [INFO] Relation Linking P: 0.5949947589098532, R: 0.5432389937106918, micro F1: 0.5577044025157236, macro F1: 0.5679402027029472 [INFO] Type Linking P: 0.5487421383647799, R: 0.5487421383647799, micro F1: 0.5487421383647799, macro F1: 0.5487421383647799 @@ -8934,8 +8934,8 @@ Question 636: How many home stadium are there, of the soccer club seasons whose [INFO] Golden Answer: [5] [INFO] Predicted Answer: [20, 15] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 637, P: 0.514, R: 0.564, macro F1: 0.520, macro F1*: 0.538 -[INFO] Current time: 923 ms, average: 2722.35 ms/ques, entity detection: 421.42 ms/ques (linking tool: 329.03 ms/ques), relation detection: 899.10 ms/ques (similarity: 706.20 ms/ques), EDG generation: 48.74 ms/ques +[INFO] Cumulative metrics, sample: 637, P: 0.514, R: 0.564, macro F1: 0.538 +[INFO] Current question consumed time: 786 ms, average: 2469.51 ms/ques, entity detection: 375.72 ms/ques (linking tool: 296.58 ms/ques), relation detection: 835.07 ms/ques (similarity: 695.52 ms/ques), EDG generation: 48.56 ms/ques [INFO] Entity Linking P: 0.7519623233908949, R: 0.7040816326530612, micro F1: 0.7192569335426487, macro F1: 0.7272347213818141 [INFO] Relation Linking P: 0.5956305599162741, R: 0.543171114599686, micro F1: 0.5578754578754582, macro F1: 0.5681925525037037 [INFO] Type Linking P: 0.5494505494505495, R: 0.5494505494505495, micro F1: 0.5494505494505495, macro F1: 0.5494505494505495 @@ -8948,8 +8948,8 @@ Question 637: Name the alma mater of Ernest de Saisset ? [INFO] Golden Answer: [http://dbpedia.org/resource/Santa_Clara_University] [INFO] Predicted Answer: [http://dbpedia.org/resource/Santa_Clara_University] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 638, P: 0.515, R: 0.565, macro F1: 0.521, macro F1*: 0.539 -[INFO] Current time: 767 ms, average: 2719.28 ms/ques, entity detection: 421.21 ms/ques (linking tool: 328.85 ms/ques), relation detection: 897.88 ms/ques (similarity: 705.23 ms/ques), EDG generation: 48.68 ms/ques +[INFO] Cumulative metrics, sample: 638, P: 0.515, R: 0.565, macro F1: 0.539 +[INFO] Current question consumed time: 536 ms, average: 2466.47 ms/ques, entity detection: 375.33 ms/ques (linking tool: 296.21 ms/ques), relation detection: 833.94 ms/ques (similarity: 694.56 ms/ques), EDG generation: 48.50 ms/ques [INFO] Entity Linking P: 0.7523510971786834, R: 0.7045454545454546, micro F1: 0.7196969696969706, macro F1: 0.7276639444471614 [INFO] Relation Linking P: 0.5962643678160919, R: 0.5438871473354232, micro F1: 0.558568443051202, macro F1: 0.5688726836032092 [INFO] Type Linking P: 0.5501567398119123, R: 0.5501567398119123, micro F1: 0.5501567398119123, macro F1: 0.5501567398119123 @@ -8962,8 +8962,8 @@ Question 638: What magazine companies are of form Limited liability company? [INFO] Golden Answer: [http://dbpedia.org/resource/Texas_Music, http://dbpedia.org/resource/Rolling_Stone, http://dbpedia.org/resource/Us_Weekly, http://dbpedia.org/resource/Milford_Magazine, http://dbpedia.org/resource/Horns_Illustrated] [INFO] Predicted Answer: [http://dbpedia.org/resource/Texas_Music, http://dbpedia.org/resource/Rolling_Stone, http://dbpedia.org/resource/Us_Weekly, http://dbpedia.org/resource/Milford_Magazine, http://dbpedia.org/resource/Horns_Illustrated] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 639, P: 0.516, R: 0.566, macro F1: 0.521, macro F1*: 0.540 -[INFO] Current time: 1106 ms, average: 2716.76 ms/ques, entity detection: 420.87 ms/ques (linking tool: 328.58 ms/ques), relation detection: 896.95 ms/ques (similarity: 704.56 ms/ques), EDG generation: 48.62 ms/ques +[INFO] Cumulative metrics, sample: 639, P: 0.516, R: 0.566, macro F1: 0.540 +[INFO] Current question consumed time: 1012 ms, average: 2464.20 ms/ques, entity detection: 375.02 ms/ques (linking tool: 295.97 ms/ques), relation detection: 833.08 ms/ques (similarity: 693.90 ms/ques), EDG generation: 48.44 ms/ques [INFO] Entity Linking P: 0.7527386541471048, R: 0.7050078247261345, micro F1: 0.7201356285863337, macro F1: 0.7280918168400874 [INFO] Relation Linking P: 0.5968961919666145, R: 0.5446009389671361, micro F1: 0.5592592592592597, macro F1: 0.5695506678059138 [INFO] Type Linking P: 0.5492957746478874, R: 0.5492957746478874, micro F1: 0.5492957746478874, macro F1: 0.5492957746478874 @@ -8976,8 +8976,8 @@ Question 639: List the driver of SA MArino Grand Prix is also the Pole driver of [INFO] Golden Answer: [http://dbpedia.org/resource/Michael_Schumacher] [INFO] Predicted Answer: [http://dbpedia.org/resource/Michael_Schumacher] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 640, P: 0.517, R: 0.566, macro F1: 0.522, macro F1*: 0.540 -[INFO] Current time: 2945 ms, average: 2717.11 ms/ques, entity detection: 420.47 ms/ques (linking tool: 328.23 ms/ques), relation detection: 896.92 ms/ques (similarity: 704.29 ms/ques), EDG generation: 48.76 ms/ques +[INFO] Cumulative metrics, sample: 640, P: 0.517, R: 0.566, macro F1: 0.540 +[INFO] Current question consumed time: 2645 ms, average: 2464.48 ms/ques, entity detection: 374.68 ms/ques (linking tool: 295.67 ms/ques), relation detection: 833.07 ms/ques (similarity: 693.65 ms/ques), EDG generation: 48.60 ms/ques [INFO] Entity Linking P: 0.753125, R: 0.7046875, micro F1: 0.7200520833333343, macro F1: 0.7281015541264737 [INFO] Relation Linking P: 0.5959635416666667, R: 0.54375, micro F1: 0.5583854166666671, macro F1: 0.5686607448874672 [INFO] Type Linking P: 0.55, R: 0.55, micro F1: 0.55, macro F1: 0.55 @@ -8990,8 +8990,8 @@ Question 640: Where does John A Danaher III work? [INFO] Golden Answer: [47, http://dbpedia.org/resource/Connecticut, http://dbpedia.org/resource/Commissioner, Department of Public Safety@en] [INFO] Predicted Answer: [Attorney@en] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 641, P: 0.516, R: 0.565, macro F1: 0.521, macro F1*: 0.539 -[INFO] Current time: 884 ms, average: 2714.25 ms/ques, entity detection: 420.09 ms/ques (linking tool: 327.93 ms/ques), relation detection: 896.00 ms/ques (similarity: 703.49 ms/ques), EDG generation: 48.70 ms/ques +[INFO] Cumulative metrics, sample: 641, P: 0.516, R: 0.565, macro F1: 0.539 +[INFO] Current question consumed time: 774 ms, average: 2461.84 ms/ques, entity detection: 374.32 ms/ques (linking tool: 295.39 ms/ques), relation detection: 832.21 ms/ques (similarity: 692.87 ms/ques), EDG generation: 48.54 ms/ques [INFO] Entity Linking P: 0.7535101404056163, R: 0.7051482059282371, micro F1: 0.7204888195527831, macro F1: 0.7285274512584782 [INFO] Relation Linking P: 0.595033801352054, R: 0.5429017160686428, micro F1: 0.5575143005720232, macro F1: 0.5677735986395929 [INFO] Type Linking P: 0.5507020280811232, R: 0.5507020280811232, micro F1: 0.5507020280811232, macro F1: 0.5507020280811232 @@ -9004,8 +9004,8 @@ Question 641: Name few movies directed by Tim Burton ? [INFO] Golden Answer: [http://dbpedia.org/resource/Planet_of_the_Apes_(2001_film), http://dbpedia.org/resource/Alice_in_Wonderland_(2010_film), http://dbpedia.org/resource/Batman_(1989_film), http://dbpedia.org/resource/Corpse_Bride, http://dbpedia.org/resource/Sleepy_Hollow_(film), http://dbpedia.org/resource/Sweeney_Todd:_The_Demon_Barber_of_Fleet_Street_(2007_film), http://dbpedia.org/resource/Charlie_and_the_Chocolate_Factory_(film), http://dbpedia.org/resource/Miss_Peregrine's_Home_for_Peculiar_Children_(film), http://dbpedia.org/resource/Mars_Attacks!, http://dbpedia.org/resource/Batman_Returns, http://dbpedia.org/resource/Beetlejuice, http://dbpedia.org/resource/Ed_Wood_(film), http://dbpedia.org/resource/Edward_Scissorhands, http://dbpedia.org/resource/Pee-wee's_Big_Adventure, http://dbpedia.org/resource/Big_Fish, http://dbpedia.org/resource/Dark_Shadows_(film), http://dbpedia.org/resource/Frankenweenie_(2012_film), http://dbpedia.org/resource/Frankenweenie_(1984_film), http://dbpedia.org/resource/Vincent_(1982_film), http://dbpedia.org/resource/Big_Eyes, http://dbpedia.org/resource/Hansel_and_Gretel_(1982_film), http://dbpedia.org/resource/Stalk_of_the_Celery_Monster, http://dbpedia.org/resource/The_Island_of_Doctor_Agor] [INFO] Predicted Answer: [http://dbpedia.org/resource/Planet_of_the_Apes_(2001_film), http://dbpedia.org/resource/Alice_in_Wonderland_(2010_film), http://dbpedia.org/resource/Batman_(1989_film), http://dbpedia.org/resource/Corpse_Bride, http://dbpedia.org/resource/Sleepy_Hollow_(film), http://dbpedia.org/resource/Sweeney_Todd:_The_Demon_Barber_of_Fleet_Street_(2007_film), http://dbpedia.org/resource/Charlie_and_the_Chocolate_Factory_(film), http://dbpedia.org/resource/Miss_Peregrine's_Home_for_Peculiar_Children_(film), http://dbpedia.org/resource/Mars_Attacks!, http://dbpedia.org/resource/Batman_Returns, http://dbpedia.org/resource/Beetlejuice, http://dbpedia.org/resource/Ed_Wood_(film), http://dbpedia.org/resource/Edward_Scissorhands, http://dbpedia.org/resource/Pee-wee's_Big_Adventure, http://dbpedia.org/resource/Big_Fish, http://dbpedia.org/resource/Dark_Shadows_(film), http://dbpedia.org/resource/Frankenweenie_(2012_film), http://dbpedia.org/resource/Frankenweenie_(1984_film), http://dbpedia.org/resource/Vincent_(1982_film), http://dbpedia.org/resource/Big_Eyes, http://dbpedia.org/resource/Hansel_and_Gretel_(1982_film), http://dbpedia.org/resource/Stalk_of_the_Celery_Monster, http://dbpedia.org/resource/The_Island_of_Doctor_Agor] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 642, P: 0.517, R: 0.566, macro F1: 0.522, macro F1*: 0.540 -[INFO] Current time: 1327 ms, average: 2712.09 ms/ques, entity detection: 419.79 ms/ques (linking tool: 327.66 ms/ques), relation detection: 895.31 ms/ques (similarity: 703.14 ms/ques), EDG generation: 48.64 ms/ques +[INFO] Cumulative metrics, sample: 642, P: 0.517, R: 0.566, macro F1: 0.540 +[INFO] Current question consumed time: 1203 ms, average: 2459.88 ms/ques, entity detection: 374.05 ms/ques (linking tool: 295.13 ms/ques), relation detection: 831.58 ms/ques (similarity: 692.52 ms/ques), EDG generation: 48.48 ms/ques [INFO] Entity Linking P: 0.7538940809968847, R: 0.705607476635514, micro F1: 0.7209241952232616, macro F1: 0.7289520142830069 [INFO] Relation Linking P: 0.5956645898234683, R: 0.543613707165109, micro F1: 0.5582035306334375, macro F1: 0.5684501087343478 [INFO] Type Linking P: 0.5498442367601246, R: 0.5498442367601246, micro F1: 0.5498442367601246, macro F1: 0.5498442367601246 @@ -9018,8 +9018,8 @@ Question 642: Where was Live in Detroit, MI recorded ? [INFO] Golden Answer: [http://dbpedia.org/resource/Detroit, http://dbpedia.org/resource/Michigan, http://dbpedia.org/resource/Eastown_Theatre] [INFO] Predicted Answer: [Raw Power@en] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 643, P: 0.516, R: 0.565, macro F1: 0.521, macro F1*: 0.539 -[INFO] Current time: 1960 ms, average: 2710.92 ms/ques, entity detection: 419.98 ms/ques (linking tool: 327.93 ms/ques), relation detection: 894.21 ms/ques (similarity: 702.23 ms/ques), EDG generation: 48.59 ms/ques +[INFO] Cumulative metrics, sample: 643, P: 0.516, R: 0.565, macro F1: 0.539 +[INFO] Current question consumed time: 1682 ms, average: 2458.67 ms/ques, entity detection: 374.22 ms/ques (linking tool: 295.38 ms/ques), relation detection: 830.55 ms/ques (similarity: 691.62 ms/ques), EDG generation: 48.42 ms/ques [INFO] Entity Linking P: 0.7527216174183515, R: 0.7045101088646968, micro F1: 0.719803006739244, macro F1: 0.7278183408548842 [INFO] Relation Linking P: 0.5947382063245205, R: 0.5427682737169518, micro F1: 0.5573354069466048, macro F1: 0.5675660494672649 [INFO] Type Linking P: 0.5505443234836703, R: 0.5505443234836703, micro F1: 0.5505443234836703, macro F1: 0.5505443234836703 @@ -9032,8 +9032,8 @@ Question 643: Name all the bands which have members of QDT in them. [INFO] Golden Answer: [http://dbpedia.org/resource/Girls'_Generation, http://dbpedia.org/resource/F(x)_(band), http://dbpedia.org/resource/Kool_Moe_Dee, http://dbpedia.org/resource/Michael_Jackson, http://dbpedia.org/resource/SWV, http://dbpedia.org/resource/MC_Hammer, http://dbpedia.org/resource/Exo_(band), http://dbpedia.org/resource/Guy_(band), http://dbpedia.org/resource/Jay_Park, http://dbpedia.org/resource/Shinee, http://dbpedia.org/resource/Snoop_Dogg, http://dbpedia.org/resource/Blackstreet, http://dbpedia.org/resource/Bobby_Brown, http://dbpedia.org/resource/Doug_E._Fresh, http://dbpedia.org/resource/Heavy_D, http://dbpedia.org/resource/Today_(group), http://dbpedia.org/resource/Rania_(band), http://dbpedia.org/resource/Patti_LaBelle, http://dbpedia.org/resource/Dr._Dre, http://dbpedia.org/resource/2nd_II_None, http://dbpedia.org/resource/Tupac_Shakur, http://dbpedia.org/resource/Kurupt, http://dbpedia.org/resource/QDT, http://dbpedia.org/resource/The_Fixxers, http://dbpedia.org/resource/AMG_(rapper)] [INFO] Predicted Answer: [http://dbpedia.org/resource/Teddy_Riley, http://dbpedia.org/resource/Snoop_Dogg, http://dbpedia.org/resource/DJ_Quik] [INFO] P: 0.3333333333333333, R: 0.04, F: 0.07142857142857142 -[INFO] Cumulative metrics, sample: 644, P: 0.516, R: 0.564, macro F1: 0.521, macro F1*: 0.539 -[INFO] Current time: 736 ms, average: 2707.86 ms/ques, entity detection: 419.39 ms/ques (linking tool: 327.46 ms/ques), relation detection: 893.16 ms/ques (similarity: 701.47 ms/ques), EDG generation: 48.55 ms/ques +[INFO] Cumulative metrics, sample: 644, P: 0.516, R: 0.564, macro F1: 0.539 +[INFO] Current question consumed time: 630 ms, average: 2455.83 ms/ques, entity detection: 373.68 ms/ques (linking tool: 294.95 ms/ques), relation detection: 829.56 ms/ques (similarity: 690.87 ms/ques), EDG generation: 48.38 ms/ques [INFO] Entity Linking P: 0.7531055900621118, R: 0.7049689440993789, micro F1: 0.7202380952380962, macro F1: 0.728242679208091 [INFO] Relation Linking P: 0.5953674948240165, R: 0.5427018633540373, micro F1: 0.5575051759834373, macro F1: 0.5678160939833827 [INFO] Type Linking P: 0.5512422360248447, R: 0.5512422360248447, micro F1: 0.5512422360248447, macro F1: 0.5512422360248447 @@ -9046,8 +9046,8 @@ Question 644: Which footballers did their high school in west Sacramento califor [INFO] Golden Answer: [http://dbpedia.org/resource/Malcom_Floyd] [INFO] Predicted Answer: [http://dbpedia.org/resource/Malcom_Floyd] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 645, P: 0.516, R: 0.565, macro F1: 0.521, macro F1*: 0.540 -[INFO] Current time: 3991 ms, average: 2709.84 ms/ques, entity detection: 419.39 ms/ques (linking tool: 327.49 ms/ques), relation detection: 894.24 ms/ques (similarity: 702.94 ms/ques), EDG generation: 48.50 ms/ques +[INFO] Cumulative metrics, sample: 645, P: 0.516, R: 0.565, macro F1: 0.540 +[INFO] Current question consumed time: 3526 ms, average: 2457.49 ms/ques, entity detection: 373.66 ms/ques (linking tool: 294.96 ms/ques), relation detection: 830.47 ms/ques (similarity: 692.38 ms/ques), EDG generation: 48.34 ms/ques [INFO] Entity Linking P: 0.7534883720930232, R: 0.7054263565891473, micro F1: 0.7206718346253239, macro F1: 0.7286656945851765 [INFO] Relation Linking P: 0.5959948320413436, R: 0.5434108527131783, micro F1: 0.5581912144702846, macro F1: 0.5684894576632026 [INFO] Type Linking P: 0.5503875968992248, R: 0.5503875968992248, micro F1: 0.5503875968992248, macro F1: 0.5503875968992248 @@ -9060,8 +9060,8 @@ Question 645: List the names of the soccer club seasons whose division is 2012 C [INFO] Golden Answer: [http://dbpedia.org/resource/Allan_(footballer,_born_1991), http://dbpedia.org/resource/Fabrício_Silva_Dornellas, http://dbpedia.org/resource/Leandro_Chaparro, http://dbpedia.org/resource/Rodolfo_Dantas_Bispo, http://dbpedia.org/resource/Alecsandro, http://dbpedia.org/resource/Diego_Souza_(footballer,_born_1985), http://dbpedia.org/resource/Diogo_Silva_(footballer), http://dbpedia.org/resource/Fagner_Conserva_Lemos, http://dbpedia.org/resource/Fernando_Prass, http://dbpedia.org/resource/Nílton_Ferreira_Junior, http://dbpedia.org/resource/William_Barbio, http://dbpedia.org/resource/Éder_Luís, http://dbpedia.org/resource/Carlos_Alberto_de_Jesus, http://dbpedia.org/resource/Felipe_Jorge_Loureiro, http://dbpedia.org/resource/Rômulo_Borges_Monteiro, http://dbpedia.org/resource/Dedé_(footballer,_born_1988), http://dbpedia.org/resource/Juninho_Pernambucano, http://dbpedia.org/resource/Douglas_Silva_Bacelar, http://dbpedia.org/resource/Eduardo_Costa, http://dbpedia.org/resource/Thiago_Feltri, http://dbpedia.org/resource/Fellipe_Bastos, http://dbpedia.org/resource/Renato_Silva, http://dbpedia.org/resource/Alessandro_Felipe_Oltramari, http://dbpedia.org/resource/Diego_Rosa_(footballer), http://dbpedia.org/resource/Matías_Abelairas, http://dbpedia.org/resource/Elano, http://dbpedia.org/resource/Jorge_Fucile, http://dbpedia.org/resource/Neymar, http://dbpedia.org/resource/Paulo_Henrique_Ganso, http://dbpedia.org/resource/Wesley_Douglas, http://dbpedia.org/resource/Felipe_Anderson, http://dbpedia.org/resource/Alan_Kardec, http://dbpedia.org/resource/André_Felipe_Ribeiro_de_Souza, http://dbpedia.org/resource/Arouca_(footballer), http://dbpedia.org/resource/David_Braz, http://dbpedia.org/resource/Edu_Dracena, http://dbpedia.org/resource/Humberlito_Borges, http://dbpedia.org/resource/Leandrinho_(footballer), http://dbpedia.org/resource/Rafael_Cabral, http://dbpedia.org/resource/Vladimir_Orlando_Cardoso_de_Araújo_Filho, http://dbpedia.org/resource/Bill_(footballer), http://dbpedia.org/resource/Juan_Maldonado_Jaimez, http://dbpedia.org/resource/Anderson_Carvalho, http://dbpedia.org/resource/Victor_Andrade, http://dbpedia.org/resource/Bruno_Rodrigo, http://dbpedia.org/resource/Henrique_Pacheco_Lima, http://dbpedia.org/resource/Crystian, http://dbpedia.org/resource/Patricio_Rodríguez, http://dbpedia.org/resource/Gérson_Magrão, http://dbpedia.org/resource/Dimba_(footballer,_born_1992), http://dbpedia.org/resource/Wason_Rentería, http://dbpedia.org/resource/Ezequiel_Miralles, http://dbpedia.org/resource/Léo_(footballer,_born_1975), http://dbpedia.org/resource/Durval, http://dbpedia.org/resource/Ewerton_Páscoa, http://dbpedia.org/resource/Bruno_Peres, http://dbpedia.org/resource/João_Pedro_(Brazilian_footballer), http://dbpedia.org/resource/Aranha_(footballer), http://dbpedia.org/resource/Adriano_Bispo, Ewerton Páscoa1@en, Gérson Magrão1@en, Maranhão1@en, http://dbpedia.org/resource/Luiz_Otávio_Santos_de_Araújo, http://dbpedia.org/resource/Jorge_Valdivia, http://dbpedia.org/resource/Luan_Michel_de_Louzã, http://dbpedia.org/resource/Marcos_Assunção, http://dbpedia.org/resource/Thiago_Heleno, http://dbpedia.org/resource/Tiago_Real, http://dbpedia.org/resource/Vinícius_Santos_Silva, http://dbpedia.org/resource/Wellington_da_Silva_Pinto, http://dbpedia.org/resource/Wesley_Lopes_Beltrame, http://dbpedia.org/resource/Carlos_Alberto_Santos_da_Silva, http://dbpedia.org/resource/Maurício_Ramos, http://dbpedia.org/resource/Adalberto_Román, http://dbpedia.org/resource/Obina, http://dbpedia.org/resource/Maikon_Leite, http://dbpedia.org/resource/Patrik_Silva, http://dbpedia.org/resource/Hernán_Barcos, http://dbpedia.org/resource/Leandro_Amaro, http://dbpedia.org/resource/Daniel_Lovinho, http://dbpedia.org/resource/Henrique_(footballer,_born_1986), http://dbpedia.org/resource/Leandro_Silva_Wanderley, http://dbpedia.org/resource/Bruno_Cortez_Cardoso, http://dbpedia.org/resource/Daniel_Carvalho, http://dbpedia.org/resource/Artur_Barbosa_de_Oliveira, http://dbpedia.org/resource/Carlos_Rodrigues_Corrêa, http://dbpedia.org/resource/Evanildo_Borges_Barbosa_Júnior, http://dbpedia.org/resource/João_Vitor_Lima_Gomes, http://dbpedia.org/resource/Márcio_Araújo_(footballer), http://dbpedia.org/resource/José_Tadeu_Mouro_Júnior, http://dbpedia.org/resource/Fabinho_Capixaba, http://dbpedia.org/resource/Mazinho_(footballer,_born_1987), http://dbpedia.org/resource/Fernando_Galhardo_Borges, Carlos@en, Emerson@en, Luiz Gustavo@en, Patrick Viera@en, Caio@en, Fábio@en, João Denoni@en, Pegorari@en, Raphael Alemão@en, http://dbpedia.org/resource/Júlio_César_(footballer,_born_1984), http://dbpedia.org/resource/Romarinho_(footballer,_born_1990), http://dbpedia.org/resource/Marquinhos, http://dbpedia.org/resource/Adilson_dos_Santos_Souza, http://dbpedia.org/resource/Alessandro_Mori_Nunes, http://dbpedia.org/resource/Antônio_Carlos_Cunha_Capocasali_Júnior, http://dbpedia.org/resource/Danilo_Gabriel_de_Andrade, http://dbpedia.org/resource/Emerson_Sheik, http://dbpedia.org/resource/Felipe_Augusto_de_Almeida_Monteiro, http://dbpedia.org/resource/Giovanni_Piccolomo, http://dbpedia.org/resource/Paulo_André_Cren_Benini, http://dbpedia.org/resource/Wallace_Reis_Silva, http://dbpedia.org/resource/Weldinho, http://dbpedia.org/resource/Willian_Arão, http://dbpedia.org/resource/Paolo_Guerrero, http://dbpedia.org/resource/Paulinho_(footballer), http://dbpedia.org/resource/Juan_Manuel_Martínez, http://dbpedia.org/resource/Fábio_Santos_Romeu, http://dbpedia.org/resource/Douglas_dos_Santos, http://dbpedia.org/resource/Jorge_Henrique_de_Souza, http://dbpedia.org/resource/Élton_Rodrigues_Brandão, http://dbpedia.org/resource/Anderson_Sebastião_Cardoso, http://dbpedia.org/resource/Denner, http://dbpedia.org/resource/Ralf_(footballer), http://dbpedia.org/resource/Luis_Alberto_Ramírez, http://dbpedia.org/resource/Cássio_Ramos, http://dbpedia.org/resource/Edenílson_Andrade_dos_Santos, http://dbpedia.org/resource/Danilo_Fernandes, http://dbpedia.org/resource/Chen_Zhizhao, http://dbpedia.org/resource/Guilherme_Andrade, http://dbpedia.org/resource/Vitor_Saba, http://dbpedia.org/resource/Everton_Silva, http://dbpedia.org/resource/Thomás_Jaguaribe, http://dbpedia.org/resource/Arthur_Sanches, http://dbpedia.org/resource/Jorge_Luiz_Maciel_Vidal, http://dbpedia.org/resource/Lucas_Quintino, http://dbpedia.org/resource/Thiago_de_Medeiros_Godilho, http://dbpedia.org/resource/Wellington_Bruno, http://dbpedia.org/resource/Adriano_(footballer,_born_1982), http://dbpedia.org/resource/Claudio_Maldonado, http://dbpedia.org/resource/Hernane, http://dbpedia.org/resource/Adryan, http://dbpedia.org/resource/Airton_Ribeiro_Santos, http://dbpedia.org/resource/Egídio_Pereira_Júnior, http://dbpedia.org/resource/Ibson, http://dbpedia.org/resource/Luiz_Antônio_de_Souza_Soares, http://dbpedia.org/resource/Marcelo_Lomba, http://dbpedia.org/resource/Mattheus_Oliveira, http://dbpedia.org/resource/Muralha, http://dbpedia.org/resource/Nixon_Darlanio_Reis_Cardoso, http://dbpedia.org/resource/Paulo_Victor_Vidotti, http://dbpedia.org/resource/Rafael_Lima_Pereira, http://dbpedia.org/resource/Rodrigo_Frauches, http://dbpedia.org/resource/Willians_Domingos_Fernandes, http://dbpedia.org/resource/Thiago_Neves, http://dbpedia.org/resource/Antônio_Carlos_Rodrigues_dos_Santos_Júnior, http://dbpedia.org/resource/Magno_de_Andrade, http://dbpedia.org/resource/Wellington_Silva_(footballer,_born_1988), http://dbpedia.org/resource/Guilherme_Negueba, http://dbpedia.org/resource/Ronaldinho, http://dbpedia.org/resource/Felipe_Ventura_dos_Santos, http://dbpedia.org/resource/Marllon_Borges, http://dbpedia.org/resource/Felipe_Dias, http://dbpedia.org/resource/José_Kléberson, http://dbpedia.org/resource/Renato_Santos, http://dbpedia.org/resource/Vágner_Love, http://dbpedia.org/resource/Paulo_Sérgio_Luiz_de_Souza, http://dbpedia.org/resource/Diego_Maurício, http://dbpedia.org/resource/Welinton, http://dbpedia.org/resource/Deivid, http://dbpedia.org/resource/Mauricio_Azevedo_Alves, http://dbpedia.org/resource/Vander_Luiz_Silva_Souza, http://dbpedia.org/resource/Víctor_Cáceres, http://dbpedia.org/resource/Ramon_Motta, http://dbpedia.org/resource/Lucas_Galdino, http://dbpedia.org/resource/Fabiano_Oliveira, http://dbpedia.org/resource/Caio_Quiroga, http://dbpedia.org/resource/César_Bernardo_Dutra, http://dbpedia.org/resource/Michel_Lorran, http://dbpedia.org/resource/Rodrigo_Longo, http://dbpedia.org/resource/Romário_Reginaldo_Alves, http://dbpedia.org/resource/Ygor_Lopes_Silvestre, http://dbpedia.org/resource/Liédson, http://dbpedia.org/resource/Marcos_González, http://dbpedia.org/resource/Darío_Bottinelli, http://dbpedia.org/resource/Alex_Sandro_da_Silva, http://dbpedia.org/resource/Rômulo_Noronha, http://dbpedia.org/resource/Cléber_Santana, http://dbpedia.org/resource/Rafael_Galhardo, http://dbpedia.org/resource/João_Felipe_Antunes, http://dbpedia.org/resource/Itamar_Batista_da_Silva, http://dbpedia.org/resource/Erick_Flores, http://dbpedia.org/resource/Júnior_César, http://dbpedia.org/resource/Léo_Moura, http://dbpedia.org/resource/Renato_Abreu, http://dbpedia.org/resource/Jael_Ferreira, http://dbpedia.org/resource/Guilherme_Camacho, http://dbpedia.org/resource/Marlon_Ventura_Rodrigues, http://dbpedia.org/resource/Fernando_Gomes_de_Jesus, http://dbpedia.org/resource/Rodrigo_Alvim, http://dbpedia.org/resource/João_Vitor_Ribeiro_Rodrigues, http://dbpedia.org/resource/Gonzalo_Fierro, http://dbpedia.org/resource/Gustavo_Silva_Conceição, http://dbpedia.org/resource/Marcelo_Carné, http://dbpedia.org/resource/Ronaldo_Angelim, Jael@en, Marllon@en, João Felipe@en] [INFO] Predicted Answer: [2012] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 646, P: 0.516, R: 0.564, macro F1: 0.521, macro F1*: 0.539 -[INFO] Current time: 1197 ms, average: 2707.50 ms/ques, entity detection: 418.98 ms/ques (linking tool: 327.15 ms/ques), relation detection: 893.40 ms/ques (similarity: 702.41 ms/ques), EDG generation: 48.46 ms/ques +[INFO] Cumulative metrics, sample: 646, P: 0.516, R: 0.564, macro F1: 0.539 +[INFO] Current question consumed time: 1046 ms, average: 2455.31 ms/ques, entity detection: 373.26 ms/ques (linking tool: 294.62 ms/ques), relation detection: 829.68 ms/ques (similarity: 691.87 ms/ques), EDG generation: 48.32 ms/ques [INFO] Entity Linking P: 0.7523219814241486, R: 0.7043343653250774, micro F1: 0.7195562435500525, macro F1: 0.7275377291136826 [INFO] Relation Linking P: 0.5950722394220845, R: 0.5425696594427245, micro F1: 0.5573271413828693, macro F1: 0.567609443022857 [INFO] Type Linking P: 0.5495356037151703, R: 0.5495356037151703, micro F1: 0.5495356037151703, macro F1: 0.5495356037151703 @@ -9074,8 +9074,8 @@ Question 646: Which successor of Philippine Ports Authority is the state of orig [INFO] Golden Answer: [http://dbpedia.org/resource/Philippines] [INFO] Predicted Answer: [http://dbpedia.org/resource/Philippines] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 647, P: 0.516, R: 0.565, macro F1: 0.521, macro F1*: 0.539 -[INFO] Current time: 1563 ms, average: 2705.73 ms/ques, entity detection: 418.96 ms/ques (linking tool: 326.97 ms/ques), relation detection: 892.62 ms/ques (similarity: 701.69 ms/ques), EDG generation: 48.42 ms/ques +[INFO] Cumulative metrics, sample: 647, P: 0.516, R: 0.565, macro F1: 0.539 +[INFO] Current question consumed time: 1295 ms, average: 2453.51 ms/ques, entity detection: 373.17 ms/ques (linking tool: 294.43 ms/ques), relation detection: 828.93 ms/ques (similarity: 691.15 ms/ques), EDG generation: 48.28 ms/ques [INFO] Entity Linking P: 0.7527047913446677, R: 0.7047913446676971, micro F1: 0.7199896960329736, macro F1: 0.727960519306826 [INFO] Relation Linking P: 0.5956980937661, R: 0.5432766615146831, micro F1: 0.5580113343637304, macro F1: 0.5682810266889992 [INFO] Type Linking P: 0.5502318392581144, R: 0.5502318392581144, micro F1: 0.5502318392581144, macro F1: 0.5502318392581144 @@ -9088,8 +9088,8 @@ Question 647: What awards have been won by the executive producer of Timmy Time [INFO] Golden Answer: [http://dbpedia.org/resource/Academy_Awards, http://dbpedia.org/resource/Order_of_the_British_Empire, http://dbpedia.org/resource/Royal_Designers_for_Industry] [INFO] Predicted Answer: [http://dbpedia.org/resource/Academy_Awards, http://dbpedia.org/resource/Order_of_the_British_Empire, http://dbpedia.org/resource/Royal_Designers_for_Industry] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 648, P: 0.517, R: 0.565, macro F1: 0.522, macro F1*: 0.540 -[INFO] Current time: 2073 ms, average: 2704.76 ms/ques, entity detection: 418.65 ms/ques (linking tool: 326.65 ms/ques), relation detection: 892.46 ms/ques (similarity: 701.64 ms/ques), EDG generation: 48.48 ms/ques +[INFO] Cumulative metrics, sample: 648, P: 0.517, R: 0.565, macro F1: 0.540 +[INFO] Current question consumed time: 1795 ms, average: 2452.50 ms/ques, entity detection: 372.85 ms/ques (linking tool: 294.11 ms/ques), relation detection: 828.76 ms/ques (similarity: 691.11 ms/ques), EDG generation: 48.35 ms/ques [INFO] Entity Linking P: 0.7530864197530864, R: 0.7052469135802469, micro F1: 0.7204218106995894, macro F1: 0.7283819975177999 [INFO] Relation Linking P: 0.5955504115226337, R: 0.5432098765432098, micro F1: 0.5579218106995889, macro F1: 0.5681772870178666 [INFO] Type Linking P: 0.5493827160493827, R: 0.5493827160493827, micro F1: 0.5493827160493827, macro F1: 0.5493827160493827 @@ -9102,8 +9102,8 @@ Question 648: Which american football team is owned by Mark davis and is located [INFO] Golden Answer: [http://dbpedia.org/resource/Oakland_Raiders] [INFO] Predicted Answer: [11] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 649, P: 0.516, R: 0.565, macro F1: 0.521, macro F1*: 0.539 -[INFO] Current time: 6913 ms, average: 2711.24 ms/ques, entity detection: 419.38 ms/ques (linking tool: 327.25 ms/ques), relation detection: 895.45 ms/ques (similarity: 702.57 ms/ques), EDG generation: 48.48 ms/ques +[INFO] Cumulative metrics, sample: 649, P: 0.516, R: 0.565, macro F1: 0.539 +[INFO] Current question consumed time: 5671 ms, average: 2457.46 ms/ques, entity detection: 373.32 ms/ques (linking tool: 294.51 ms/ques), relation detection: 831.31 ms/ques (similarity: 692.04 ms/ques), EDG generation: 48.35 ms/ques [INFO] Entity Linking P: 0.7519260400616333, R: 0.7041602465331279, micro F1: 0.719311761684644, macro F1: 0.7272596831918867 [INFO] Relation Linking P: 0.5946327683615819, R: 0.5423728813559322, micro F1: 0.5570621468926558, macro F1: 0.5673018212443415 [INFO] Type Linking P: 0.5485362095531587, R: 0.5485362095531587, micro F1: 0.5485362095531587, macro F1: 0.5485362095531587 @@ -9116,8 +9116,8 @@ Question 649: The successor of Ulrich II, Count of East Frisia was born in what [INFO] Golden Answer: [http://dbpedia.org/resource/Germany, http://dbpedia.org/resource/Aurich] [INFO] Predicted Answer: [http://dbpedia.org/resource/Großes_Meer] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 650, P: 0.515, R: 0.564, macro F1: 0.520, macro F1*: 0.538 -[INFO] Current time: 1929 ms, average: 2710.04 ms/ques, entity detection: 419.33 ms/ques (linking tool: 327.17 ms/ques), relation detection: 894.77 ms/ques (similarity: 702.24 ms/ques), EDG generation: 48.48 ms/ques +[INFO] Cumulative metrics, sample: 650, P: 0.515, R: 0.564, macro F1: 0.538 +[INFO] Current question consumed time: 1634 ms, average: 2456.19 ms/ques, entity detection: 373.23 ms/ques (linking tool: 294.41 ms/ques), relation detection: 830.67 ms/ques (similarity: 691.72 ms/ques), EDG generation: 48.34 ms/ques [INFO] Entity Linking P: 0.7507692307692307, R: 0.703076923076923, micro F1: 0.7182051282051292, macro F1: 0.7261408221408222 [INFO] Relation Linking P: 0.5937179487179487, R: 0.5415384615384615, micro F1: 0.5562051282051286, macro F1: 0.5664290492116579 [INFO] Type Linking P: 0.5476923076923077, R: 0.5476923076923077, micro F1: 0.5476923076923077, macro F1: 0.5476923076923077 @@ -9130,8 +9130,8 @@ Question 650: Who is the prime minister of Michael Jeffery who is also the minis [INFO] Golden Answer: [http://dbpedia.org/resource/John_Howard] [INFO] Predicted Answer: [http://dbpedia.org/resource/John_Howard, http://dbpedia.org/resource/Kevin_Rudd] [INFO] P: 0.5, R: 1.0, F: 0.6666666666666666 -[INFO] Cumulative metrics, sample: 651, P: 0.515, R: 0.564, macro F1: 0.521, macro F1*: 0.539 -[INFO] Current time: 3135 ms, average: 2710.69 ms/ques, entity detection: 419.31 ms/ques (linking tool: 327.00 ms/ques), relation detection: 895.26 ms/ques (similarity: 702.20 ms/ques), EDG generation: 48.51 ms/ques +[INFO] Cumulative metrics, sample: 651, P: 0.515, R: 0.564, macro F1: 0.539 +[INFO] Current question consumed time: 2621 ms, average: 2456.44 ms/ques, entity detection: 373.14 ms/ques (linking tool: 294.22 ms/ques), relation detection: 831.06 ms/ques (similarity: 691.69 ms/ques), EDG generation: 48.41 ms/ques [INFO] Entity Linking P: 0.7511520737327189, R: 0.7027649769585254, micro F1: 0.7181259600614449, macro F1: 0.7261533517859882 [INFO] Relation Linking P: 0.594342037890425, R: 0.5414746543778802, micro F1: 0.5563748079877117, macro F1: 0.5666779714361542 [INFO] Type Linking P: 0.5483870967741935, R: 0.5483870967741935, micro F1: 0.5483870967741935, macro F1: 0.5483870967741935 @@ -9144,8 +9144,8 @@ Question 651: Who is the owner of the FC which owns the Cobham Training Centre? [INFO] Golden Answer: [http://dbpedia.org/resource/Roman_Abramovich] [INFO] Predicted Answer: [http://dbpedia.org/resource/Cobham_Training_Centre, http://dbpedia.org/resource/Roman_Abramovich, http://dbpedia.org/resource/Chelsea_TV] [INFO] P: 0.3333333333333333, R: 1.0, F: 0.5 -[INFO] Cumulative metrics, sample: 652, P: 0.515, R: 0.565, macro F1: 0.521, macro F1*: 0.539 -[INFO] Current time: 1916 ms, average: 2709.47 ms/ques, entity detection: 419.10 ms/ques (linking tool: 326.78 ms/ques), relation detection: 895.16 ms/ques (similarity: 701.79 ms/ques), EDG generation: 48.47 ms/ques +[INFO] Cumulative metrics, sample: 652, P: 0.515, R: 0.565, macro F1: 0.539 +[INFO] Current question consumed time: 1671 ms, average: 2455.24 ms/ques, entity detection: 372.93 ms/ques (linking tool: 294.01 ms/ques), relation detection: 830.96 ms/ques (similarity: 691.28 ms/ques), EDG generation: 48.37 ms/ques [INFO] Entity Linking P: 0.7515337423312883, R: 0.7032208588957055, micro F1: 0.71855828220859, macro F1: 0.7265750571616145 [INFO] Relation Linking P: 0.5949642126789365, R: 0.5421779141104295, micro F1: 0.5570552147239268, macro F1: 0.5673458896670898 [INFO] Type Linking P: 0.5475460122699386, R: 0.5475460122699386, micro F1: 0.5475460122699386, macro F1: 0.5475460122699386 @@ -9158,8 +9158,8 @@ Question 652: Who is considered to be the deciding person when it comes to crust [INFO] Golden Answer: [http://dbpedia.org/resource/10th_edition_of_Systema_Naturae, http://dbpedia.org/resource/Carl_Linnaeus, http://dbpedia.org/resource/Jean_Guillaume_Bruguière, http://dbpedia.org/resource/Charles_Darwin, http://dbpedia.org/resource/Henry_Augustus_Pilsbry, http://dbpedia.org/resource/Augustus_Addison_Gould, http://dbpedia.org/resource/Juan_Ignacio_Molina] [INFO] Predicted Answer: [http://dbpedia.org/resource/Balanus_balanus, http://dbpedia.org/resource/Balanus_crenatus, http://dbpedia.org/resource/Balanus_glandula, http://dbpedia.org/resource/Balanus_nubilus, http://dbpedia.org/resource/Balanus_perforatus, http://dbpedia.org/resource/Balanus_trigonus, http://dbpedia.org/resource/Fistulobalanus_albicostatus, http://dbpedia.org/resource/Balanus_aquila, http://dbpedia.org/resource/Bay_barnacle, http://dbpedia.org/resource/Ivory_barnacle, http://dbpedia.org/resource/Balanus, http://dbpedia.org/resource/Megabalanus_coccopoma, http://dbpedia.org/resource/Megabalanus_concinnus, http://dbpedia.org/resource/Megabalanus_stultus, http://dbpedia.org/resource/Megabalanus_tintinnabulum, http://dbpedia.org/resource/Amphibalanus_amphitrite, http://dbpedia.org/resource/Amphibalanus, http://dbpedia.org/resource/Austromegabalanus, http://dbpedia.org/resource/Concavus, http://dbpedia.org/resource/Elminius_modestus, http://dbpedia.org/resource/Fistulobalanus, http://dbpedia.org/resource/Menesiniella, http://dbpedia.org/resource/Austromegabalanus_psittacus, http://dbpedia.org/resource/Megabalanus, http://dbpedia.org/resource/Balanoidea] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 653, P: 0.514, R: 0.564, macro F1: 0.520, macro F1*: 0.538 -[INFO] Current time: 649 ms, average: 2706.32 ms/ques, entity detection: 418.80 ms/ques (linking tool: 326.60 ms/ques), relation detection: 894.06 ms/ques (similarity: 700.88 ms/ques), EDG generation: 48.45 ms/ques +[INFO] Cumulative metrics, sample: 653, P: 0.514, R: 0.564, macro F1: 0.538 +[INFO] Current question consumed time: 564 ms, average: 2452.34 ms/ques, entity detection: 372.65 ms/ques (linking tool: 293.84 ms/ques), relation detection: 829.92 ms/ques (similarity: 690.39 ms/ques), EDG generation: 48.35 ms/ques [INFO] Entity Linking P: 0.7519142419601837, R: 0.7036753445635529, micro F1: 0.718989280245024, macro F1: 0.7269954638205249 [INFO] Relation Linking P: 0.5955844818785094, R: 0.5421133231240429, micro F1: 0.5572230729964271, macro F1: 0.5675923452652627 [INFO] Type Linking P: 0.5467075038284839, R: 0.5467075038284839, micro F1: 0.5467075038284839, macro F1: 0.5467075038284839 @@ -9172,8 +9172,8 @@ Question 653: What have some US citizens died of? [INFO] Golden Answer: [http://dbpedia.org/resource/Blunt_trauma, http://dbpedia.org/resource/Blunt_force_trauma, http://dbpedia.org/resource/Pick's_disease, http://dbpedia.org/resource/Stroke, http://dbpedia.org/resource/Myocardial_infarction, http://dbpedia.org/resource/Multiple_myeloma, http://dbpedia.org/resource/Bone_tumor, http://dbpedia.org/resource/Murder, http://dbpedia.org/resource/Respiratory_failure, http://dbpedia.org/resource/Cancer, http://dbpedia.org/resource/Traumatic_asphyxia, http://dbpedia.org/resource/Apoplexy, http://dbpedia.org/resource/Drowning, http://dbpedia.org/resource/Cholera, http://dbpedia.org/resource/Lung_cancer, http://dbpedia.org/resource/Diabetic_ketoacidosis, http://dbpedia.org/resource/Liver_cancer, http://dbpedia.org/resource/Appendicitis, http://dbpedia.org/resource/Coronary_thrombosis, http://dbpedia.org/resource/Nephritis, http://dbpedia.org/resource/Homicide] [INFO] Predicted Answer: [http://dbpedia.org/resource/London] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 654, P: 0.514, R: 0.563, macro F1: 0.519, macro F1*: 0.537 -[INFO] Current time: 11044 ms, average: 2719.07 ms/ques, entity detection: 433.78 ms/ques (linking tool: 341.70 ms/ques), relation detection: 893.34 ms/ques (similarity: 700.19 ms/ques), EDG generation: 48.40 ms/ques +[INFO] Cumulative metrics, sample: 654, P: 0.514, R: 0.563, macro F1: 0.537 +[INFO] Current question consumed time: 10050 ms, average: 2463.96 ms/ques, entity detection: 386.35 ms/ques (linking tool: 307.64 ms/ques), relation detection: 829.24 ms/ques (similarity: 689.72 ms/ques), EDG generation: 48.30 ms/ques [INFO] Entity Linking P: 0.7507645259938838, R: 0.7025993883792049, micro F1: 0.7178899082568817, macro F1: 0.725883849961472 [INFO] Relation Linking P: 0.5946738022426096, R: 0.5412844036697247, micro F1: 0.5563710499490321, macro F1: 0.566724467061493 [INFO] Type Linking P: 0.5474006116207951, R: 0.5474006116207951, micro F1: 0.5474006116207951, macro F1: 0.5474006116207951 @@ -9186,8 +9186,8 @@ Question 654: What are some apes? [INFO] Golden Answer: [http://dbpedia.org/resource/Black_crested_gibbon, http://dbpedia.org/resource/Hylobates, http://dbpedia.org/resource/Northern_buffed-cheeked_gibbon, http://dbpedia.org/resource/Northern_white-cheeked_gibbon, http://dbpedia.org/resource/Southern_white-cheeked_gibbon, http://dbpedia.org/resource/Yellow-cheeked_gibbon, http://dbpedia.org/resource/Homininae, http://dbpedia.org/resource/Afropithecus, http://dbpedia.org/resource/Dendropithecus, http://dbpedia.org/resource/Dryopithecus, http://dbpedia.org/resource/Equatorius, http://dbpedia.org/resource/Heliopithecus, http://dbpedia.org/resource/Kamoyapithecus, http://dbpedia.org/resource/Kenyapithecus, http://dbpedia.org/resource/Nacholapithecus, http://dbpedia.org/resource/Oreopithecus, http://dbpedia.org/resource/Pierolapithecus, http://dbpedia.org/resource/Proconsul_(primate), http://dbpedia.org/resource/Propliopithecus, http://dbpedia.org/resource/Rangwapithecus, http://dbpedia.org/resource/Pliobates, http://dbpedia.org/resource/Proconsulidae, http://dbpedia.org/resource/Ugandapithecus_major, http://dbpedia.org/resource/Gorilla, http://dbpedia.org/resource/Ekembo, http://dbpedia.org/resource/Proconsul_major, http://dbpedia.org/resource/Ugandapithecus, http://dbpedia.org/resource/Homo_antecessor, http://dbpedia.org/resource/Dryopithecus_fontani, http://dbpedia.org/resource/Proconsul_nyanzae, http://dbpedia.org/resource/Human_taxonomy, http://dbpedia.org/resource/Pliopithecidae, http://dbpedia.org/resource/Gibbon, http://dbpedia.org/resource/Chororapithecus, http://dbpedia.org/resource/Pongidae, http://dbpedia.org/resource/Samburupithecus, http://dbpedia.org/resource/Proconsul_africanus, http://dbpedia.org/resource/Gibbon–human_last_common_ancestor, http://dbpedia.org/resource/Gorilla–human_last_common_ancestor, http://dbpedia.org/resource/Orangutan–human_last_common_ancestor, http://dbpedia.org/resource/Morotopithecus] [INFO] Predicted Answer: [http://dbpedia.org/resource/Blood_Moon_(album)] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 655, P: 0.513, R: 0.562, macro F1: 0.518, macro F1*: 0.536 -[INFO] Current time: 781 ms, average: 2716.11 ms/ques, entity detection: 433.29 ms/ques (linking tool: 341.25 ms/ques), relation detection: 892.38 ms/ques (similarity: 699.36 ms/ques), EDG generation: 48.33 ms/ques +[INFO] Cumulative metrics, sample: 655, P: 0.513, R: 0.562, macro F1: 0.536 +[INFO] Current question consumed time: 660 ms, average: 2461.21 ms/ques, entity detection: 385.89 ms/ques (linking tool: 307.23 ms/ques), relation detection: 828.35 ms/ques (similarity: 688.91 ms/ques), EDG generation: 48.23 ms/ques [INFO] Entity Linking P: 0.749618320610687, R: 0.7015267175572519, micro F1: 0.7167938931297719, macro F1: 0.7247756303432104 [INFO] Relation Linking P: 0.593765903307888, R: 0.5404580152671755, micro F1: 0.5555216284987282, macro F1: 0.5658592388675061 [INFO] Type Linking P: 0.5480916030534351, R: 0.5480916030534351, micro F1: 0.5480916030534351, macro F1: 0.5480916030534351 @@ -9200,8 +9200,8 @@ Question 655: Who is buried at the place governed by National Park Service? [INFO] Golden Answer: [http://dbpedia.org/resource/Ulysses_S._Grant, http://dbpedia.org/resource/Bibliography_of_Ulysses_S._Grant, http://dbpedia.org/resource/Franklin_D._Roosevelt, http://dbpedia.org/resource/Lyndon_B._Johnson, http://dbpedia.org/resource/Andrew_Johnson, http://dbpedia.org/resource/Albert_G._Riddle] [INFO] Predicted Answer: [http://dbpedia.org/resource/Long_Now_Foundation, http://dbpedia.org/resource/Massachusetts_Water_Resources_Authority, http://dbpedia.org/resource/USS_Constitution_Museum, http://dbpedia.org/resource/McCarthy_General_Store, http://dbpedia.org/resource/National_Gallery_of_Art, http://dbpedia.org/resource/National_Museum_of_Natural_History, http://dbpedia.org/resource/Washington_Monument, http://dbpedia.org/resource/Hirshhorn_Museum_and_Sculpture_Garden, http://dbpedia.org/resource/Vietnam_Women's_Memorial, http://dbpedia.org/resource/The_Three_Soldiers, http://dbpedia.org/resource/Smithsonian_Gardens, http://dbpedia.org/resource/Nisqually_Entrance_Historic_District, http://dbpedia.org/resource/Sunrise_Historic_District, http://dbpedia.org/resource/Second_Division_Memorial, http://dbpedia.org/resource/First_Division_Monument, http://dbpedia.org/resource/Butt-Millet_Memorial_Fountain, http://dbpedia.org/resource/Hay–Adams_Hotel, http://dbpedia.org/resource/Old_Point_Loma_Lighthouse, http://dbpedia.org/resource/Yosemite_Valley_Chapel, http://dbpedia.org/resource/Glacier_Point_Hotel, http://dbpedia.org/resource/Wawona_Tunnel, http://dbpedia.org/resource/Port_Kennedy_Bone_Cave, http://dbpedia.org/resource/Kennedy_Mansion_(Valley_Forge), http://dbpedia.org/resource/Washington's_Headquarters_(Valley_Forge), http://dbpedia.org/resource/Cuyahoga_County,_Ohio, http://dbpedia.org/resource/Ohio, http://dbpedia.org/resource/Valley_View,_Ohio, http://dbpedia.org/resource/Independence,_Ohio, http://dbpedia.org/resource/California, http://dbpedia.org/resource/San_Francisco, http://dbpedia.org/resource/New_York_City, http://dbpedia.org/resource/Morningside_Heights,_Manhattan, http://dbpedia.org/resource/Riverside_Drive_(Manhattan), http://dbpedia.org/resource/Quincy,_Massachusetts, http://dbpedia.org/resource/Boston, http://dbpedia.org/resource/Montana, http://dbpedia.org/resource/North_Dakota, http://dbpedia.org/resource/Roosevelt_County,_Montana, http://dbpedia.org/resource/Williams_County,_North_Dakota, http://dbpedia.org/resource/McKenzie_County,_North_Dakota, http://dbpedia.org/resource/Richland_County,_Montana, http://dbpedia.org/resource/United_States, http://dbpedia.org/resource/Powell_County,_Montana, http://dbpedia.org/resource/Stanton,_North_Dakota, http://dbpedia.org/resource/Adams_County,_Mississippi, http://dbpedia.org/resource/Mississippi, http://dbpedia.org/resource/Natchez,_Mississippi, http://dbpedia.org/resource/Louisiana, http://dbpedia.org/resource/New_Orleans, http://dbpedia.org/resource/New_Bedford,_Massachusetts, http://dbpedia.org/resource/Vicksburg,_Mississippi, http://dbpedia.org/resource/Arlington_County,_Virginia, http://dbpedia.org/resource/Potomac_River, http://dbpedia.org/resource/Washington,_D.C., http://dbpedia.org/resource/Cove_Neck,_New_York, http://dbpedia.org/resource/List_of_numbered_streets_in_Manhattan, http://dbpedia.org/resource/Park_County,_Wyoming, http://dbpedia.org/resource/Wyoming, http://dbpedia.org/resource/Yellowstone_National_Park, http://dbpedia.org/resource/Cincinnati, http://dbpedia.org/resource/Franklin_County,_Virginia, http://dbpedia.org/resource/Rome,_New_York, http://dbpedia.org/resource/Valdez–Cordova_Census_Area,_Alaska, http://dbpedia.org/resource/Appomattox_County,_Virginia, http://dbpedia.org/resource/Northwest_Arctic_Borough,_Alaska, http://dbpedia.org/resource/Oregon, http://dbpedia.org/resource/Crater_Lake_National_Park, http://dbpedia.org/resource/Hyde_Park,_New_York, http://dbpedia.org/resource/New_York, http://dbpedia.org/resource/John_Day_Fossil_Beds_National_Monument, http://dbpedia.org/resource/Oregon_Caves_National_Monument_and_Preserve, http://dbpedia.org/resource/Petrified_Forest_National_Park, http://dbpedia.org/resource/Arkansas, http://dbpedia.org/resource/Buffalo_National_River, http://dbpedia.org/resource/Newton_County,_Arkansas, http://dbpedia.org/resource/Northwest,_Washington,_D.C., http://dbpedia.org/resource/Macon,_Georgia, http://dbpedia.org/resource/U.S._Route_89, http://dbpedia.org/resource/Springdale,_Utah, http://dbpedia.org/resource/Massachusetts, http://dbpedia.org/resource/Lake_Chelan_National_Recreation_Area, http://dbpedia.org/resource/East_Glacier_Park_Village,_Montana, http://dbpedia.org/resource/St._Mary,_Montana, http://dbpedia.org/resource/Michigan, http://dbpedia.org/resource/Sleeping_Bear_Dunes_National_Lakeshore, http://dbpedia.org/resource/North_Manitou_Island, http://dbpedia.org/resource/El_Portal,_California, http://dbpedia.org/resource/Yosemite_National_Park, http://dbpedia.org/resource/Wawona,_California, http://dbpedia.org/resource/Moose,_Wyoming, http://dbpedia.org/resource/Georgia_(U.S._state), http://dbpedia.org/resource/Sumter_County,_Georgia, http://dbpedia.org/resource/Macon_County,_Georgia, http://dbpedia.org/resource/Philadelphia, http://dbpedia.org/resource/Indiana, http://dbpedia.org/resource/Lincoln_City,_Indiana, http://dbpedia.org/resource/Constitution_Avenue, http://dbpedia.org/resource/Independence_Avenue_(Washington,_D.C.), http://dbpedia.org/resource/Grand_Canyon, http://dbpedia.org/resource/Canton,_Ohio, http://dbpedia.org/resource/Oklahoma, http://dbpedia.org/resource/Oklahoma_City, http://dbpedia.org/resource/South_Boston, http://dbpedia.org/resource/Florida, http://dbpedia.org/resource/St._Augustine,_Florida, http://dbpedia.org/resource/Pennsylvania, http://dbpedia.org/resource/Blair_County,_Pennsylvania, http://dbpedia.org/resource/Cambria_County,_Pennsylvania, http://dbpedia.org/resource/Quadrants_of_Washington,_D.C., http://dbpedia.org/resource/Manatee_County,_Florida, http://dbpedia.org/resource/Jackson_Hole, http://dbpedia.org/resource/Teton_County,_Wyoming, http://dbpedia.org/resource/Grand_Teton_National_Park, http://dbpedia.org/resource/Danville,_California, http://dbpedia.org/resource/Fayette_County,_Pennsylvania, http://dbpedia.org/resource/Springhill_Township,_Fayette_County,_Pennsylvania, http://dbpedia.org/resource/Vincennes,_Indiana, http://dbpedia.org/resource/Louisa_County,_Virginia, http://dbpedia.org/resource/Hawaii_County,_Hawaii, http://dbpedia.org/resource/Moran,_Wyoming, http://dbpedia.org/resource/Martinez,_California, http://dbpedia.org/resource/Inyo_County,_California, http://dbpedia.org/resource/Columbia_County,_New_York, http://dbpedia.org/resource/Kinderhook_(town),_New_York, http://dbpedia.org/resource/M_Street_(Washington,_D.C.), http://dbpedia.org/resource/Richmond,_California, http://dbpedia.org/resource/New_Hampshire, http://dbpedia.org/resource/Cornish,_New_Hampshire, http://dbpedia.org/resource/Mount_Vernon,_New_York, http://dbpedia.org/resource/Essex_County,_Massachusetts, http://dbpedia.org/resource/Saugus,_Massachusetts, http://dbpedia.org/resource/Mastic_Beach,_New_York, http://dbpedia.org/resource/Tuolumne_Meadows, http://dbpedia.org/resource/Mount_Rainier_National_Park, http://dbpedia.org/resource/Grand_Canyon_National_Park, http://dbpedia.org/resource/Rock_Creek_(Potomac_River), http://dbpedia.org/resource/Greeneville,_Tennessee, http://dbpedia.org/resource/Clatsop_County,_Oregon, http://dbpedia.org/resource/Concord,_Massachusetts, http://dbpedia.org/resource/Manhattan, http://dbpedia.org/resource/Battery_Park, http://dbpedia.org/resource/Virginia, http://dbpedia.org/resource/Fairfax_County,_Virginia, http://dbpedia.org/resource/Walnut_Street_(Philadelphia), http://dbpedia.org/resource/Pike_County,_Pennsylvania, http://dbpedia.org/resource/Lackawaxen_Township,_Pike_County,_Pennsylvania, http://dbpedia.org/resource/Estes_Park,_Colorado, http://dbpedia.org/resource/Rocky_Mountain_National_Park, http://dbpedia.org/resource/Arizona, http://dbpedia.org/resource/Cochise_County,_Arizona, http://dbpedia.org/resource/Washington_(state), http://dbpedia.org/resource/Port_Kennedy,_Pennsylvania, http://dbpedia.org/resource/Valley_Forge_National_Historical_Park, http://dbpedia.org/resource/Colorado, http://dbpedia.org/resource/Larimer_County,_Colorado, http://dbpedia.org/resource/Assateague_Island_National_Seashore, http://dbpedia.org/resource/Chincoteague,_Virginia, http://dbpedia.org/resource/Charlestown,_Boston, http://dbpedia.org/resource/Otero_County,_Colorado, http://dbpedia.org/resource/Cumberland_Township,_Adams_County,_Pennsylvania, http://dbpedia.org/resource/Brooklyn, http://dbpedia.org/resource/Marine_Park, http://dbpedia.org/resource/Glacier_National_Park_(U.S.), http://dbpedia.org/resource/Glacier_County,_Montana, http://dbpedia.org/resource/Flathead_County,_Montana, http://dbpedia.org/resource/Coconino_County,_Arizona, http://dbpedia.org/resource/Lake_and_Peninsula_Borough,_Alaska, http://dbpedia.org/resource/Hardin_County,_Tennessee, http://dbpedia.org/resource/Beverly_Shores,_Indiana, http://dbpedia.org/resource/Paradise,_Washington, http://dbpedia.org/resource/Denali_National_Park_and_Preserve, http://dbpedia.org/resource/Alaska, http://dbpedia.org/resource/Vancouver,_Washington, http://dbpedia.org/resource/Atlanta, http://dbpedia.org/resource/Hawaii, http://dbpedia.org/resource/Molokai, http://dbpedia.org/resource/Kalaupapa,_Hawaii, http://dbpedia.org/resource/St._Johns_County,_Florida, http://dbpedia.org/resource/Cook_County,_Minnesota, http://dbpedia.org/resource/Sweet_Township,_Pipestone_County,_Minnesota, http://dbpedia.org/resource/Paterson,_New_Jersey, http://dbpedia.org/resource/Many_Glacier,_Montana, http://dbpedia.org/resource/Wallowa_County,_Oregon, http://dbpedia.org/resource/Stehekin,_Washington, http://dbpedia.org/resource/Port_Alsworth,_Alaska, http://dbpedia.org/resource/Lake_Clark_National_Park_and_Preserve, http://dbpedia.org/resource/Upper_Manhattan, http://dbpedia.org/resource/Montgomery_County,_Pennsylvania, http://dbpedia.org/resource/Chester_County,_Pennsylvania, http://dbpedia.org/resource/North_Carolina, http://dbpedia.org/resource/Flat_Rock,_Henderson_County,_North_Carolina, http://dbpedia.org/resource/Pawnee_County,_Kansas, http://dbpedia.org/resource/Fort_Scott,_Kansas, http://dbpedia.org/resource/Dare_County,_North_Carolina, http://dbpedia.org/resource/New_Mexico, http://dbpedia.org/resource/Catron_County,_New_Mexico, http://dbpedia.org/resource/Morristown,_New_Jersey, http://dbpedia.org/resource/West_Orange,_New_Jersey, http://dbpedia.org/resource/Germantown,_Philadelphia, http://dbpedia.org/resource/Rock_Creek_Park, http://dbpedia.org/resource/Hawaiʻi_Volcanoes_National_Park, http://dbpedia.org/resource/Chain_of_Craters_Road, http://dbpedia.org/resource/Manitou_Island_(Wisconsin), http://dbpedia.org/resource/Wisconsin, http://dbpedia.org/resource/Apostle_Islands_National_Lakeshore, http://dbpedia.org/resource/McLean_County,_Illinois, http://dbpedia.org/resource/Illinois, http://dbpedia.org/resource/Ellsworth,_Illinois, http://dbpedia.org/resource/West_Glacier,_Montana, http://dbpedia.org/resource/Brownsville,_Texas, http://dbpedia.org/resource/New_Castle,_Delaware, http://dbpedia.org/resource/Cape_Cod_National_Seashore, http://dbpedia.org/resource/Foundry_Branch, http://dbpedia.org/resource/Monroe_County,_Florida, http://dbpedia.org/resource/Great_Smoky_Mountains_National_Park, http://dbpedia.org/resource/Big_Bend_National_Park, http://dbpedia.org/resource/Alexandria,_Virginia, http://dbpedia.org/resource/West_Virginia, http://dbpedia.org/resource/Fayette_County,_West_Virginia, http://dbpedia.org/resource/U.S._Route_40_in_Utah, http://dbpedia.org/resource/Utah, http://dbpedia.org/resource/Dinosaur_National_Monument, http://dbpedia.org/resource/Texas, http://dbpedia.org/resource/Guadalupe_Mountains_National_Park, http://dbpedia.org/resource/Frederick,_Maryland, http://dbpedia.org/resource/Leelanau_Peninsula, http://dbpedia.org/resource/Yellowstone_Lake, http://dbpedia.org/resource/Chinook_Pass_Entrance,_Washington, http://dbpedia.org/resource/South_Carolina, http://dbpedia.org/resource/Cherokee_County,_South_Carolina, http://dbpedia.org/resource/York_County,_South_Carolina, http://dbpedia.org/resource/Chestnut_Street_(Philadelphia), http://dbpedia.org/resource/Mentor,_Ohio, http://dbpedia.org/resource/Jacksonville,_Florida, http://dbpedia.org/resource/Timucuan_Preserve, http://dbpedia.org/resource/Devils_Tower, http://dbpedia.org/resource/Bristol_Bay_Borough,_Alaska, http://dbpedia.org/resource/Katmai_National_Park_and_Preserve, http://dbpedia.org/resource/Gatlinburg,_Tennessee, http://dbpedia.org/resource/Wears_Valley,_Tennessee, http://dbpedia.org/resource/Kelly,_Wyoming, http://dbpedia.org/resource/Pioneer_Yosemite_History_Center, http://dbpedia.org/resource/Kelso,_California, http://dbpedia.org/resource/Bryce_Canyon_National_Park, http://dbpedia.org/resource/Kane_County,_Utah, http://dbpedia.org/resource/Teller_County,_Colorado, http://dbpedia.org/resource/Florissant_Fossil_Beds_National_Monument, http://dbpedia.org/resource/Port_Angeles,_Washington, http://dbpedia.org/resource/Ganado,_Arizona, http://dbpedia.org/resource/Tumacacori,_Arizona, http://dbpedia.org/resource/Oregon_City,_Oregon, http://dbpedia.org/resource/Anacapa_Island, http://dbpedia.org/resource/Honolulu, http://dbpedia.org/resource/Baltimore, http://dbpedia.org/resource/Inner_Harbor, http://dbpedia.org/resource/Live_Earth_concert,_Washington,_D.C., http://dbpedia.org/resource/List_of_public_art_in_Philadelphia, http://dbpedia.org/resource/Illilouette_Fall, http://dbpedia.org/resource/Lehamite_Falls, http://dbpedia.org/resource/Nevada_Fall, http://dbpedia.org/resource/Royal_Arch_Cascade, http://dbpedia.org/resource/Sentinel_Fall, http://dbpedia.org/resource/Silver_Strand_Falls, http://dbpedia.org/resource/Snow_Creek_Falls, http://dbpedia.org/resource/Staircase_Falls, http://dbpedia.org/resource/Vernal_Fall, http://dbpedia.org/resource/Waterwheel_Falls, http://dbpedia.org/resource/Bridalveil_Fall, http://dbpedia.org/resource/Yosemite_Falls, Independence and@en, Riverside Drive and West 122nd Street, Morningside Heights, Manhattan, New York City@en, 135, Southeast of Chelsea Street, Charlestown, Boston, Massachusetts@en, McKenzie and Williams counties, North Dakota, and Richland and Roosevelt counties, Montana@en, http://dbpedia.org/resource/Delta,_Louisiana, 28, New York, New York, U.S.@en, 2038, 112, 1, 3501, Between Utah State Route 9 and US 89, Springdale, Utah@en, N side Cut Bank Creek, Glacier NP, East Glacier, Montana@en, Glacier Rt. 3 at Apikuni Flat, Glacier NP, St. Mary, Montana@en, East Coast, North Manitou Island, Sleeping Bear Dunes National Lakeshore, Michigan@en, CA 140, El Portal, California@en, Yosemite National Park, Wawona, California@en, N of El Portal in Yosemite National Park, El Portal, California@en, Macon / Sumter counties, Georgia, USA@en, 120, IN 162, Lincoln City, Indiana, United States@en, West End of National Mall, Washington, D.C.@en, Between Independence and Constitution Avenues from the Capitol to the Lincoln Memorial@en, 10, 205, South Boston, Boston, Massachusetts@en, Roughly a one block area surrounding Telegraph Hill, South Boston, Massachusetts@en, 11, Blair and Cambria counties, Pennsylvania, United States@en, 6625, 15389, R Street, N.W., Washington, District of Columbia@en, Kuss Road, Danville, California@en, 2, Hawaii County, Hawaii, United States@en, Moran, Grand Teton National Park, Wyoming, USA 83013@en, 4202, 3051, Tilden Street and Beach Drive, N.W., Washington, District of Columbia, United States@en, Shipyards of Richmond, Richmond, California@en, Eastchester, Mount Vernon, New York@en, 244, 301, Fort Wadsworth Reservation, New York, New York@en, 20, Tuolumne Meadows, Yosemite National Park, California@en, Longmire, Mount Rainier National Park, Washington@en, Grand Canyon National Park, Grand Canyon, Arizona@en, Rock Creek and Potomac Pkwy. at 27th and L Sts., NW., Washington, District of Columbia@en, 420, 5, 143, between Walnut and 6th Sts., West and South Washington Square@en, Off TN 73 SW of Gatlinburg@en, Mount Rainier National Park@en, Sunrise , Mount Rainier National Park, Washington@en, 1050, Mt. Rainier National Park, Nisqually Entrance, Washington@en, White River Entrance, Mount Rainier National Park, Washington@en, Beach St., Assateague Island National Seashore, Chincoteague, Virginia vicinity@en, 25, Glacier National Park, Flathead / Glacier counties, Montana, USA@en, 215, Mt. Rainier National Park, Paradise, Washington@en, Off Rte. 8A, in Grand Canyon National Park, Grand Canyon, Arizona@en, Off AZ 64 in Grand Canyon National Park, Grand Canyon, Arizona@en, Tilden Street and Beach Drive, N.W., Washington, District of Columbia@en, Mt. Rainier National Park, Sunrise, Washington@en, Near Igloo Creek at Mile 34.1 N of Park Rd., Denali National Park and Preserve, Alaska@en, cross-country and north of Mile 25 on Park Road, Denali National Park and Preserve, Alaska, USA@en, Near the E fork of the Toklat River at Mile 43 S of Park Rd., Denali National Park and Preserve, Alaska, USA@en, 6, Roughly bounded by Courtland, Randolph, Chamberlain Sts. and Irwin Ave. and Roughly bounded by Freedom Pkwy., John Wesley Dobbs Ave., Decatur St., Southern RR tracks, and I-75/85 , Atlanta, Georgia@en, SE of Lincoln Memorial, off Independence Ave., Washington DC@en, 1096, W end of Glacier Rt. 3, Glacier NP, Many Glacier, Montana@en, Stehekin Landing, Stehekin, Washington@en, Lake Clark National Park and Preserve Headquarters, Port Alsworth, Alaska@en, Montgomery County and Chester County, Pennsylvania, USA@en, Pawnee County, Kansas, Kansas route 156, USA@en, Old Fort Blvd., Fort Scott, Kansas@en, Reservation 309-G, 16th St. & Park Rd.NW., Washington, District of Columbia@en, At jct. of U.S. 202 and NJ 24, Morristown, New Jersey@en, 37, 5442, Near Picnic Grove 6 on Beach Drive, N.W. in Rock Creek Park, Washington, D.C.@en, Off Chain of Craters Road in Hawaii Volcanoes National Park@en, Manitou Island, Apostle Islands National Lakeshore, Wisconsin@en, Ellsworth, McLean County, Illinois@en, S end of Waterton Lake, Glacier NP, St. Mary, Montana@en, North Fork drainage, Fish Creek to Kintla Lake, Glacier NP, West Glacier, Montana@en, Going-to-the-Sun Rd., near West Glacier, Glacier NP, West Glacier, Montana@en, Pioneer Yosemite Historic Center, Yosemite National Park, California@en, 211, Washington, D.C.@en, NW.@en, Foundry Branch, NW@en, Along Rio Grande at jct. of Park Rtes. 5, 9, and 35, Big Bend National Park, Texas@en, Castolon, Texas@en, Mount Vernon Memorial Hwy., Alexandria, Virginia@en, County Route 85/2, Fayette County, West Virginia@en, At jct. of N and S branch of McKittrick Canyon, Guadalupe Mountains National Park, Texas@en, 4801, Leelanau Peninsula, Michigan@en, Yellowstone Lake, Wyoming@en, Mt. Rainier National Park, Chinook Pass Entrance, Washington@en, York / Cherokee counties, South Carolina@en, Chestnut Street at 5th Street@en, 8095, Northside Dr., Yosemite National Park, Yosemite, California@en, within the Timucuan Ecological and Historic Preserve, Jacksonville, Florida@en, 17, One Park Rd., Petrified Forest National Park, Arizona@en, Devils Tower National Monument, Devils Tower, Wyoming@en, southeast of Gatlinburg near Greenbrier Cove in GSMNP@en, W of Gatlinburg off State Route 73 in Great Smoky Mountains National Park@en, 7, Grand Teton National Park, Teton County, at Kelly, Wyoming@en, Massachusetts Ave., 10th & L Sts. NW, Reservation 69, Washington, District of Columbia@en, N of Going-to-the-Sun Rd. at St. Mary Lake, Glacier NP, St. Mary, Montana@en, Pioneer Yosemite History Center, Wawona, California@en, near the top of Fall River Rd., Rocky Mountain National Park, Estes Park, Colorado@en, N of South Campground facilities, Springdale, Utah@en, Kelbaker Rd., jct. of Kelbaker and Cima Rds. at Union Pacific Railroad crossing, Kelso, California@en, 30, Mi. 3.4 McKinley Park Hwy., Denali National Park and Preserve, Alaska, USA@en, Near main branch of Toklat River at Mile 53.7, W of park rd., Denali National Park and Preserve, Alaska, USA@en, Approx. 11.6 mi. S of WA 101 on Upper Sol Duc Rd., Port Angeles, Washington@en, http://dbpedia.org/resource/Camp_David, http://dbpedia.org/resource/Gindarja_Springs, http://dbpedia.org/resource/Racetrack_mountain_area, http://dbpedia.org/resource/Racetrack_Playa, http://dbpedia.org/resource/Scotty's_Castle, http://dbpedia.org/resource/Dune_Shacks_of_Peaked_Hill_Bars_Historic_District, http://dbpedia.org/resource/Otter_Cliffs_Radio_Station, http://dbpedia.org/resource/Tubbs_Cordage_Company_Office_Building, http://dbpedia.org/resource/Entrance_Station_(Devils_Tower_National_Monument), http://dbpedia.org/resource/Old_Headquarters_Area_Historic_District, http://dbpedia.org/resource/Tower_Ladder_(Devils_Tower_National_Monument), http://dbpedia.org/resource/2002_Denali_earthquake, http://dbpedia.org/resource/Stampede_Airport, http://dbpedia.org/resource/Igloo_Creek_Cabin_No._25, http://dbpedia.org/resource/Sushana_River_Ranger_Cabin_No._17, http://dbpedia.org/resource/Upper_East_Fork_Cabin_No._29, http://dbpedia.org/resource/Upper_Windy_Creek_Ranger_Cabin_No._7, http://dbpedia.org/resource/McKinley_National_Park_Airport, http://dbpedia.org/resource/Sanctuary_River_Cabin_No._31, http://dbpedia.org/resource/Riley_Creek_Ranger_Cabin_No._20, http://dbpedia.org/resource/Ewe_Creek_Ranger_Cabin_No._8, http://dbpedia.org/resource/Lower_East_Fork_Ranger_Cabin_No._9, http://dbpedia.org/resource/Lower_Toklat_River_Ranger_Cabin_No._18, http://dbpedia.org/resource/Lower_Windy_Creek_Ranger_Cabin_No._15, http://dbpedia.org/resource/Moose_Creek_Ranger_Cabin_No._19, http://dbpedia.org/resource/Mount_McKinley_National_Park_Headquarters_District, http://dbpedia.org/resource/Upper_Toklat_River_Cabin_No._24, http://dbpedia.org/resource/Toklat_Ranger_Station_(Pearson_Cabin),_No._4, http://dbpedia.org/resource/Stamp_Act_Congress, http://dbpedia.org/resource/Lake_Vernon, http://dbpedia.org/resource/Kibbie_Lake, http://dbpedia.org/resource/Hodgdon_Homestead_Cabin, http://dbpedia.org/resource/Merced_Grove_Ranger_Station, http://dbpedia.org/resource/Parsons_Memorial_Lodge, http://dbpedia.org/resource/Mirror_Lake_(California), http://dbpedia.org/resource/Lyell_Glacier, http://dbpedia.org/resource/Tenaya_Lake, http://dbpedia.org/resource/Rangers'_Club, http://dbpedia.org/resource/Ostrander_Lake, http://dbpedia.org/resource/Cathedral_Lakes, http://dbpedia.org/resource/Maclure_Glacier, http://dbpedia.org/resource/Chris_Jorgensen_Studio, http://dbpedia.org/resource/Ahwahnee_Hotel, http://dbpedia.org/resource/Badger_Pass_Ski_Area, http://dbpedia.org/resource/Camp_4_(Yosemite), http://dbpedia.org/resource/Wawona_Hotel, http://dbpedia.org/resource/Lake_Eleanor, http://dbpedia.org/resource/May_Lake_(California), http://dbpedia.org/resource/Acting_Superintendent's_Headquarters, http://dbpedia.org/resource/Flora_Lake, http://dbpedia.org/resource/Assateague_Beach_Coast_Guard_Station, http://dbpedia.org/resource/Deep_Lake_(Florida), http://dbpedia.org/resource/Little_Red_Lighthouse, http://dbpedia.org/resource/Fort_Washington_Light, http://dbpedia.org/resource/Franklin_Court, http://dbpedia.org/resource/Purple_Point-Stehekin_Ranger_Station_House, http://dbpedia.org/resource/Buckner_Cabin, http://dbpedia.org/resource/Buckner_Homestead_Historic_District, http://dbpedia.org/resource/Noatak_Wilderness, http://dbpedia.org/resource/Ross_Dam, http://dbpedia.org/resource/Diablo_Lake, http://dbpedia.org/resource/Ross_Lake_(Washington), http://dbpedia.org/resource/Devil's_Corner_Cliff_Walk, http://dbpedia.org/resource/Mission_Los_Santos_Ángeles_de_Guevavi, http://dbpedia.org/resource/Cape_Lookout_Village_Historic_District, http://dbpedia.org/resource/Trinity_Dam, http://dbpedia.org/resource/Ellis_Island_Immigrant_Hospital, http://dbpedia.org/resource/Arlington_Hotel_(Hot_Springs_National_Park), http://dbpedia.org/resource/Bathhouse_Row, http://dbpedia.org/resource/Rim_Village_Historic_District, http://dbpedia.org/resource/Crater_Lake_Superintendent's_Residence, http://dbpedia.org/resource/Munson_Valley_Historic_District, http://dbpedia.org/resource/Comfort_Station_No._68, http://dbpedia.org/resource/Comfort_Station_No._72, http://dbpedia.org/resource/Crater_Lake_Lodge, http://dbpedia.org/resource/Rim_Drive, http://dbpedia.org/resource/Sinnott_Memorial_Observation_Station, http://dbpedia.org/resource/Watchman_Lookout_Station, http://dbpedia.org/resource/Soda_Lake_(San_Bernardino_County), http://dbpedia.org/resource/Mojave_phone_booth, http://dbpedia.org/resource/Zion_Wilderness, http://dbpedia.org/resource/Georgetown_(Washington,_D.C.), http://dbpedia.org/resource/Joaquin_Miller_Cabin, http://dbpedia.org/resource/Mount_Pleasant,_Washington,_D.C., http://dbpedia.org/resource/Dumbarton_Bridge_(Washington,_D.C.), http://dbpedia.org/resource/William_H.G._FitzGerald_Tennis_Center, http://dbpedia.org/resource/Ross_Drive_Bridge, http://dbpedia.org/resource/Boulder_Bridge, http://dbpedia.org/resource/Cabin_Branch_Pyrite_Mine_Historic_District, http://dbpedia.org/resource/Alcatraz_Island_Light, http://dbpedia.org/resource/Glacier_Bay_Wilderness, http://dbpedia.org/resource/Muir_Glacier, http://dbpedia.org/resource/Margerie_Glacier, http://dbpedia.org/resource/Johns_Hopkins_Glacier, http://dbpedia.org/resource/Second_Continental_Congress, http://dbpedia.org/resource/Curtis_Publishing_Company__1, http://dbpedia.org/resource/Kirk's_Cabin_Complex, http://dbpedia.org/resource/Tour_of_the_Moon, http://dbpedia.org/resource/Hornbek_House, http://dbpedia.org/resource/Grinnell_Glacier, http://dbpedia.org/resource/Carcajou_Lake, http://dbpedia.org/resource/Boy_Lake_(Glacier_County,_Montana), http://dbpedia.org/resource/Sexton_Glacier, http://dbpedia.org/resource/Rainbow_Glacier, http://dbpedia.org/resource/Redhorn_Lake, http://dbpedia.org/resource/The_Salamander_Glacier, http://dbpedia.org/resource/Harrison_Glacier, http://dbpedia.org/resource/Cobalt_Lake, http://dbpedia.org/resource/Great_Northern_Railway_Buildings, http://dbpedia.org/resource/Lake_Sherburne, http://dbpedia.org/resource/Lake_Sherburne_Dam, http://dbpedia.org/resource/Agassiz_Glacier, http://dbpedia.org/resource/Twin_Lakes_(Glacier_National_Park), http://dbpedia.org/resource/Goat_Lake_(Glacier_County,_Montana), http://dbpedia.org/resource/Lake_Josephine_(Montana), http://dbpedia.org/resource/Lower_Two_Medicine_Lake, http://dbpedia.org/resource/Two_Medicine_Lake, http://dbpedia.org/resource/Upper_Two_Medicine_Lake, http://dbpedia.org/resource/Winona_Lake_(Flathead_County,_Montana), http://dbpedia.org/resource/Going-to-the-Sun_Road, http://dbpedia.org/resource/Saint_Mary_Lake, http://dbpedia.org/resource/Sue_Lake, http://dbpedia.org/resource/Gunsight_Lake, http://dbpedia.org/resource/Rogers_Lake_(Flathead_County,_Montana), http://dbpedia.org/resource/Ptarmigan_Lake_(Glacier_County,_Montana), http://dbpedia.org/resource/Avalanche_Lake_(Flathead_County,_Montana), http://dbpedia.org/resource/Blackfoot_Glacier, http://dbpedia.org/resource/Bowman_Lake_(Montana), http://dbpedia.org/resource/Cracker_Lake, http://dbpedia.org/resource/Hidden_Lake_(Flathead_County,_Montana), http://dbpedia.org/resource/Iceberg_Lake, http://dbpedia.org/resource/Jackson_Glacier, http://dbpedia.org/resource/Lake_McDonald, http://dbpedia.org/resource/Swiftcurrent_Lake, http://dbpedia.org/resource/Sperry_Glacier, http://dbpedia.org/resource/Bullhead_Lake_(Glacier_County,_Montana), http://dbpedia.org/resource/Waterton_Lake, http://dbpedia.org/resource/Lake_Janet, http://dbpedia.org/resource/Carter_Glaciers, http://dbpedia.org/resource/Atsina_Lake, http://dbpedia.org/resource/Arrow_Lake_(Flathead_County,_Montana), http://dbpedia.org/resource/Redrock_Lake, http://dbpedia.org/resource/Lee_Creek_Snowshoe_Cabin, http://dbpedia.org/resource/Logging_Lake, http://dbpedia.org/resource/Quartz_Lake, http://dbpedia.org/resource/Slide_Lake, http://dbpedia.org/resource/Akaiyan_Lake, http://dbpedia.org/resource/Aurice_Lake, http://dbpedia.org/resource/Beaver_Woman_Lake, http://dbpedia.org/resource/Bench_Lake_(Glacier_County,_Montana), http://dbpedia.org/resource/Buffalo_Woman_Lake, http://dbpedia.org/resource/Camas_Lake, http://dbpedia.org/resource/Cerulean_Lake, http://dbpedia.org/resource/Cosley_Lake, http://dbpedia.org/resource/Falling_Leaf_Lake, http://dbpedia.org/resource/Feather_Woman_Lake, http://dbpedia.org/resource/Fishercap_Lake, http://dbpedia.org/resource/Glenns_Lake, http://dbpedia.org/resource/Goat_Haunt_Lake, http://dbpedia.org/resource/Green_Lake_(Glacier_County,_Montana), http://dbpedia.org/resource/Gyrfalcon_Lake, http://dbpedia.org/resource/Harrison_Lake_(Flathead_County,_Montana), http://dbpedia.org/resource/Ipasha_Lake, http://dbpedia.org/resource/Jackstraw_Lake, http://dbpedia.org/resource/Kaina_Lake, http://dbpedia.org/resource/Katoya_Lake, http://dbpedia.org/resource/Kennedy_Lake_(Glacier_County,_Montana), http://dbpedia.org/resource/Kootenai_Lakes, http://dbpedia.org/resource/Lake_Ellen_Wilson, http://dbpedia.org/resource/Lake_Evangeline, http://dbpedia.org/resource/Lake_Isabel_(Flathead_County,_Montana), http://dbpedia.org/resource/Lake_Nooney, http://dbpedia.org/resource/Lake_Wurdeman, http://dbpedia.org/resource/Lake_of_the_Seven_Winds, http://dbpedia.org/resource/Lena_Lake_(Glacier_County,_Montana), http://dbpedia.org/resource/Lincoln_Lake_(Flathead_County,_Montana), http://dbpedia.org/resource/Lonely_Lakes, http://dbpedia.org/resource/Lower_Quartz_Lake, http://dbpedia.org/resource/Margaret_Lake_(Glacier_County,_Montana), http://dbpedia.org/resource/Medicine_Grizzly_Lake, http://dbpedia.org/resource/Medicine_Owl_Lake, http://dbpedia.org/resource/Miche_Wabun_Lake, http://dbpedia.org/resource/Middle_Quartz_Lake, http://dbpedia.org/resource/Mokowanis_Lake, http://dbpedia.org/resource/Morning_Star_Lake_(Glacier_County,_Montana), http://dbpedia.org/resource/Nahsukin_Lake, http://dbpedia.org/resource/Natahki_Lake, http://dbpedia.org/resource/No_Name_Lake, http://dbpedia.org/resource/Oldman_Lake, http://dbpedia.org/resource/Otatso_Lake, http://dbpedia.org/resource/Otokomi_Lake, http://dbpedia.org/resource/Pitamakan_Lake, http://dbpedia.org/resource/Poia_Lake, http://dbpedia.org/resource/Pray_Lake, http://dbpedia.org/resource/Ruger_Lake, http://dbpedia.org/resource/Running_Crane_Lake, http://dbpedia.org/resource/Shaheeya_Lake, http://dbpedia.org/resource/Snow_Moon_Lake, http://dbpedia.org/resource/Snyder_Lake, http://dbpedia.org/resource/Stoney_Indian_Lake, http://dbpedia.org/resource/Striped_Elk_Lake, http://dbpedia.org/resource/Three_Bears_Lake, http://dbpedia.org/resource/Upper_Grinnell_Lake, http://dbpedia.org/resource/Upper_Snyder_Lake, http://dbpedia.org/resource/Wahseeja_Lake, http://dbpedia.org/resource/Whitecrow_Lake, http://dbpedia.org/resource/Windmaker_Lake, http://dbpedia.org/resource/Young_Man_Lake, http://dbpedia.org/resource/Grace_Lake, http://dbpedia.org/resource/Grinnell_Lake, http://dbpedia.org/resource/Red_Eagle_Glacier, http://dbpedia.org/resource/Red_Eagle_Lake, http://dbpedia.org/resource/Ptarmigan_Tunnel, http://dbpedia.org/resource/Ahern_Glacier_(Montana), http://dbpedia.org/resource/Baby_Glacier_(Montana), http://dbpedia.org/resource/Boulder_Glacier_(Montana), http://dbpedia.org/resource/Chaney_Glacier, http://dbpedia.org/resource/Dixon_Glacier, http://dbpedia.org/resource/Harris_Glacier, http://dbpedia.org/resource/Herbst_Glacier, http://dbpedia.org/resource/Hudson_Glacier, http://dbpedia.org/resource/Ipasha_Glacier, http://dbpedia.org/resource/Kintla_Glacier, http://dbpedia.org/resource/Logan_Glacier_(Montana), http://dbpedia.org/resource/Lupfer_Glacier, http://dbpedia.org/resource/Miche_Wabun_Glacier, http://dbpedia.org/resource/North_Swiftcurrent_Glacier, http://dbpedia.org/resource/Piegan_Glacier, http://dbpedia.org/resource/Pumpelly_Glacier, http://dbpedia.org/resource/Pumpkin_Glacier, http://dbpedia.org/resource/Shepard_Glacier, http://dbpedia.org/resource/Siyeh_Glacier, http://dbpedia.org/resource/Swiftcurrent_Glacier, http://dbpedia.org/resource/Thunderbird_Glacier, http://dbpedia.org/resource/Two_Ocean_Glacier, http://dbpedia.org/resource/Weasel_Collar_Glacier, http://dbpedia.org/resource/Whitecrow_Glacier, http://dbpedia.org/resource/Gem_Glacier, http://dbpedia.org/resource/Gem_Lake_(Flathead_County,_Montana), http://dbpedia.org/resource/Helen_Lake, http://dbpedia.org/resource/Elizabeth_Lake_(Glacier_County,_Montana), http://dbpedia.org/resource/Trout_Lake_(Flathead_County,_Montana), http://dbpedia.org/resource/Vulture_Glacier_(Montana), http://dbpedia.org/resource/Upper_Kintla_Lake, http://dbpedia.org/resource/Alice_Beck_Cabin, http://dbpedia.org/resource/Kintla_Lake, http://dbpedia.org/resource/Lake_Frances_(Glacier_County,_Montana), http://dbpedia.org/resource/Old_Sun_Glacier, http://dbpedia.org/resource/Triple_Glaciers, http://dbpedia.org/resource/Marion_Lake_(Teton_County,_Wyoming), http://dbpedia.org/resource/Teepe_Glacier, http://dbpedia.org/resource/Talus_Lake, http://dbpedia.org/resource/Kit_Lake, http://dbpedia.org/resource/Triangle_X_Barn, http://dbpedia.org/resource/Bradley_Lake, http://dbpedia.org/resource/Double_Diamond_Dude_Ranch_Dining_Hall, http://dbpedia.org/resource/Emma_Matilda_Lake, http://dbpedia.org/resource/Icefloe_Lake, http://dbpedia.org/resource/Jackson_Lake_Lodge, http://dbpedia.org/resource/Jenny_Lake, http://dbpedia.org/resource/Lake_Solitude_(Wyoming), http://dbpedia.org/resource/Leigh_Lake, http://dbpedia.org/resource/Phelps_Lake_(Wyoming), http://dbpedia.org/resource/Schoolroom_Glacier, http://dbpedia.org/resource/Signal_Mountain_Lodge, http://dbpedia.org/resource/Taggart_Lake, http://dbpedia.org/resource/Teton_Glacier, http://dbpedia.org/resource/Trapper_Lake_(Teton_County,_Wyoming), http://dbpedia.org/resource/Two_Ocean_Lake, http://dbpedia.org/resource/Murie_Ranch_Historic_District, http://dbpedia.org/resource/Bar_B_C_Dude_Ranch, http://dbpedia.org/resource/Cirque_Lake_(Teton_County,_Wyoming), http://dbpedia.org/resource/Laurel_Lake_(Teton_County,_Wyoming), http://dbpedia.org/resource/String_Lake, http://dbpedia.org/resource/Indian_Lake_(Teton_County,_Wyoming), http://dbpedia.org/resource/Lake_of_the_Crags, http://dbpedia.org/resource/4_Lazy_F_Dude_Ranch, http://dbpedia.org/resource/Hunter_Hereford_Ranch_Historic_District, http://dbpedia.org/resource/White_Grass_Dude_Ranch, http://dbpedia.org/resource/Holly_Lake, http://dbpedia.org/resource/Amphitheater_Lake, http://dbpedia.org/resource/Delta_Lake_(Teton_County,_Wyoming), http://dbpedia.org/resource/Timberline_Lake_(Wyoming), http://dbpedia.org/resource/Mica_Lake, http://dbpedia.org/resource/Andy_Chambers_Ranch_Historic_District, http://dbpedia.org/resource/Christian_Pond, http://dbpedia.org/resource/Arrowhead_Pool, http://dbpedia.org/resource/Dudley_Lake_(Teton_County,_Wyoming), http://dbpedia.org/resource/Mink_Lake_(Teton_County,_Wyoming), http://dbpedia.org/resource/Ramshead_Lake, http://dbpedia.org/resource/Bearpaw_Lake_(Teton_County,_Wyoming), http://dbpedia.org/resource/Falling_Ice_Glacier, http://dbpedia.org/resource/Middle_Teton_Glacier, http://dbpedia.org/resource/Skillet_Glacier, http://dbpedia.org/resource/AMK_Ranch, http://dbpedia.org/resource/Coyote_Lake_(Teton_County,_Wyoming), http://dbpedia.org/resource/Petersen_Glacier, http://dbpedia.org/resource/Rimrock_Lake_(Wyoming), http://dbpedia.org/resource/Jackson_Lake, http://dbpedia.org/resource/Jackson_Lake_Dam, http://dbpedia.org/resource/Ramshorn_Dude_Ranch_Lodge, http://dbpedia.org/resource/Surprise_Lake_(Teton_County,_Wyoming), http://dbpedia.org/resource/Lake_Taminah, http://dbpedia.org/resource/Snowdrift_Lake_(Wyoming), http://dbpedia.org/resource/Cow_Lake_(Teton_County,_Wyoming), http://dbpedia.org/resource/Elk_Ranch_Reservoir, http://dbpedia.org/resource/Forget-me-not_Lakes_(Wyoming), http://dbpedia.org/resource/Grizzly_Bear_Lake, http://dbpedia.org/resource/Lake_Helen_(Lassen_Peak), http://dbpedia.org/resource/Butte_Lake_(California), http://dbpedia.org/resource/Loomis_Museum, http://dbpedia.org/resource/Manzanita_Lake, http://dbpedia.org/resource/Snag_Lake, http://dbpedia.org/resource/Juniper_Lake_(Lassen_Peak), http://dbpedia.org/resource/Summit_Lake_(Reading_Peak,_Shasta_County), http://dbpedia.org/resource/Horseshoe_Lake_(California), http://dbpedia.org/resource/Emerald_Lake_(Lassen_Peak), http://dbpedia.org/resource/Mammoth_Cave_Railroad, http://dbpedia.org/resource/Good_Spring_Baptist_Church_and_Cemetery, http://dbpedia.org/resource/Joppa_Baptist_Church_and_Cemetery, http://dbpedia.org/resource/Mammoth_Cave_Baptist_Church_and_Cemetery, http://dbpedia.org/resource/Mesa_Verde_Administrative_District, http://dbpedia.org/resource/Longmire_Buildings, http://dbpedia.org/resource/Sarvant_Glacier, http://dbpedia.org/resource/Emmons_Glacier, http://dbpedia.org/resource/Mount_Rainier_Wilderness, http://dbpedia.org/resource/Paradise_Inn_(Washington), http://dbpedia.org/resource/Snow_Lake_(Mount_Rainier), http://dbpedia.org/resource/Mowich_Lake, http://dbpedia.org/resource/Yakima_Park_Stockade_Group, http://dbpedia.org/resource/White_River_Entrance, http://dbpedia.org/resource/Carbon_Glacier, http://dbpedia.org/resource/Flett_Glacier, http://dbpedia.org/resource/Cowlitz_Glacier, http://dbpedia.org/resource/Nisqually_Glacier, http://dbpedia.org/resource/Fryingpan_Glacier, http://dbpedia.org/resource/Whitman_Glacier, http://dbpedia.org/resource/Hidden_Lake_(Skagit_County,_Washington), http://dbpedia.org/resource/Berdeen_Lake, http://dbpedia.org/resource/Luna_Lake_(Washington), http://dbpedia.org/resource/Thornton_Lakes, http://dbpedia.org/resource/Stout_Lake, http://dbpedia.org/resource/Wilcox_Lakes, http://dbpedia.org/resource/Boston_Glacier, http://dbpedia.org/resource/Inspiration_Glacier, http://dbpedia.org/resource/McAllister_Glacier, http://dbpedia.org/resource/North_Klawatti_Glacier, http://dbpedia.org/resource/Blum_Lakes, http://dbpedia.org/resource/Dagger_Lake, http://dbpedia.org/resource/Doubtful_Lake, http://dbpedia.org/resource/Green_View_Lake, http://dbpedia.org/resource/Ipsoot_Lake, http://dbpedia.org/resource/Kettling_Lake, http://dbpedia.org/resource/Monogram_Lake, http://dbpedia.org/resource/Waddell_Lake, http://dbpedia.org/resource/Fremont_Glacier_(Washington), http://dbpedia.org/resource/Price_Lake_(Whatcom_County,_Washington), http://dbpedia.org/resource/Silver_Lake_(North_Cascades_National_Park), http://dbpedia.org/resource/Green_Lake_(North_Cascades_National_Park), http://dbpedia.org/resource/Sulphide_Lake, http://dbpedia.org/resource/Klawatti_Lake, http://dbpedia.org/resource/Moraine_Lake_(Skagit_County,_Washington), http://dbpedia.org/resource/Hanging_Lake_(Washington_-_British_Columbia), http://dbpedia.org/resource/Trapper_Lake_(Chelan_County,_Washington), http://dbpedia.org/resource/Banded_Glacier, http://dbpedia.org/resource/Buckner_Glacier, http://dbpedia.org/resource/Douglas_Glacier_(Washington), http://dbpedia.org/resource/Eldorado_Glacier, http://dbpedia.org/resource/Forbidden_Glacier, http://dbpedia.org/resource/Klawatti_Glacier, http://dbpedia.org/resource/Lower_Curtis_Glacier, http://dbpedia.org/resource/Sahale_Glacier, http://dbpedia.org/resource/Thunder_Glacier_(Skagit_County,_Washington), http://dbpedia.org/resource/Copper_Lake_(Whatcom_County,_Washington), http://dbpedia.org/resource/Egg_Lake_(Whatcom_County,_Washington), http://dbpedia.org/resource/Davenport_Glacier, http://dbpedia.org/resource/Quien_Sabe_Glacier, http://dbpedia.org/resource/Azure_Lake_(Whatcom_County,_Washington), http://dbpedia.org/resource/Alva_B._Adams_Tunnel, http://dbpedia.org/resource/Beaver_Meadows_Visitor_Center, http://dbpedia.org/resource/Fall_River_Pass, http://dbpedia.org/resource/Murphy_Lake_(Grand_County,_Colorado), http://dbpedia.org/resource/Snowdrift_Lake, http://dbpedia.org/resource/McGraw_Ranch, http://dbpedia.org/resource/Dream_Lake, http://dbpedia.org/resource/Fern_Lake, http://dbpedia.org/resource/La_Poudre_Pass_Lake, http://dbpedia.org/resource/Fall_River_Pump_House_and_Catchment_Basin, http://dbpedia.org/resource/East_Longs_Peak_Trail_–_Longs_Peak_Trail_–_Keyhole_Route_–_Shelf_Trail, http://dbpedia.org/resource/Fern_Lake_Trail, http://dbpedia.org/resource/Crystal_Cave_(Sequoia_National_Park), http://dbpedia.org/resource/Lilliput_Glacier, http://dbpedia.org/resource/Mineral_King, http://dbpedia.org/resource/Eagle_Lake_(Tulare_County), http://dbpedia.org/resource/Wolfe_Ranch, http://dbpedia.org/resource/Anhinga_Trail, http://dbpedia.org/resource/Shark_Valley, http://dbpedia.org/resource/Archeological_Resources_of_Everglades_National_Park_MPS, http://dbpedia.org/resource/Ingraham_Lake, http://dbpedia.org/resource/Keys_Desert_Queen_Ranch, http://dbpedia.org/resource/Kobuk_Valley_Wilderness, http://dbpedia.org/resource/Grand_Sable_Dunes, http://dbpedia.org/resource/Au_Sable_Light, http://dbpedia.org/resource/Ainahou_Ranch, http://dbpedia.org/resource/1790_Footprints__1790_Footprints__1, http://dbpedia.org/resource/Bryce_Canyon_Lodge, http://dbpedia.org/resource/Rainbow_Point_Comfort_Station_and_Overlook_Shelter, http://dbpedia.org/resource/Cades_Cove, http://dbpedia.org/resource/John_Ownby_Cabin, http://dbpedia.org/resource/Little_Greenbrier_School, http://dbpedia.org/resource/Oconaluftee_Baptist_Church, http://dbpedia.org/resource/Roaring_Fork_(Great_Smoky_Mountains), http://dbpedia.org/resource/Tyson_McCarter_Place, http://dbpedia.org/resource/Walker_Sisters_Place, http://dbpedia.org/resource/Wheeler_Peak_Glacier, http://dbpedia.org/resource/Pearce_Ferry_Airport, http://dbpedia.org/resource/Temple_Bar_Airport, http://dbpedia.org/resource/Anderson_Glacier, http://dbpedia.org/resource/Blue_Glacier, http://dbpedia.org/resource/Eel_Glacier, http://dbpedia.org/resource/Hanging_Glacier_(Jefferson_County,_Washington), http://dbpedia.org/resource/Hubert_Glacier, http://dbpedia.org/resource/Humes_Glacier, http://dbpedia.org/resource/Ice_River_Glacier, http://dbpedia.org/resource/Jeffers_Glacier, http://dbpedia.org/resource/Queets_Glacier, http://dbpedia.org/resource/White_Glacier_(Mount_Tom), http://dbpedia.org/resource/Carrie_Glacier, http://dbpedia.org/resource/Boulder_Lake, http://dbpedia.org/resource/Glines_Canyon_Dam, http://dbpedia.org/resource/Hoh_Glacier, http://dbpedia.org/resource/Hurricane_Ridge_Ski_and_Snowboard_Area, http://dbpedia.org/resource/Lake_Mills_(Washington), http://dbpedia.org/resource/Olympic_Wilderness, http://dbpedia.org/resource/Cameron_Glaciers, http://dbpedia.org/resource/Lillian_Glacier, http://dbpedia.org/resource/Fairchild_Glacier, http://dbpedia.org/resource/Chiricahua_National_Monument_Historic_Designed_Landscape, http://dbpedia.org/resource/Gates_of_the_Arctic_Wilderness, http://dbpedia.org/resource/Pinery_Station, http://dbpedia.org/resource/Wallace_Pratt_Lodge, http://dbpedia.org/resource/Fure's_Cabin, http://dbpedia.org/resource/Katmai_Wilderness, http://dbpedia.org/resource/Brooks_River_Archeological_District, http://dbpedia.org/resource/Takli_Island, http://dbpedia.org/resource/Exit_Glacier, http://dbpedia.org/resource/Holgate_Glacier, http://dbpedia.org/resource/Wassillie_Trefon_Dena'ina_Fish_Cache, http://dbpedia.org/resource/Libby's_No._23, http://dbpedia.org/resource/Richard_Proenneke__,_Richard,_Site__1, http://dbpedia.org/resource/Canby's_Cross, http://dbpedia.org/resource/Blue_Ridge_Parkway_tunnels, http://dbpedia.org/resource/Moses_H._Cone_Memorial_Park, http://dbpedia.org/resource/Elbert_Crouse_Farmstead, http://dbpedia.org/resource/Grandview_Mine, http://dbpedia.org/resource/Grand_Canyon_Inn_and_Campground, http://dbpedia.org/resource/Grand_Canyon_Park_Operations_Building, http://dbpedia.org/resource/Grand_Canyon_Power_House, http://dbpedia.org/resource/Grand_Canyon_South_Rim_Ranger's_Dormitory, http://dbpedia.org/resource/Mary_Jane_Colter_buildings, http://dbpedia.org/resource/Desert_View_Watchtower, http://dbpedia.org/resource/Buckey_O'Neill_Cabin, http://dbpedia.org/resource/El_Tovar_Hotel, http://dbpedia.org/resource/Grand_Canyon_Lodge, http://dbpedia.org/resource/Hopi_House, http://dbpedia.org/resource/Lookout_Studio, http://dbpedia.org/resource/Hermit's_Rest, http://dbpedia.org/resource/North_Manitou_Island_Lifesaving_Station, http://dbpedia.org/resource/South_Manitou_Island, http://dbpedia.org/resource/Platte_River_Campground_Site, http://dbpedia.org/resource/Manitou_Camp, http://dbpedia.org/resource/Edisen_Fishery, http://dbpedia.org/resource/Rock_Harbor_Light, http://dbpedia.org/resource/Washington_Island_(Michigan), http://dbpedia.org/resource/SS_Algoma__Algoma__1, http://dbpedia.org/resource/SS_America_(1898)__AMERICA__1, http://dbpedia.org/resource/SS_Cumberland__CUMBERLAND__1, http://dbpedia.org/resource/SS_Emperor__EMPEROR__1, http://dbpedia.org/resource/SS_Glenlyon__GLENLYON__1, http://dbpedia.org/resource/SS_Henry_Chisholm__HENRY_CHISHOLM__1, http://dbpedia.org/resource/SS_Monarch__MONARCH__1, http://dbpedia.org/resource/Amygdaloid_Island, http://dbpedia.org/resource/Passage_Island_(Michigan), http://dbpedia.org/resource/Chester_A._Congdon_(ship)__CHESTER_A._CONGDON__1, http://dbpedia.org/resource/SS_Kamloops__KAMLOOPS__1, http://dbpedia.org/resource/USS_Puritan_(ID-2222)__GEORGE_M._COX__1, http://dbpedia.org/resource/Johns_Hotel, http://dbpedia.org/resource/Minong_Mine_Historic_District, http://dbpedia.org/resource/Tallgrass_Prairie_bison_herd, http://dbpedia.org/resource/Bandelier_CCC_Historic_District, http://dbpedia.org/resource/Ocracoke_Island_Airport, http://dbpedia.org/resource/Billy_Mitchell_Airport, http://dbpedia.org/resource/Cape_Lookout_Coast_Guard_Station, http://dbpedia.org/resource/First_Flight_Airport, http://dbpedia.org/resource/George_Perkins_Marsh_Boyhood_Home, http://dbpedia.org/resource/Hole_in_the_Rock_(rock_formation), http://dbpedia.org/resource/Darwin_Glacier_(California), http://dbpedia.org/resource/Rae_Lakes, http://dbpedia.org/resource/Charlotte_Lake_(California), http://dbpedia.org/resource/Mendel_Glacier, http://dbpedia.org/resource/Martha_Lake_(California), http://dbpedia.org/resource/Mount_Fiske_Glacier, http://dbpedia.org/resource/Mount_Warlow_Glacier, http://dbpedia.org/resource/Emery_County,_Utah, http://dbpedia.org/resource/Garfield_County,_Utah, http://dbpedia.org/resource/Sevier_County,_Utah, http://dbpedia.org/resource/Wayne_County,_Utah, http://dbpedia.org/resource/San_Juan,_Puerto_Rico, http://dbpedia.org/resource/Frederick_County,_Maryland, http://dbpedia.org/resource/Kansas, http://dbpedia.org/resource/Missouri, http://dbpedia.org/resource/Fairfield_County,_Connecticut, http://dbpedia.org/resource/Connecticut, http://dbpedia.org/resource/Nevada, http://dbpedia.org/resource/Barnstable_County,_Massachusetts, http://dbpedia.org/resource/Hancock_County,_Maine, http://dbpedia.org/resource/Knox_County,_Maine, http://dbpedia.org/resource/Maine, http://dbpedia.org/resource/South_Dakota, http://dbpedia.org/resource/Pennington_County,_South_Dakota, http://dbpedia.org/resource/Oglala_Lakota_County,_South_Dakota, http://dbpedia.org/resource/Jackson_County,_South_Dakota, http://dbpedia.org/resource/Beaverhead_County,_Montana, http://dbpedia.org/resource/Big_Horn_County,_Montana, http://dbpedia.org/resource/Carbon_County,_Montana, http://dbpedia.org/resource/Clayton_County,_Iowa, http://dbpedia.org/resource/Allamakee_County,_Iowa, http://dbpedia.org/resource/Iowa, http://dbpedia.org/resource/Hanford,_Washington, http://dbpedia.org/resource/Los_Alamos,_New_Mexico, http://dbpedia.org/resource/Oak_Ridge,_Tennessee, http://dbpedia.org/resource/Nebraska, http://dbpedia.org/resource/Yavapai_County,_Arizona, http://dbpedia.org/resource/Rock_County,_Nebraska, http://dbpedia.org/resource/Brown_County,_Nebraska, http://dbpedia.org/resource/Cherry_County,_Nebraska, http://dbpedia.org/resource/Keya_Paha_County,_Nebraska, http://dbpedia.org/resource/Minnesota, http://dbpedia.org/resource/St._Louis_County,_Minnesota, http://dbpedia.org/resource/Padre_Island, http://dbpedia.org/resource/Kentucky, http://dbpedia.org/resource/Tennessee, http://dbpedia.org/resource/Calloway_County,_Kentucky, http://dbpedia.org/resource/Marin_County,_California, http://dbpedia.org/resource/Rutherford_County,_Tennessee, http://dbpedia.org/resource/Lincoln_County,_Wyoming, http://dbpedia.org/resource/Fire_Island, http://dbpedia.org/resource/Dorchester_County,_Maryland, http://dbpedia.org/resource/Maryland, http://dbpedia.org/resource/Porter_County,_Indiana, http://dbpedia.org/resource/Chaco_Culture_National_Historical_Park, http://dbpedia.org/resource/Matanuska-Susitna_Borough,_Alaska, http://dbpedia.org/resource/Denali_Borough,_Alaska, http://dbpedia.org/resource/Wall_Street, http://dbpedia.org/resource/Springfield,_Illinois, http://dbpedia.org/resource/Alabama, http://dbpedia.org/resource/Cherokee_County,_Alabama, http://dbpedia.org/resource/American_Samoa, http://dbpedia.org/resource/Jackson_County,_Alabama, http://dbpedia.org/resource/National_Mall, http://dbpedia.org/resource/Alberta, http://dbpedia.org/resource/Canada, http://dbpedia.org/resource/Madera_County,_California, http://dbpedia.org/resource/Mariposa_County,_California, http://dbpedia.org/resource/Tuolumne_County,_California, http://dbpedia.org/resource/Dutchess_County,_New_York, http://dbpedia.org/resource/Haviland,_New_York, http://dbpedia.org/resource/Custer_County,_South_Dakota, http://dbpedia.org/resource/Vermont_Avenue, http://dbpedia.org/resource/Val_Verde_County,_Texas, http://dbpedia.org/resource/Anacostia, http://dbpedia.org/resource/Washington_County,_Maryland, http://dbpedia.org/resource/Worcester_County,_Maryland, http://dbpedia.org/resource/Seward_Peninsula, http://dbpedia.org/resource/Collier_County,_Florida, http://dbpedia.org/resource/Miami-Dade_County,_Florida, http://dbpedia.org/resource/Lee_County,_Mississippi, http://dbpedia.org/resource/Topeka,_Kansas, http://dbpedia.org/resource/Delaware, http://dbpedia.org/resource/El_Paso,_Texas, http://dbpedia.org/resource/Camden_County,_Georgia, http://dbpedia.org/resource/Prince_George's_County,_Maryland, http://dbpedia.org/resource/Fort_Foote__location__1, http://dbpedia.org/resource/Brookline,_Massachusetts, http://dbpedia.org/resource/Fredericksburg,_Virginia, http://dbpedia.org/resource/Spotsylvania_County,_Virginia, http://dbpedia.org/resource/Baltimore_County,_Maryland, http://dbpedia.org/resource/Jackson_County,_Missouri, http://dbpedia.org/resource/Union_Township,_Berks_County,_Pennsylvania, http://dbpedia.org/resource/Berks_County,_Pennsylvania, http://dbpedia.org/resource/Apache_County,_Arizona, http://dbpedia.org/resource/Adams_Township,_Cambria_County,_Pennsylvania, http://dbpedia.org/resource/Croyle_Township,_Cambria_County,_Pennsylvania, http://dbpedia.org/resource/Chelan_County,_Washington, http://dbpedia.org/resource/Moore_County,_Texas, http://dbpedia.org/resource/Potter_County,_Texas, http://dbpedia.org/resource/Hutchinson_County,_Texas, http://dbpedia.org/resource/Lincoln_County,_Washington, http://dbpedia.org/resource/Grant_County,_Washington, http://dbpedia.org/resource/Ferry_County,_Washington, http://dbpedia.org/resource/Stevens_County,_Washington, http://dbpedia.org/resource/Lowell,_Massachusetts]... [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 656, P: 0.512, R: 0.562, macro F1: 0.517, macro F1*: 0.536 -[INFO] Current time: 3874 ms, average: 2717.87 ms/ques, entity detection: 433.20 ms/ques (linking tool: 341.19 ms/ques), relation detection: 893.39 ms/ques (similarity: 699.94 ms/ques), EDG generation: 48.28 ms/ques +[INFO] Cumulative metrics, sample: 656, P: 0.512, R: 0.562, macro F1: 0.536 +[INFO] Current question consumed time: 3349 ms, average: 2462.56 ms/ques, entity detection: 385.80 ms/ques (linking tool: 307.17 ms/ques), relation detection: 829.24 ms/ques (similarity: 689.49 ms/ques), EDG generation: 48.18 ms/ques [INFO] Entity Linking P: 0.75, R: 0.7019817073170732, micro F1: 0.7172256097560985, macro F1: 0.7251968503937007 [INFO] Relation Linking P: 0.5936229674796747, R: 0.5403963414634146, micro F1: 0.5554369918699191, macro F1: 0.5657605250719249 [INFO] Type Linking P: 0.5472560975609756, R: 0.5472560975609756, micro F1: 0.5472560975609756, macro F1: 0.5472560975609756 @@ -9214,8 +9214,8 @@ Question 656: Name the municipality of Roberto Clemente Bridge ? [INFO] Golden Answer: [http://dbpedia.org/resource/Pittsburgh] [INFO] Predicted Answer: [http://dbpedia.org/resource/Pittsburgh] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 657, P: 0.513, R: 0.562, macro F1: 0.518, macro F1*: 0.536 -[INFO] Current time: 1384 ms, average: 2715.84 ms/ques, entity detection: 432.96 ms/ques (linking tool: 340.86 ms/ques), relation detection: 892.94 ms/ques (similarity: 699.40 ms/ques), EDG generation: 48.22 ms/ques +[INFO] Cumulative metrics, sample: 657, P: 0.513, R: 0.562, macro F1: 0.536 +[INFO] Current question consumed time: 1088 ms, average: 2460.47 ms/ques, entity detection: 385.44 ms/ques (linking tool: 306.86 ms/ques), relation detection: 828.80 ms/ques (similarity: 688.97 ms/ques), EDG generation: 48.12 ms/ques [INFO] Entity Linking P: 0.7503805175038052, R: 0.7024353120243532, micro F1: 0.717656012176561, macro F1: 0.7256167812006413 [INFO] Relation Linking P: 0.5942415017757483, R: 0.541095890410959, micro F1: 0.5561136478944703, macro F1: 0.5664248121048692 [INFO] Type Linking P: 0.547945205479452, R: 0.547945205479452, micro F1: 0.547945205479452, macro F1: 0.547945205479452 @@ -9228,8 +9228,8 @@ Question 657: Which past members of the The Conglomerate (American group) also s [INFO] Golden Answer: [http://dbpedia.org/resource/Blackstreet] [INFO] Predicted Answer: [http://dbpedia.org/resource/Papoose_(rapper), http://dbpedia.org/resource/Rampage_(rapper), http://dbpedia.org/resource/Roc_Marciano, http://dbpedia.org/resource/Dready, http://dbpedia.org/resource/Lonny_Bereal, http://dbpedia.org/resource/Blackstreet, http://dbpedia.org/resource/Rah_Digga, http://dbpedia.org/resource/Busta_Rhymes, http://dbpedia.org/resource/Lord_Have_Mercy_(rapper), http://dbpedia.org/resource/Reek_da_Villian, http://dbpedia.org/resource/N.O.R.E., Serious@en, Labba@en, Meka@en, Spliff Star@en, Show Money@en, Seige Monstracity@en, J-Doe@en, Baby Sham@en, DJ Ice Cold@en, DJ Scrachator@en, M. Dollars@en, Nikki Grier@en] [INFO] P: 0.043478260869565216, R: 1.0, F: 0.08333333333333333 -[INFO] Cumulative metrics, sample: 658, P: 0.512, R: 0.563, macro F1: 0.518, macro F1*: 0.536 -[INFO] Current time: 2433 ms, average: 2715.41 ms/ques, entity detection: 432.89 ms/ques (linking tool: 340.76 ms/ques), relation detection: 892.57 ms/ques (similarity: 698.93 ms/ques), EDG generation: 48.20 ms/ques +[INFO] Cumulative metrics, sample: 658, P: 0.512, R: 0.563, macro F1: 0.536 +[INFO] Current question consumed time: 2026 ms, average: 2459.81 ms/ques, entity detection: 385.31 ms/ques (linking tool: 306.73 ms/ques), relation detection: 828.46 ms/ques (similarity: 688.52 ms/ques), EDG generation: 48.10 ms/ques [INFO] Entity Linking P: 0.7507598784194529, R: 0.7021276595744681, micro F1: 0.7175785207700112, macro F1: 0.7256298406480906 [INFO] Relation Linking P: 0.5948581560283688, R: 0.541033434650456, micro F1: 0.556281661600811, macro F1: 0.5666705413208143 [INFO] Type Linking P: 0.5486322188449848, R: 0.5486322188449848, micro F1: 0.5486322188449848, macro F1: 0.5486322188449848 @@ -9242,8 +9242,8 @@ Question 658: List all the notable works of Russell T Davies? [INFO] Golden Answer: [http://dbpedia.org/resource/Doctor_Who, http://dbpedia.org/resource/Torchwood, http://dbpedia.org/resource/Cucumber_(TV_series), http://dbpedia.org/resource/Queer_as_Folk_(UK_TV_series), http://dbpedia.org/resource/The_Sarah_Jane_Adventures, http://dbpedia.org/resource/Casanova_(2005_TV_serial), http://dbpedia.org/resource/Banana_(TV_series), http://dbpedia.org/resource/Bob_&_Rose, http://dbpedia.org/resource/The_Second_Coming_(TV_serial)] [INFO] Predicted Answer: [http://dbpedia.org/resource/Doctor_Who, http://dbpedia.org/resource/Torchwood, http://dbpedia.org/resource/Cucumber_(TV_series), http://dbpedia.org/resource/Queer_as_Folk_(UK_TV_series), http://dbpedia.org/resource/The_Sarah_Jane_Adventures, http://dbpedia.org/resource/Casanova_(2005_TV_serial), http://dbpedia.org/resource/Banana_(TV_series), http://dbpedia.org/resource/Bob_&_Rose, http://dbpedia.org/resource/The_Second_Coming_(TV_serial)] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 659, P: 0.513, R: 0.564, macro F1: 0.518, macro F1*: 0.537 -[INFO] Current time: 996 ms, average: 2712.80 ms/ques, entity detection: 432.60 ms/ques (linking tool: 340.47 ms/ques), relation detection: 891.66 ms/ques (similarity: 698.12 ms/ques), EDG generation: 48.14 ms/ques +[INFO] Cumulative metrics, sample: 659, P: 0.513, R: 0.564, macro F1: 0.537 +[INFO] Current question consumed time: 847 ms, average: 2457.36 ms/ques, entity detection: 385.02 ms/ques (linking tool: 306.45 ms/ques), relation detection: 827.59 ms/ques (similarity: 687.72 ms/ques), EDG generation: 48.04 ms/ques [INFO] Entity Linking P: 0.7511380880121397, R: 0.7025796661608498, micro F1: 0.7180070814365209, macro F1: 0.7260478804793751 [INFO] Relation Linking P: 0.5954729387961557, R: 0.5417298937784522, micro F1: 0.5569549822964092, macro F1: 0.5673314955638232 [INFO] Type Linking P: 0.5493171471927162, R: 0.5493171471927162, micro F1: 0.5493171471927162, macro F1: 0.5493171471927162 @@ -9256,8 +9256,8 @@ Question 659: Which military unit garrison is Arlington County, Virginia and The [INFO] Golden Answer: [http://dbpedia.org/resource/United_States_Navy, http://dbpedia.org/resource/United_States_Army_Air_Forces] [INFO] Predicted Answer: [http://dbpedia.org/resource/Roman_Catholic_Diocese_of_Richmond, http://dbpedia.org/resource/Ukrainian_Catholic_Archeparchy_of_Philadelphia, http://dbpedia.org/resource/List_of_counties_in_Virginia] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 660, P: 0.512, R: 0.563, macro F1: 0.517, macro F1*: 0.536 -[INFO] Current time: 7636 ms, average: 2720.26 ms/ques, entity detection: 433.57 ms/ques (linking tool: 341.37 ms/ques), relation detection: 895.21 ms/ques (similarity: 699.57 ms/ques), EDG generation: 48.12 ms/ques +[INFO] Cumulative metrics, sample: 660, P: 0.512, R: 0.563, macro F1: 0.536 +[INFO] Current question consumed time: 6662 ms, average: 2463.73 ms/ques, entity detection: 385.82 ms/ques (linking tool: 307.20 ms/ques), relation detection: 830.82 ms/ques (similarity: 689.19 ms/ques), EDG generation: 48.01 ms/ques [INFO] Entity Linking P: 0.75, R: 0.7015151515151515, micro F1: 0.7169191919191928, macro F1: 0.724947807933194 [INFO] Relation Linking P: 0.594570707070707, R: 0.5409090909090909, micro F1: 0.5561111111111116, macro F1: 0.5664719023887266 [INFO] Type Linking P: 0.5484848484848485, R: 0.5484848484848485, micro F1: 0.5484848484848485, macro F1: 0.5484848484848485 @@ -9271,8 +9271,8 @@ Question 660: Name all the products produced by the producer of United States do [INFO] Predicted Answer: [Blue Cross Blue Shield revenue = US$61.7 Billion@en, Savings accounts in sterling, US dollars and euros@en] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 661, P: 0.511, R: 0.562, macro F1: 0.517, macro F1*: 0.535 -[INFO] Current time: 23856 ms, average: 2752.24 ms/ques, entity detection: 449.29 ms/ques (linking tool: 357.13 ms/ques), relation detection: 895.58 ms/ques (similarity: 699.31 ms/ques), EDG generation: 48.18 ms/ques +[INFO] Cumulative metrics, sample: 661, P: 0.511, R: 0.562, macro F1: 0.535 +[INFO] Current question consumed time: 20626 ms, average: 2491.21 ms/ques, entity detection: 399.68 ms/ques (linking tool: 321.10 ms/ques), relation detection: 831.05 ms/ques (similarity: 688.90 ms/ques), EDG generation: 48.07 ms/ques [INFO] Entity Linking P: 0.7503782148260212, R: 0.7019667170953101, micro F1: 0.7173474533535058, macro F1: 0.7253656076651538 [INFO] Relation Linking P: 0.5944276348966212, R: 0.5416036308623298, micro F1: 0.5562783661119521, macro F1: 0.5667875084931495 [INFO] Type Linking P: 0.5491679273827534, R: 0.5491679273827534, micro F1: 0.5491679273827534, macro F1: 0.5491679273827534 @@ -9285,8 +9285,8 @@ Question 661: What is the launch site of Aquarius (rocket) ? [INFO] Golden Answer: [http://dbpedia.org/resource/Hawaii] [INFO] Predicted Answer: [http://dbpedia.org/resource/Hawaii] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 662, P: 0.512, R: 0.563, macro F1: 0.517, macro F1*: 0.536 -[INFO] Current time: 925 ms, average: 2749.48 ms/ques, entity detection: 448.82 ms/ques (linking tool: 356.72 ms/ques), relation detection: 894.69 ms/ques (similarity: 698.53 ms/ques), EDG generation: 48.13 ms/ques +[INFO] Cumulative metrics, sample: 662, P: 0.512, R: 0.563, macro F1: 0.536 +[INFO] Current question consumed time: 755 ms, average: 2488.58 ms/ques, entity detection: 399.23 ms/ques (linking tool: 320.71 ms/ques), relation detection: 830.21 ms/ques (similarity: 688.13 ms/ques), EDG generation: 48.02 ms/ques [INFO] Entity Linking P: 0.7507552870090635, R: 0.702416918429003, micro F1: 0.7177744209466274, macro F1: 0.7257821381688452 [INFO] Relation Linking P: 0.5950402819738166, R: 0.5422960725075529, micro F1: 0.5569486404833842, macro F1: 0.5674451654107808 [INFO] Type Linking P: 0.5498489425981873, R: 0.5498489425981873, micro F1: 0.5498489425981873, macro F1: 0.5498489425981873 @@ -9299,8 +9299,8 @@ Question 662: What are the players whose former teams are Carolina Panthers and [INFO] Golden Answer: [http://dbpedia.org/resource/Andre_Royal, http://dbpedia.org/resource/Darius_Butler, http://dbpedia.org/resource/Mike_Teeter, http://dbpedia.org/resource/Terrance_Taylor_(American_football), http://dbpedia.org/resource/Ben_Hartsock, http://dbpedia.org/resource/Cleveland_Pinkney, http://dbpedia.org/resource/Fred_Lane_(American_football), http://dbpedia.org/resource/Mario_Addison, http://dbpedia.org/resource/Tauren_Poole, http://dbpedia.org/resource/Drake_Nevis, http://dbpedia.org/resource/Kevin_McMahan, http://dbpedia.org/resource/Ed_Johnson_(American_football), http://dbpedia.org/resource/Chris_Hetherington, http://dbpedia.org/resource/Pat_Ross_(American_football), http://dbpedia.org/resource/Alan_Campos, http://dbpedia.org/resource/Devin_Moore_(American_football), http://dbpedia.org/resource/Steve_Justice, http://dbpedia.org/resource/Stevie_Brown, http://dbpedia.org/resource/Emmanuel_McDaniel, http://dbpedia.org/resource/Tyler_Brayton, http://dbpedia.org/resource/Mike_Pollak, http://dbpedia.org/resource/Jason_Baker, http://dbpedia.org/resource/Anthony_Johnson_(running_back), http://dbpedia.org/resource/Aaron_Francisco, http://dbpedia.org/resource/Mark_Thomas_(American_football), http://dbpedia.org/resource/Nate_Jacquet, http://dbpedia.org/resource/Jordan_Senn, http://dbpedia.org/resource/Jamie_Petrowski, http://dbpedia.org/resource/Josh_Vaughan, http://dbpedia.org/resource/Brad_Scioli, http://dbpedia.org/resource/Tupe_Peko, http://dbpedia.org/resource/Andrew_McDonald_(American_football), http://dbpedia.org/resource/Josh_Thomas_(cornerback), http://dbpedia.org/resource/Hayworth_Hicks, http://dbpedia.org/resource/Bryan_Jurewicz, http://dbpedia.org/resource/Matt_Blanchard, http://dbpedia.org/resource/Jim_Kubiak, http://dbpedia.org/resource/Jerome_Felton, http://dbpedia.org/resource/Teddy_Williams_(American_football), http://dbpedia.org/resource/Tony_Hills_(American_football), http://dbpedia.org/resource/Jason_Pociask, http://dbpedia.org/resource/Thomas_Austin_(American_football), http://dbpedia.org/resource/Tanard_Davis, http://dbpedia.org/resource/Marlon_Favorite, http://dbpedia.org/resource/Shawn_King, http://dbpedia.org/resource/Jim_Harbaugh, http://dbpedia.org/resource/Kerry_Collins, http://dbpedia.org/resource/Dewell_Brewer, http://dbpedia.org/resource/Jermale_Hines, http://dbpedia.org/resource/DeCori_Birmingham, http://dbpedia.org/resource/Jeff_Zgonina, http://dbpedia.org/resource/Jermaine_Wiggins, http://dbpedia.org/resource/Mike_Seidman, http://dbpedia.org/resource/Adam_Seward, http://dbpedia.org/resource/Jeff_Brady, http://dbpedia.org/resource/Idrees_Bashir, http://dbpedia.org/resource/Kenneth_Moore_(American_football), http://dbpedia.org/resource/Tyrone_Poole, http://dbpedia.org/resource/Lee_Ziemba, http://dbpedia.org/resource/Andre_Brown_(running_back), http://dbpedia.org/resource/Jason_Shirley, http://dbpedia.org/resource/DeMario_Pressley, http://dbpedia.org/resource/Mike_McNeill_(American_football)] [INFO] Predicted Answer: [http://dbpedia.org/resource/Andre_Royal, http://dbpedia.org/resource/Darius_Butler, http://dbpedia.org/resource/Mike_Teeter, http://dbpedia.org/resource/Terrance_Taylor_(American_football), http://dbpedia.org/resource/Ben_Hartsock, http://dbpedia.org/resource/Cleveland_Pinkney, http://dbpedia.org/resource/Fred_Lane_(American_football), http://dbpedia.org/resource/Mario_Addison, http://dbpedia.org/resource/Tauren_Poole, http://dbpedia.org/resource/Drake_Nevis, http://dbpedia.org/resource/Kevin_McMahan, http://dbpedia.org/resource/Ed_Johnson_(American_football), http://dbpedia.org/resource/Chris_Hetherington, http://dbpedia.org/resource/Pat_Ross_(American_football), http://dbpedia.org/resource/Alan_Campos, http://dbpedia.org/resource/Devin_Moore_(American_football), http://dbpedia.org/resource/Steve_Justice, http://dbpedia.org/resource/Stevie_Brown, http://dbpedia.org/resource/Emmanuel_McDaniel, http://dbpedia.org/resource/Tyler_Brayton, http://dbpedia.org/resource/Mike_Pollak, http://dbpedia.org/resource/Jason_Baker, http://dbpedia.org/resource/Anthony_Johnson_(running_back), http://dbpedia.org/resource/Aaron_Francisco, http://dbpedia.org/resource/Mark_Thomas_(American_football), http://dbpedia.org/resource/Nate_Jacquet, http://dbpedia.org/resource/Jordan_Senn, http://dbpedia.org/resource/Jamie_Petrowski, http://dbpedia.org/resource/Josh_Vaughan, http://dbpedia.org/resource/Brad_Scioli, http://dbpedia.org/resource/Tupe_Peko, http://dbpedia.org/resource/Andrew_McDonald_(American_football), http://dbpedia.org/resource/Josh_Thomas_(cornerback), http://dbpedia.org/resource/Hayworth_Hicks, http://dbpedia.org/resource/Bryan_Jurewicz, http://dbpedia.org/resource/Matt_Blanchard, http://dbpedia.org/resource/Jim_Kubiak, http://dbpedia.org/resource/Jerome_Felton, http://dbpedia.org/resource/Teddy_Williams_(American_football), http://dbpedia.org/resource/Tony_Hills_(American_football), http://dbpedia.org/resource/Jason_Pociask, http://dbpedia.org/resource/Thomas_Austin_(American_football), http://dbpedia.org/resource/Tanard_Davis, http://dbpedia.org/resource/Marlon_Favorite, http://dbpedia.org/resource/Shawn_King, http://dbpedia.org/resource/Jim_Harbaugh, http://dbpedia.org/resource/Kerry_Collins, http://dbpedia.org/resource/Dewell_Brewer, http://dbpedia.org/resource/Jermale_Hines, http://dbpedia.org/resource/DeCori_Birmingham, http://dbpedia.org/resource/Jeff_Zgonina, http://dbpedia.org/resource/Jermaine_Wiggins, http://dbpedia.org/resource/Mike_Seidman, http://dbpedia.org/resource/Adam_Seward, http://dbpedia.org/resource/Jeff_Brady, http://dbpedia.org/resource/Idrees_Bashir, http://dbpedia.org/resource/Kenneth_Moore_(American_football), http://dbpedia.org/resource/Tyrone_Poole, http://dbpedia.org/resource/Lee_Ziemba, http://dbpedia.org/resource/Andre_Brown_(running_back), http://dbpedia.org/resource/Jason_Shirley, http://dbpedia.org/resource/DeMario_Pressley, http://dbpedia.org/resource/Mike_McNeill_(American_football)] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 663, P: 0.513, R: 0.563, macro F1: 0.518, macro F1*: 0.537 -[INFO] Current time: 8229 ms, average: 2757.74 ms/ques, entity detection: 449.62 ms/ques (linking tool: 357.25 ms/ques), relation detection: 897.62 ms/ques (similarity: 701.83 ms/ques), EDG generation: 48.11 ms/ques +[INFO] Cumulative metrics, sample: 663, P: 0.513, R: 0.563, macro F1: 0.537 +[INFO] Current question consumed time: 7055 ms, average: 2495.47 ms/ques, entity detection: 399.79 ms/ques (linking tool: 321.09 ms/ques), relation detection: 832.69 ms/ques (similarity: 691.52 ms/ques), EDG generation: 48.01 ms/ques [INFO] Entity Linking P: 0.751131221719457, R: 0.702865761689291, micro F1: 0.7182001005530427, macro F1: 0.7261974052308442 [INFO] Relation Linking P: 0.5956510809451986, R: 0.5429864253393665, micro F1: 0.557616892911011, macro F1: 0.5681008212127735 [INFO] Type Linking P: 0.55052790346908, R: 0.55052790346908, micro F1: 0.55052790346908, macro F1: 0.55052790346908 @@ -9313,8 +9313,8 @@ Question 663: Count the total number of mammals whose phylum is Chordate and or [INFO] Golden Answer: [249] [INFO] Predicted Answer: [250] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 664, P: 0.512, R: 0.562, macro F1: 0.517, macro F1*: 0.536 -[INFO] Current time: 10586 ms, average: 2769.53 ms/ques, entity detection: 454.57 ms/ques (linking tool: 362.21 ms/ques), relation detection: 899.53 ms/ques (similarity: 701.07 ms/ques), EDG generation: 48.17 ms/ques +[INFO] Cumulative metrics, sample: 664, P: 0.512, R: 0.562, macro F1: 0.536 +[INFO] Current question consumed time: 9166 ms, average: 2505.52 ms/ques, entity detection: 404.18 ms/ques (linking tool: 325.50 ms/ques), relation detection: 834.20 ms/ques (similarity: 690.80 ms/ques), EDG generation: 48.07 ms/ques [INFO] Entity Linking P: 0.7515060240963856, R: 0.7033132530120482, micro F1: 0.7186244979919688, macro F1: 0.7266114146025096 [INFO] Relation Linking P: 0.5962600401606425, R: 0.5436746987951807, micro F1: 0.558283132530121, macro F1: 0.5687544850767156 [INFO] Type Linking P: 0.5512048192771084, R: 0.5512048192771084, micro F1: 0.5512048192771084, macro F1: 0.5512048192771084 @@ -9327,8 +9327,8 @@ Question 664: Who is the music composer of Kenny Dies ? [INFO] Golden Answer: [http://dbpedia.org/resource/Willie_Nelson, http://dbpedia.org/resource/On_the_Road_Again_(Willie_Nelson_song)] [INFO] Predicted Answer: [http://dbpedia.org/resource/Willie_Nelson, http://dbpedia.org/resource/On_the_Road_Again_(Willie_Nelson_song)] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 665, P: 0.513, R: 0.563, macro F1: 0.518, macro F1*: 0.537 -[INFO] Current time: 814 ms, average: 2766.59 ms/ques, entity detection: 454.25 ms/ques (linking tool: 361.88 ms/ques), relation detection: 898.45 ms/ques (similarity: 700.18 ms/ques), EDG generation: 48.12 ms/ques +[INFO] Cumulative metrics, sample: 665, P: 0.513, R: 0.563, macro F1: 0.537 +[INFO] Current question consumed time: 642 ms, average: 2502.72 ms/ques, entity detection: 403.85 ms/ques (linking tool: 325.18 ms/ques), relation detection: 833.19 ms/ques (similarity: 689.92 ms/ques), EDG generation: 48.02 ms/ques [INFO] Entity Linking P: 0.7518796992481203, R: 0.7037593984962406, micro F1: 0.71904761904762, macro F1: 0.7270241720002485 [INFO] Relation Linking P: 0.5968671679197994, R: 0.544360902255639, micro F1: 0.5589473684210531, macro F1: 0.5694061661235552 [INFO] Type Linking P: 0.5518796992481203, R: 0.5518796992481203, micro F1: 0.5518796992481203, macro F1: 0.5518796992481203 @@ -9341,8 +9341,8 @@ Question 665: How many cities are around the sea into which the aegean sea flows [INFO] Golden Answer: [13] [INFO] Predicted Answer: [12] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 666, P: 0.512, R: 0.562, macro F1: 0.517, macro F1*: 0.536 -[INFO] Current time: 3736 ms, average: 2768.05 ms/ques, entity detection: 453.78 ms/ques (linking tool: 361.52 ms/ques), relation detection: 899.03 ms/ques (similarity: 701.01 ms/ques), EDG generation: 48.09 ms/ques +[INFO] Cumulative metrics, sample: 666, P: 0.512, R: 0.562, macro F1: 0.536 +[INFO] Current question consumed time: 3240 ms, average: 2503.82 ms/ques, entity detection: 403.41 ms/ques (linking tool: 324.83 ms/ques), relation detection: 833.70 ms/ques (similarity: 690.77 ms/ques), EDG generation: 47.99 ms/ques [INFO] Entity Linking P: 0.7522522522522522, R: 0.7042042042042042, micro F1: 0.7194694694694704, macro F1: 0.7274356831057861 [INFO] Relation Linking P: 0.5974724724724724, R: 0.545045045045045, micro F1: 0.5596096096096101, macro F1: 0.5700558734180462 [INFO] Type Linking P: 0.551051051051051, R: 0.551051051051051, micro F1: 0.551051051051051, macro F1: 0.551051051051051 @@ -9355,8 +9355,8 @@ Question 666: Where did the office holder who is predecessor to Marco Aurelio R [INFO] Golden Answer: [http://dbpedia.org/resource/Panama] [INFO] Predicted Answer: [http://dbpedia.org/resource/Max_Delvalle] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 667, P: 0.511, R: 0.561, macro F1: 0.517, macro F1*: 0.535 -[INFO] Current time: 811 ms, average: 2765.11 ms/ques, entity detection: 453.36 ms/ques (linking tool: 361.12 ms/ques), relation detection: 897.96 ms/ques (similarity: 700.21 ms/ques), EDG generation: 48.05 ms/ques +[INFO] Cumulative metrics, sample: 667, P: 0.511, R: 0.561, macro F1: 0.535 +[INFO] Current question consumed time: 684 ms, average: 2501.10 ms/ques, entity detection: 402.99 ms/ques (linking tool: 324.45 ms/ques), relation detection: 832.71 ms/ques (similarity: 689.98 ms/ques), EDG generation: 47.96 ms/ques [INFO] Entity Linking P: 0.7526236881559221, R: 0.704647676161919, micro F1: 0.7198900549725147, macro F1: 0.7278459535664267 [INFO] Relation Linking P: 0.5980759620189905, R: 0.5449775112443778, micro F1: 0.5597701149425293, macro F1: 0.5702934411032543 [INFO] Type Linking P: 0.5502248875562219, R: 0.5502248875562219, micro F1: 0.5502248875562219, macro F1: 0.5502248875562219 @@ -9369,8 +9369,8 @@ Question 667: Who is associated with the musical artists from Africa '70 (band) [INFO] Golden Answer: [http://dbpedia.org/resource/Fela_Kuti] [INFO] Predicted Answer: [http://dbpedia.org/resource/Dele_Sosimi] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 668, P: 0.510, R: 0.561, macro F1: 0.516, macro F1*: 0.534 -[INFO] Current time: 3221 ms, average: 2765.80 ms/ques, entity detection: 454.02 ms/ques (linking tool: 361.75 ms/ques), relation detection: 898.06 ms/ques (similarity: 700.01 ms/ques), EDG generation: 48.08 ms/ques +[INFO] Cumulative metrics, sample: 668, P: 0.510, R: 0.561, macro F1: 0.534 +[INFO] Current question consumed time: 2718 ms, average: 2501.42 ms/ques, entity detection: 403.49 ms/ques (linking tool: 324.94 ms/ques), relation detection: 832.74 ms/ques (similarity: 689.79 ms/ques), EDG generation: 47.99 ms/ques [INFO] Entity Linking P: 0.7529940119760479, R: 0.7043413173652695, micro F1: 0.7198103792415179, macro F1: 0.72785553699996 [INFO] Relation Linking P: 0.5971806387225548, R: 0.5441616766467066, micro F1: 0.5589321357285434, macro F1: 0.5694397084069919 [INFO] Type Linking P: 0.5494011976047904, R: 0.5494011976047904, micro F1: 0.5494011976047904, macro F1: 0.5494011976047904 @@ -9383,8 +9383,8 @@ Question 668: Is the Hyundai Lavita assembled in Kulim? [INFO] Golden Answer: [true] [INFO] Predicted Answer: [true] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 669, P: 0.511, R: 0.561, macro F1: 0.516, macro F1*: 0.535 -[INFO] Current time: 3047 ms, average: 2766.22 ms/ques, entity detection: 454.03 ms/ques (linking tool: 361.58 ms/ques), relation detection: 899.50 ms/ques (similarity: 700.50 ms/ques), EDG generation: 48.03 ms/ques +[INFO] Cumulative metrics, sample: 669, P: 0.511, R: 0.561, macro F1: 0.535 +[INFO] Current question consumed time: 2593 ms, average: 2501.56 ms/ques, entity detection: 403.41 ms/ques (linking tool: 324.74 ms/ques), relation detection: 833.99 ms/ques (similarity: 690.28 ms/ques), EDG generation: 47.94 ms/ques [INFO] Entity Linking P: 0.7526158445440957, R: 0.7040358744394619, micro F1: 0.7194818136522182, macro F1: 0.7275157778969094 [INFO] Relation Linking P: 0.5962879920279023, R: 0.5433482810164425, micro F1: 0.5580966616841061, macro F1: 0.5685885279758902 [INFO] Type Linking P: 0.5500747384155455, R: 0.5500747384155455, micro F1: 0.5500747384155455, macro F1: 0.5500747384155455 @@ -9397,8 +9397,8 @@ Question 669: What is the programme format of WWTR ? [INFO] Golden Answer: [http://dbpedia.org/resource/South_Asia] [INFO] Predicted Answer: [http://dbpedia.org/resource/South_Asia] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 670, P: 0.512, R: 0.562, macro F1: 0.517, macro F1*: 0.536 -[INFO] Current time: 395 ms, average: 2762.68 ms/ques, entity detection: 453.41 ms/ques (linking tool: 361.08 ms/ques), relation detection: 898.47 ms/ques (similarity: 699.64 ms/ques), EDG generation: 47.97 ms/ques +[INFO] Cumulative metrics, sample: 670, P: 0.512, R: 0.562, macro F1: 0.536 +[INFO] Current question consumed time: 351 ms, average: 2498.35 ms/ques, entity detection: 402.86 ms/ques (linking tool: 324.29 ms/ques), relation detection: 833.03 ms/ques (similarity: 689.44 ms/ques), EDG generation: 47.88 ms/ques [INFO] Entity Linking P: 0.7529850746268657, R: 0.7044776119402985, micro F1: 0.7199004975124388, macro F1: 0.7279241274426638 [INFO] Relation Linking P: 0.5968905472636815, R: 0.5440298507462686, micro F1: 0.5587562189054731, macro F1: 0.5692356380096681 [INFO] Type Linking P: 0.5507462686567164, R: 0.5507462686567164, micro F1: 0.5507462686567164, macro F1: 0.5507462686567164 @@ -9411,8 +9411,8 @@ Question 670: list the founders of Bream Bay ? [INFO] Golden Answer: [http://dbpedia.org/resource/James_Cook] [INFO] Predicted Answer: [http://dbpedia.org/resource/James_Cook] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 671, P: 0.513, R: 0.562, macro F1: 0.518, macro F1*: 0.536 -[INFO] Current time: 1073 ms, average: 2760.16 ms/ques, entity detection: 452.98 ms/ques (linking tool: 360.68 ms/ques), relation detection: 897.82 ms/ques (similarity: 698.99 ms/ques), EDG generation: 47.92 ms/ques +[INFO] Cumulative metrics, sample: 671, P: 0.513, R: 0.562, macro F1: 0.536 +[INFO] Current question consumed time: 898 ms, average: 2495.96 ms/ques, entity detection: 402.44 ms/ques (linking tool: 323.91 ms/ques), relation detection: 832.41 ms/ques (similarity: 688.81 ms/ques), EDG generation: 47.82 ms/ques [INFO] Entity Linking P: 0.7533532041728763, R: 0.7049180327868853, micro F1: 0.7203179334326886, macro F1: 0.7283312530889535 [INFO] Relation Linking P: 0.5974913065076999, R: 0.5447093889716841, micro F1: 0.5594138102334829, macro F1: 0.5698808025101167 [INFO] Type Linking P: 0.5514157973174366, R: 0.5514157973174366, micro F1: 0.5514157973174366, macro F1: 0.5514157973174366 @@ -9425,8 +9425,8 @@ Question 671: Is Essex the Ceremonial County of West Tilbury? [INFO] Golden Answer: [true] [INFO] Predicted Answer: [true] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 672, P: 0.513, R: 0.563, macro F1: 0.519, macro F1*: 0.537 -[INFO] Current time: 1355 ms, average: 2758.07 ms/ques, entity detection: 453.00 ms/ques (linking tool: 360.65 ms/ques), relation detection: 896.72 ms/ques (similarity: 698.09 ms/ques), EDG generation: 47.86 ms/ques +[INFO] Cumulative metrics, sample: 672, P: 0.513, R: 0.563, macro F1: 0.537 +[INFO] Current question consumed time: 1108 ms, average: 2493.90 ms/ques, entity detection: 402.40 ms/ques (linking tool: 323.84 ms/ques), relation detection: 831.38 ms/ques (similarity: 687.92 ms/ques), EDG generation: 47.77 ms/ques [INFO] Entity Linking P: 0.7537202380952381, R: 0.7053571428571429, micro F1: 0.720734126984128, macro F1: 0.7287371603409339 [INFO] Relation Linking P: 0.5980902777777777, R: 0.5453869047619048, micro F1: 0.560069444444445, macro F1: 0.5705240302931714 [INFO] Type Linking P: 0.5520833333333334, R: 0.5520833333333334, micro F1: 0.5520833333333334, macro F1: 0.5520833333333334 @@ -9439,8 +9439,8 @@ Question 672: What is the total number of participant in conflict of SFOR? [INFO] Golden Answer: [87] [INFO] Predicted Answer: [36] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 673, P: 0.512, R: 0.562, macro F1: 0.518, macro F1*: 0.536 -[INFO] Current time: 343 ms, average: 2754.48 ms/ques, entity detection: 452.41 ms/ques (linking tool: 360.17 ms/ques), relation detection: 895.50 ms/ques (similarity: 697.12 ms/ques), EDG generation: 47.84 ms/ques +[INFO] Cumulative metrics, sample: 673, P: 0.512, R: 0.562, macro F1: 0.536 +[INFO] Current question consumed time: 286 ms, average: 2490.62 ms/ques, entity detection: 401.87 ms/ques (linking tool: 323.41 ms/ques), relation detection: 830.24 ms/ques (similarity: 686.96 ms/ques), EDG generation: 47.74 ms/ques [INFO] Entity Linking P: 0.7540861812778603, R: 0.7057949479940565, micro F1: 0.7211490837048053, macro F1: 0.7291418546707046 [INFO] Relation Linking P: 0.5986874690440812, R: 0.5453194650817236, micro F1: 0.56022783556216, macro F1: 0.5707586564931558 [INFO] Type Linking P: 0.5527488855869243, R: 0.5527488855869243, micro F1: 0.5527488855869243, macro F1: 0.5527488855869243 @@ -9453,8 +9453,8 @@ Question 673: Name the people whose academic advisor has also mentored the thesi [INFO] Golden Answer: [http://dbpedia.org/resource/Sigmund_Freud, http://dbpedia.org/resource/Kazimierz_Twardowski] [INFO] Predicted Answer: [http://dbpedia.org/resource/Friedrich_Adolf_Trendelenburg] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 674, P: 0.512, R: 0.561, macro F1: 0.517, macro F1*: 0.535 -[INFO] Current time: 2024 ms, average: 2753.40 ms/ques, entity detection: 451.95 ms/ques (linking tool: 359.77 ms/ques), relation detection: 895.25 ms/ques (similarity: 697.08 ms/ques), EDG generation: 47.94 ms/ques +[INFO] Cumulative metrics, sample: 674, P: 0.512, R: 0.561, macro F1: 0.535 +[INFO] Current question consumed time: 1717 ms, average: 2489.47 ms/ques, entity detection: 401.44 ms/ques (linking tool: 323.03 ms/ques), relation detection: 829.96 ms/ques (similarity: 686.91 ms/ques), EDG generation: 47.84 ms/ques [INFO] Entity Linking P: 0.7544510385756676, R: 0.7062314540059347, micro F1: 0.7215628090999021, macro F1: 0.7295453415175576 [INFO] Relation Linking P: 0.5985410484668644, R: 0.5452522255192879, micro F1: 0.560138476755688, macro F1: 0.5706552856423894 [INFO] Type Linking P: 0.5519287833827893, R: 0.5519287833827893, micro F1: 0.5519287833827893, macro F1: 0.5519287833827893 @@ -9467,8 +9467,8 @@ Question 674: Where is Denver Broncos located ? [INFO] Golden Answer: [http://dbpedia.org/resource/Colorado, http://dbpedia.org/resource/Denver, http://dbpedia.org/resource/Sports_Authority_Field_at_Mile_High] [INFO] Predicted Answer: [http://dbpedia.org/resource/Colorado, http://dbpedia.org/resource/Denver, http://dbpedia.org/resource/Sports_Authority_Field_at_Mile_High] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 675, P: 0.512, R: 0.562, macro F1: 0.518, macro F1*: 0.536 -[INFO] Current time: 1910 ms, average: 2752.15 ms/ques, entity detection: 452.05 ms/ques (linking tool: 359.80 ms/ques), relation detection: 894.79 ms/ques (similarity: 696.50 ms/ques), EDG generation: 47.87 ms/ques +[INFO] Cumulative metrics, sample: 675, P: 0.512, R: 0.562, macro F1: 0.536 +[INFO] Current question consumed time: 1569 ms, average: 2488.11 ms/ques, entity detection: 401.45 ms/ques (linking tool: 323.01 ms/ques), relation detection: 829.51 ms/ques (similarity: 686.34 ms/ques), EDG generation: 47.78 ms/ques [INFO] Entity Linking P: 0.7548148148148148, R: 0.7066666666666667, micro F1: 0.7219753086419763, macro F1: 0.7299476262882244 [INFO] Relation Linking P: 0.5991358024691358, R: 0.5459259259259259, micro F1: 0.5607901234567907, macro F1: 0.5712945592492762 [INFO] Type Linking P: 0.5525925925925926, R: 0.5525925925925926, micro F1: 0.5525925925925926, macro F1: 0.5525925925925926 @@ -9481,8 +9481,8 @@ Question 675: Which serving railway line of the Daund Junction railway station i [INFO] Golden Answer: [http://dbpedia.org/resource/Mumbai] [INFO] Predicted Answer: [http://dbpedia.org/resource/HMNB_Portsmouth, http://dbpedia.org/resource/Maharashtra, http://dbpedia.org/resource/Mumbai] [INFO] P: 0.3333333333333333, R: 1.0, F: 0.5 -[INFO] Cumulative metrics, sample: 676, P: 0.512, R: 0.563, macro F1: 0.518, macro F1*: 0.536 -[INFO] Current time: 1693 ms, average: 2750.58 ms/ques, entity detection: 451.92 ms/ques (linking tool: 359.65 ms/ques), relation detection: 893.92 ms/ques (similarity: 695.75 ms/ques), EDG generation: 47.86 ms/ques +[INFO] Cumulative metrics, sample: 676, P: 0.512, R: 0.563, macro F1: 0.536 +[INFO] Current question consumed time: 1437 ms, average: 2486.55 ms/ques, entity detection: 401.34 ms/ques (linking tool: 322.87 ms/ques), relation detection: 828.70 ms/ques (similarity: 685.61 ms/ques), EDG generation: 47.76 ms/ques [INFO] Entity Linking P: 0.7551775147928994, R: 0.7063609467455622, micro F1: 0.7218934911242614, macro F1: 0.7299539743190474 [INFO] Relation Linking P: 0.5997287968441815, R: 0.5458579881656804, micro F1: 0.5609467455621308, macro F1: 0.5715267647532628 [INFO] Type Linking P: 0.5532544378698225, R: 0.5532544378698225, micro F1: 0.5532544378698225, macro F1: 0.5532544378698225 @@ -9495,8 +9495,8 @@ Question 676: Which river flows through Hudson Highlands State park and also thr [INFO] Golden Answer: [http://dbpedia.org/resource/Hudson_River] [INFO] Predicted Answer: [http://dbpedia.org/resource/Hudson_River] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 677, P: 0.513, R: 0.563, macro F1: 0.519, macro F1*: 0.537 -[INFO] Current time: 7656 ms, average: 2757.83 ms/ques, entity detection: 452.11 ms/ques (linking tool: 359.69 ms/ques), relation detection: 896.83 ms/ques (similarity: 699.42 ms/ques), EDG generation: 47.85 ms/ques +[INFO] Cumulative metrics, sample: 677, P: 0.513, R: 0.563, macro F1: 0.537 +[INFO] Current question consumed time: 6763 ms, average: 2492.87 ms/ques, entity detection: 401.49 ms/ques (linking tool: 322.89 ms/ques), relation detection: 831.22 ms/ques (similarity: 689.24 ms/ques), EDG generation: 47.75 ms/ques [INFO] Entity Linking P: 0.7555391432791728, R: 0.7060561299852289, micro F1: 0.7218119153126549, macro F1: 0.7299600009852342 [INFO] Relation Linking P: 0.6003200393894632, R: 0.5465288035450517, micro F1: 0.5615952732644024, macro F1: 0.5721629225907947 [INFO] Type Linking P: 0.552437223042836, R: 0.552437223042836, micro F1: 0.552437223042836, macro F1: 0.552437223042836 @@ -9509,8 +9509,8 @@ Question 677: What is the university to which Erica Frank went, and where Ivan T [INFO] Golden Answer: [http://dbpedia.org/resource/Stanford_University] [INFO] Predicted Answer: [http://dbpedia.org/resource/Centers_for_Disease_Control_and_Prevention, http://dbpedia.org/resource/Stanford_University, http://dbpedia.org/resource/Yale_University, http://dbpedia.org/resource/University_of_British_Columbia] [INFO] P: 0.25, R: 1.0, F: 0.4 -[INFO] Cumulative metrics, sample: 678, P: 0.513, R: 0.564, macro F1: 0.518, macro F1*: 0.537 -[INFO] Current time: 1123 ms, average: 2755.42 ms/ques, entity detection: 451.69 ms/ques (linking tool: 359.29 ms/ques), relation detection: 895.80 ms/ques (similarity: 698.67 ms/ques), EDG generation: 47.85 ms/ques +[INFO] Cumulative metrics, sample: 678, P: 0.513, R: 0.564, macro F1: 0.537 +[INFO] Current question consumed time: 944 ms, average: 2490.58 ms/ques, entity detection: 401.11 ms/ques (linking tool: 322.53 ms/ques), relation detection: 830.26 ms/ques (similarity: 688.51 ms/ques), EDG generation: 47.76 ms/ques [INFO] Entity Linking P: 0.7558997050147492, R: 0.7057522123893806, micro F1: 0.7217305801376609, macro F1: 0.7299657090808427 [INFO] Relation Linking P: 0.6009095378564404, R: 0.5464601769911505, micro F1: 0.5617502458210428, macro F1: 0.5723928881220641 [INFO] Type Linking P: 0.551622418879056, R: 0.551622418879056, micro F1: 0.551622418879056, macro F1: 0.551622418879056 @@ -9523,8 +9523,8 @@ Question 678: How many movies have been directed by Orson Welles ? [INFO] Golden Answer: [35] [INFO] Predicted Answer: [35, 40] [INFO] P: 0.5, R: 1.0, F: 0.6666666666666666 -[INFO] Cumulative metrics, sample: 679, P: 0.512, R: 0.565, macro F1: 0.519, macro F1*: 0.537 -[INFO] Current time: 2389 ms, average: 2754.88 ms/ques, entity detection: 451.52 ms/ques (linking tool: 359.09 ms/ques), relation detection: 895.66 ms/ques (similarity: 698.86 ms/ques), EDG generation: 47.81 ms/ques +[INFO] Cumulative metrics, sample: 679, P: 0.512, R: 0.565, macro F1: 0.537 +[INFO] Current question consumed time: 2028 ms, average: 2489.90 ms/ques, entity detection: 400.89 ms/ques (linking tool: 322.31 ms/ques), relation detection: 830.09 ms/ques (similarity: 688.71 ms/ques), EDG generation: 47.71 ms/ques [INFO] Entity Linking P: 0.7562592047128129, R: 0.7061855670103093, micro F1: 0.7221404025527748, macro F1: 0.7303651332523541 [INFO] Relation Linking P: 0.6014972999509082, R: 0.5471281296023565, micro F1: 0.5623956799214537, macro F1: 0.5730259564443101 [INFO] Type Linking P: 0.5508100147275405, R: 0.5508100147275405, micro F1: 0.5508100147275405, macro F1: 0.5508100147275405 @@ -9537,8 +9537,8 @@ Question 679: Which borough is common for Duddeston and Bordesley railway statio [INFO] Golden Answer: [http://dbpedia.org/resource/Birmingham] [INFO] Predicted Answer: [http://dbpedia.org/resource/Birmingham] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 680, P: 0.513, R: 0.565, macro F1: 0.519, macro F1*: 0.538 -[INFO] Current time: 2261 ms, average: 2754.15 ms/ques, entity detection: 451.32 ms/ques (linking tool: 358.84 ms/ques), relation detection: 895.22 ms/ques (similarity: 698.86 ms/ques), EDG generation: 47.78 ms/ques +[INFO] Cumulative metrics, sample: 680, P: 0.513, R: 0.565, macro F1: 0.538 +[INFO] Current question consumed time: 1779 ms, average: 2488.86 ms/ques, entity detection: 400.62 ms/ques (linking tool: 322.04 ms/ques), relation detection: 829.64 ms/ques (similarity: 688.70 ms/ques), EDG generation: 47.67 ms/ques [INFO] Entity Linking P: 0.7566176470588235, R: 0.7066176470588236, micro F1: 0.7225490196078442, macro F1: 0.7307633757020395 [INFO] Relation Linking P: 0.6020833333333333, R: 0.5477941176470589, micro F1: 0.5630392156862751, macro F1: 0.5736571459021635 [INFO] Type Linking P: 0.5514705882352942, R: 0.5514705882352942, micro F1: 0.5514705882352942, macro F1: 0.5514705882352942 @@ -9551,8 +9551,8 @@ Question 680: Does Mauritius follow a parliamentary republic? [INFO] Golden Answer: [true] [INFO] Predicted Answer: [true] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 681, P: 0.514, R: 0.566, macro F1: 0.520, macro F1*: 0.539 -[INFO] Current time: 1418 ms, average: 2752.19 ms/ques, entity detection: 451.40 ms/ques (linking tool: 358.86 ms/ques), relation detection: 893.99 ms/ques (similarity: 697.97 ms/ques), EDG generation: 47.72 ms/ques +[INFO] Cumulative metrics, sample: 681, P: 0.514, R: 0.566, macro F1: 0.539 +[INFO] Current question consumed time: 1122 ms, average: 2486.85 ms/ques, entity detection: 400.59 ms/ques (linking tool: 321.98 ms/ques), relation detection: 828.50 ms/ques (similarity: 687.81 ms/ques), EDG generation: 47.61 ms/ques [INFO] Entity Linking P: 0.7569750367107195, R: 0.7070484581497798, micro F1: 0.7229564366128254, macro F1: 0.7311604416774553 [INFO] Relation Linking P: 0.6026676456191874, R: 0.5484581497797357, micro F1: 0.5636808614782189, macro F1: 0.5742864649017111 [INFO] Type Linking P: 0.5521292217327459, R: 0.5521292217327459, micro F1: 0.5521292217327459, macro F1: 0.5521292217327459 @@ -9565,8 +9565,8 @@ Question 681: Who is the performer of the album whose subsequent work is Willie [INFO] Golden Answer: [http://dbpedia.org/resource/Willie_Nelson] [INFO] Predicted Answer: [http://dbpedia.org/resource/Willie_Nelson] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 682, P: 0.515, R: 0.567, macro F1: 0.521, macro F1*: 0.539 -[INFO] Current time: 1687 ms, average: 2750.63 ms/ques, entity detection: 451.37 ms/ques (linking tool: 358.89 ms/ques), relation detection: 893.35 ms/ques (similarity: 697.46 ms/ques), EDG generation: 47.69 ms/ques +[INFO] Cumulative metrics, sample: 682, P: 0.515, R: 0.567, macro F1: 0.539 +[INFO] Current question consumed time: 1359 ms, average: 2485.20 ms/ques, entity detection: 400.49 ms/ques (linking tool: 321.93 ms/ques), relation detection: 827.87 ms/ques (similarity: 687.33 ms/ques), EDG generation: 47.59 ms/ques [INFO] Entity Linking P: 0.7573313782991202, R: 0.7074780058651027, micro F1: 0.7233626588465308, macro F1: 0.731556336395046 [INFO] Relation Linking P: 0.6032502443792765, R: 0.5491202346041055, micro F1: 0.5643206256109488, macro F1: 0.574913921798424 [INFO] Type Linking P: 0.5513196480938416, R: 0.5513196480938416, micro F1: 0.5513196480938416, macro F1: 0.5513196480938416 @@ -9579,8 +9579,8 @@ Question 682: What is the birthplace of Ludovic Boi which is also the residence [INFO] Golden Answer: [http://dbpedia.org/resource/Mauritius] [INFO] Predicted Answer: [http://dbpedia.org/resource/Mauritius] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 683, P: 0.515, R: 0.567, macro F1: 0.521, macro F1*: 0.540 -[INFO] Current time: 1557 ms, average: 2748.88 ms/ques, entity detection: 451.23 ms/ques (linking tool: 358.69 ms/ques), relation detection: 892.48 ms/ques (similarity: 696.69 ms/ques), EDG generation: 47.71 ms/ques +[INFO] Cumulative metrics, sample: 683, P: 0.515, R: 0.567, macro F1: 0.540 +[INFO] Current question consumed time: 1240 ms, average: 2483.37 ms/ques, entity detection: 400.30 ms/ques (linking tool: 321.72 ms/ques), relation detection: 827.03 ms/ques (similarity: 686.57 ms/ques), EDG generation: 47.60 ms/ques [INFO] Entity Linking P: 0.7576866764275256, R: 0.7079062957540263, micro F1: 0.723767691556858, macro F1: 0.7319510650403769 [INFO] Relation Linking P: 0.6038311371400683, R: 0.5497803806734992, micro F1: 0.5649585163494394, macro F1: 0.5755395248975458 [INFO] Type Linking P: 0.5519765739385066, R: 0.5519765739385066, micro F1: 0.5519765739385066, macro F1: 0.5519765739385066 @@ -9593,8 +9593,8 @@ Question 683: Who is the producer of album, which has the song Eclipse ? [INFO] Golden Answer: [http://dbpedia.org/resource/Pink_Floyd] [INFO] Predicted Answer: [http://dbpedia.org/resource/Clash_(film), http://dbpedia.org/resource/The_Rebel_(2007_film), http://dbpedia.org/resource/Once_Upon_a_Time_in_Vietnam] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 684, P: 0.515, R: 0.566, macro F1: 0.521, macro F1*: 0.539 -[INFO] Current time: 1589 ms, average: 2747.18 ms/ques, entity detection: 450.82 ms/ques (linking tool: 358.30 ms/ques), relation detection: 891.98 ms/ques (similarity: 696.34 ms/ques), EDG generation: 47.68 ms/ques +[INFO] Cumulative metrics, sample: 684, P: 0.515, R: 0.566, macro F1: 0.539 +[INFO] Current question consumed time: 1329 ms, average: 2481.69 ms/ques, entity detection: 399.92 ms/ques (linking tool: 321.35 ms/ques), relation detection: 826.55 ms/ques (similarity: 686.23 ms/ques), EDG generation: 47.57 ms/ques [INFO] Entity Linking P: 0.756578947368421, R: 0.7068713450292398, micro F1: 0.7227095516569211, macro F1: 0.7308809611441192 [INFO] Relation Linking P: 0.6036793372319688, R: 0.5497076023391813, micro F1: 0.5648635477582852, macro F1: 0.5754306896779641 [INFO] Type Linking P: 0.5511695906432749, R: 0.5511695906432749, micro F1: 0.5511695906432749, macro F1: 0.5511695906432749 @@ -9607,8 +9607,8 @@ Question 684: Which parties have their members live in the PM House, Colombo? [INFO] Golden Answer: [http://dbpedia.org/resource/United_National_Party, http://dbpedia.org/resource/United_National_Front_(Sri_Lanka)] [INFO] Predicted Answer: [] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 685, P: 0.514, R: 0.566, macro F1: 0.520, macro F1*: 0.538 -[INFO] Current time: 958 ms, average: 2744.57 ms/ques, entity detection: 450.16 ms/ques (linking tool: 357.77 ms/ques), relation detection: 890.98 ms/ques (similarity: 695.53 ms/ques), EDG generation: 47.68 ms/ques +[INFO] Cumulative metrics, sample: 685, P: 0.514, R: 0.566, macro F1: 0.538 +[INFO] Current question consumed time: 828 ms, average: 2479.27 ms/ques, entity detection: 399.33 ms/ques (linking tool: 320.88 ms/ques), relation detection: 825.63 ms/ques (similarity: 685.44 ms/ques), EDG generation: 47.57 ms/ques [INFO] Entity Linking P: 0.7554744525547445, R: 0.7058394160583942, micro F1: 0.7216545012165461, macro F1: 0.7298139816387992 [INFO] Relation Linking P: 0.6027980535279804, R: 0.5489051094890511, micro F1: 0.5640389294403899, macro F1: 0.5745906448755146 [INFO] Type Linking P: 0.5503649635036496, R: 0.5503649635036496, micro F1: 0.5503649635036496, macro F1: 0.5503649635036496 @@ -9621,8 +9621,8 @@ Question 685: Which awards have been given to scientists that graduated from NKU [INFO] Golden Answer: [http://dbpedia.org/resource/Italian_Physical_Society, http://dbpedia.org/resource/Alexander_S._Onassis_Foundation, http://dbpedia.org/resource/Institute_of_Electrical_and_Electronics_Engineers, http://dbpedia.org/resource/American_Physical_Society, http://dbpedia.org/resource/Fulbright_Program, http://dbpedia.org/resource/University_of_California,_Davis, http://dbpedia.org/resource/Washington_Academy_of_Sciences, http://dbpedia.org/resource/Lasker-DeBakey_Clinical_Medical_Research_Award, http://dbpedia.org/resource/International_Statistical_Institute, http://dbpedia.org/resource/Jacob_Wolfowitz, http://dbpedia.org/resource/Institute_for_Operations_Research_and_the_Management_Sciences] [INFO] Predicted Answer: [http://dbpedia.org/resource/Operations_research, http://dbpedia.org/resource/Institute_for_Operations_Research_and_the_Management_Sciences, http://dbpedia.org/resource/National_Academy_of_Engineering, http://dbpedia.org/resource/American_Automatic_Control_Council] [INFO] P: 0.25, R: 0.09090909090909091, F: 0.13333333333333333 -[INFO] Cumulative metrics, sample: 686, P: 0.513, R: 0.565, macro F1: 0.519, macro F1*: 0.538 -[INFO] Current time: 8259 ms, average: 2752.61 ms/ques, entity detection: 450.73 ms/ques (linking tool: 358.41 ms/ques), relation detection: 894.33 ms/ques (similarity: 699.46 ms/ques), EDG generation: 47.64 ms/ques +[INFO] Cumulative metrics, sample: 686, P: 0.513, R: 0.565, macro F1: 0.538 +[INFO] Current question consumed time: 7232 ms, average: 2486.20 ms/ques, entity detection: 399.65 ms/ques (linking tool: 321.26 ms/ques), relation detection: 828.61 ms/ques (similarity: 689.57 ms/ques), EDG generation: 47.53 ms/ques [INFO] Entity Linking P: 0.7543731778425656, R: 0.7048104956268222, micro F1: 0.7206025267249767, macro F1: 0.7287501128609002 [INFO] Relation Linking P: 0.6033770651117589, R: 0.5495626822157434, micro F1: 0.5646744412050541, macro F1: 0.5752139590276256 [INFO] Type Linking P: 0.5495626822157434, R: 0.5495626822157434, micro F1: 0.5495626822157434, macro F1: 0.5495626822157434 @@ -9635,8 +9635,8 @@ Question 686: Ricky Gervais is the executive producer of the series having how m [INFO] Golden Answer: [19] [INFO] Predicted Answer: [18101] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 687, P: 0.513, R: 0.564, macro F1: 0.519, macro F1*: 0.537 -[INFO] Current time: 2885 ms, average: 2752.80 ms/ques, entity detection: 450.08 ms/ques (linking tool: 357.89 ms/ques), relation detection: 893.45 ms/ques (similarity: 698.73 ms/ques), EDG generation: 47.73 ms/ques +[INFO] Cumulative metrics, sample: 687, P: 0.513, R: 0.564, macro F1: 0.537 +[INFO] Current question consumed time: 2564 ms, average: 2486.31 ms/ques, entity detection: 399.07 ms/ques (linking tool: 320.80 ms/ques), relation detection: 827.79 ms/ques (similarity: 688.85 ms/ques), EDG generation: 47.64 ms/ques [INFO] Entity Linking P: 0.7532751091703057, R: 0.7037845705967977, micro F1: 0.7195536147501224, macro F1: 0.7276893412264591 [INFO] Relation Linking P: 0.6024987869966035, R: 0.5487627365356623, micro F1: 0.5638524987869972, macro F1: 0.5743766752444704 [INFO] Type Linking P: 0.5487627365356623, R: 0.5487627365356623, micro F1: 0.5487627365356623, macro F1: 0.5487627365356623 @@ -9649,8 +9649,8 @@ Question 687: Did Jerry Rice study in Oktoc Mississippi? [INFO] Golden Answer: [true] [INFO] Predicted Answer: [false] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 688, P: 0.512, R: 0.563, macro F1: 0.518, macro F1*: 0.536 -[INFO] Current time: 13126 ms, average: 2767.88 ms/ques, entity detection: 450.26 ms/ques (linking tool: 357.90 ms/ques), relation detection: 896.85 ms/ques (similarity: 700.28 ms/ques), EDG generation: 47.69 ms/ques +[INFO] Cumulative metrics, sample: 688, P: 0.512, R: 0.563, macro F1: 0.536 +[INFO] Current question consumed time: 10526 ms, average: 2498.00 ms/ques, entity detection: 399.14 ms/ques (linking tool: 320.76 ms/ques), relation detection: 830.78 ms/ques (similarity: 690.40 ms/ques), EDG generation: 47.61 ms/ques [INFO] Entity Linking P: 0.7521802325581395, R: 0.7027616279069767, micro F1: 0.7185077519379856, macro F1: 0.7266316532304903 [INFO] Relation Linking P: 0.6016230620155039, R: 0.5479651162790697, micro F1: 0.5630329457364347, macro F1: 0.5735418254258012 [INFO] Type Linking P: 0.5494186046511628, R: 0.5494186046511628, micro F1: 0.5494186046511628, macro F1: 0.5494186046511628 @@ -9663,8 +9663,8 @@ Question 688: Cable stayed bridges have been used to cross over how many things? [INFO] Golden Answer: [153] [INFO] Predicted Answer: [6] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 689, P: 0.511, R: 0.562, macro F1: 0.517, macro F1*: 0.536 -[INFO] Current time: 1210 ms, average: 2765.62 ms/ques, entity detection: 449.76 ms/ques (linking tool: 357.45 ms/ques), relation detection: 896.06 ms/ques (similarity: 699.75 ms/ques), EDG generation: 47.66 ms/ques +[INFO] Cumulative metrics, sample: 689, P: 0.511, R: 0.562, macro F1: 0.536 +[INFO] Current question consumed time: 1048 ms, average: 2495.89 ms/ques, entity detection: 398.69 ms/ques (linking tool: 320.36 ms/ques), relation detection: 830.03 ms/ques (similarity: 689.87 ms/ques), EDG generation: 47.57 ms/ques [INFO] Entity Linking P: 0.7510885341074021, R: 0.7017416545718432, micro F1: 0.7174649250120959, macro F1: 0.7255770354464114 [INFO] Relation Linking P: 0.6007498790517658, R: 0.5471698113207547, micro F1: 0.5622157716497345, macro F1: 0.5727093989738045 [INFO] Type Linking P: 0.5500725689404935, R: 0.5500725689404935, micro F1: 0.5500725689404935, macro F1: 0.5500725689404935 @@ -9677,8 +9677,8 @@ Question 689: Is north caucasian emirate have an islamic government? [INFO] Golden Answer: [true] [INFO] Predicted Answer: [false] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 690, P: 0.510, R: 0.562, macro F1: 0.516, macro F1*: 0.535 -[INFO] Current time: 1760 ms, average: 2764.16 ms/ques, entity detection: 450.09 ms/ques (linking tool: 357.72 ms/ques), relation detection: 894.95 ms/ques (similarity: 698.84 ms/ques), EDG generation: 47.60 ms/ques +[INFO] Cumulative metrics, sample: 690, P: 0.510, R: 0.562, macro F1: 0.535 +[INFO] Current question consumed time: 1523 ms, average: 2494.48 ms/ques, entity detection: 398.97 ms/ques (linking tool: 320.60 ms/ques), relation detection: 828.98 ms/ques (similarity: 688.97 ms/ques), EDG generation: 47.52 ms/ques [INFO] Entity Linking P: 0.75, R: 0.7007246376811594, micro F1: 0.716425120772948, macro F1: 0.7245254745254744 [INFO] Relation Linking P: 0.59987922705314, R: 0.5463768115942029, micro F1: 0.5614009661835755, macro F1: 0.5718793853521033 [INFO] Type Linking P: 0.5507246376811594, R: 0.5507246376811594, micro F1: 0.5507246376811594, macro F1: 0.5507246376811594 @@ -9691,8 +9691,8 @@ Question 690: What are the television shows whose starting music is composed by [INFO] Golden Answer: [http://dbpedia.org/resource/Doctor_Who, http://dbpedia.org/resource/TFI_Friday, http://dbpedia.org/resource/Edward_&_Mrs._Simpson, http://dbpedia.org/resource/That_Was_the_Week_That_Was, http://dbpedia.org/resource/The_Prisoner] [INFO] Predicted Answer: [http://dbpedia.org/resource/Doctor_Who, http://dbpedia.org/resource/TFI_Friday, http://dbpedia.org/resource/Edward_&_Mrs._Simpson, http://dbpedia.org/resource/That_Was_the_Week_That_Was, http://dbpedia.org/resource/The_Prisoner] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 691, P: 0.511, R: 0.562, macro F1: 0.517, macro F1*: 0.536 -[INFO] Current time: 1096 ms, average: 2761.75 ms/ques, entity detection: 449.74 ms/ques (linking tool: 357.38 ms/ques), relation detection: 894.07 ms/ques (similarity: 698.27 ms/ques), EDG generation: 47.57 ms/ques +[INFO] Cumulative metrics, sample: 691, P: 0.511, R: 0.562, macro F1: 0.536 +[INFO] Current question consumed time: 878 ms, average: 2492.14 ms/ques, entity detection: 398.63 ms/ques (linking tool: 320.27 ms/ques), relation detection: 828.14 ms/ques (similarity: 688.39 ms/ques), EDG generation: 47.48 ms/ques [INFO] Entity Linking P: 0.7503617945007236, R: 0.7011577424023154, micro F1: 0.7168355041003387, macro F1: 0.7249258014668006 [INFO] Relation Linking P: 0.6004582730342498, R: 0.5470332850940666, micro F1: 0.5620356970574053, macro F1: 0.5725020970012311 [INFO] Type Linking P: 0.5499276410998553, R: 0.5499276410998553, micro F1: 0.5499276410998553, macro F1: 0.5499276410998553 @@ -9705,8 +9705,8 @@ Question 691: Who influenced the author of The Shooting of Dan McGrew ? [INFO] Golden Answer: [http://dbpedia.org/resource/Robert_Burns] [INFO] Predicted Answer: [http://dbpedia.org/resource/Robert_Burns] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 692, P: 0.512, R: 0.563, macro F1: 0.518, macro F1*: 0.536 -[INFO] Current time: 1515 ms, average: 2759.95 ms/ques, entity detection: 449.38 ms/ques (linking tool: 357.09 ms/ques), relation detection: 893.34 ms/ques (similarity: 697.60 ms/ques), EDG generation: 47.61 ms/ques +[INFO] Cumulative metrics, sample: 692, P: 0.512, R: 0.563, macro F1: 0.536 +[INFO] Current question consumed time: 1275 ms, average: 2490.39 ms/ques, entity detection: 398.29 ms/ques (linking tool: 319.99 ms/ques), relation detection: 827.45 ms/ques (similarity: 687.74 ms/ques), EDG generation: 47.52 ms/ques [INFO] Entity Linking P: 0.7507225433526011, R: 0.7015895953757225, micro F1: 0.7172447013487486, macro F1: 0.7253249647715181 [INFO] Relation Linking P: 0.6010356454720616, R: 0.5476878612716763, micro F1: 0.5626685934489408, macro F1: 0.5731229931034566 [INFO] Type Linking P: 0.5505780346820809, R: 0.5505780346820809, micro F1: 0.5505780346820809, macro F1: 0.5505780346820809 @@ -9719,8 +9719,8 @@ Question 692: Raymond Johansen is the mayor of what city? [INFO] Golden Answer: [http://dbpedia.org/resource/Oslo] [INFO] Predicted Answer: [http://dbpedia.org/resource/Oslo] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 693, P: 0.513, R: 0.564, macro F1: 0.518, macro F1*: 0.537 -[INFO] Current time: 761 ms, average: 2757.06 ms/ques, entity detection: 449.01 ms/ques (linking tool: 356.69 ms/ques), relation detection: 892.23 ms/ques (similarity: 696.80 ms/ques), EDG generation: 47.57 ms/ques +[INFO] Cumulative metrics, sample: 693, P: 0.513, R: 0.564, macro F1: 0.537 +[INFO] Current question consumed time: 598 ms, average: 2487.66 ms/ques, entity detection: 397.91 ms/ques (linking tool: 319.62 ms/ques), relation detection: 826.41 ms/ques (similarity: 686.95 ms/ques), EDG generation: 47.48 ms/ques [INFO] Entity Linking P: 0.7510822510822511, R: 0.702020202020202, micro F1: 0.7176527176527187, macro F1: 0.7257229695164155 [INFO] Relation Linking P: 0.6016113516113516, R: 0.5483405483405484, micro F1: 0.5632996632996639, macro F1: 0.5737420816370937 [INFO] Type Linking P: 0.5497835497835498, R: 0.5497835497835498, micro F1: 0.5497835497835498, macro F1: 0.5497835497835498 @@ -9733,8 +9733,8 @@ Question 693: What region was Gloria Schaffer born in, which also contains Newin [INFO] Golden Answer: [http://dbpedia.org/resource/Connecticut] [INFO] Predicted Answer: [http://dbpedia.org/resource/American-Canadian_Macedonian_Orthodox_Diocese, http://dbpedia.org/resource/Roman_Catholic_Diocese_of_Allentown, http://dbpedia.org/resource/Albanian_Orthodox_Diocese_of_America, http://dbpedia.org/resource/Syrian_Catholic_Eparchy_of_Our_Lady_of_Deliverance_of_Newark, http://dbpedia.org/resource/Roman_Catholic_Diocese_of_Monterey_in_California] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 694, P: 0.512, R: 0.563, macro F1: 0.518, macro F1*: 0.536 -[INFO] Current time: 38768 ms, average: 2808.95 ms/ques, entity detection: 449.12 ms/ques (linking tool: 356.54 ms/ques), relation detection: 916.78 ms/ques (similarity: 705.99 ms/ques), EDG generation: 47.54 ms/ques +[INFO] Cumulative metrics, sample: 694, P: 0.512, R: 0.563, macro F1: 0.536 +[INFO] Current question consumed time: 35811 ms, average: 2535.67 ms/ques, entity detection: 397.96 ms/ques (linking tool: 319.46 ms/ques), relation detection: 847.90 ms/ques (similarity: 696.18 ms/ques), EDG generation: 47.46 ms/ques [INFO] Entity Linking P: 0.75, R: 0.7010086455331412, micro F1: 0.7166186359269943, macro F1: 0.7246772591857 [INFO] Relation Linking P: 0.6014649375600384, R: 0.5482708933717579, micro F1: 0.5632084534101831, macro F1: 0.5736373691695321 [INFO] Type Linking P: 0.5489913544668588, R: 0.5489913544668588, micro F1: 0.5489913544668588, macro F1: 0.5489913544668588 @@ -9747,8 +9747,8 @@ Question 694: List some teams for which someone who has played as a defender, no [INFO] Golden Answer: [http://dbpedia.org/resource/Montreal_Canadiens, http://dbpedia.org/resource/Detroit_Red_Wings, http://dbpedia.org/resource/Vancouver_Canucks, http://dbpedia.org/resource/Florida_Panthers, http://dbpedia.org/resource/Los_Angeles_Kings, http://dbpedia.org/resource/HK_Acroni_Jesenice, http://dbpedia.org/resource/Boston_Blades, http://dbpedia.org/resource/Kölner_Haie, http://dbpedia.org/resource/HC_Litvínov, http://dbpedia.org/resource/Dornbirner_EC, http://dbpedia.org/resource/Asiago_Hockey_1935, http://dbpedia.org/resource/New_Jersey_Devils, http://dbpedia.org/resource/San_Jose_Sharks, http://dbpedia.org/resource/Edmonton_Oilers, http://dbpedia.org/resource/Toronto_Maple_Leafs, http://dbpedia.org/resource/Oulun_Kärpät, http://dbpedia.org/resource/HC_Kometa_Brno, http://dbpedia.org/resource/Frölunda_HC, http://dbpedia.org/resource/Tampa_Bay_Lightning, http://dbpedia.org/resource/Växjö_Lakers, http://dbpedia.org/resource/Lukko, http://dbpedia.org/resource/HV71, http://dbpedia.org/resource/New_York_Rangers, http://dbpedia.org/resource/KalPa, http://dbpedia.org/resource/HC_TPS, http://dbpedia.org/resource/Anaheim_Ducks, http://dbpedia.org/resource/Mississippi_Surge, http://dbpedia.org/resource/HIFK_(ice_hockey), http://dbpedia.org/resource/Philadelphia_Flyers, http://dbpedia.org/resource/Coventry_Blaze, http://dbpedia.org/resource/Washington_Capitals, http://dbpedia.org/resource/Avangard_Omsk, http://dbpedia.org/resource/Thomas_Sabo_Ice_Tigers, http://dbpedia.org/resource/Linköpings_HC, http://dbpedia.org/resource/Krefeld_Pinguine, http://dbpedia.org/resource/Winnipeg_Jets, http://dbpedia.org/resource/Fife_Flyers, http://dbpedia.org/resource/Carolina_Hurricanes, http://dbpedia.org/resource/Utah_Grizzlies, http://dbpedia.org/resource/Minnesota_Wild, http://dbpedia.org/resource/Vienna_Capitals, http://dbpedia.org/resource/Nashville_Predators, http://dbpedia.org/resource/Reading_Royals, http://dbpedia.org/resource/Chicago_Blackhawks, http://dbpedia.org/resource/Arizona_Coyotes, http://dbpedia.org/resource/Colorado_Avalanche, http://dbpedia.org/resource/Färjestad_BK, http://dbpedia.org/resource/Calgary_Flames, http://dbpedia.org/resource/Bridgeport_Sound_Tigers, http://dbpedia.org/resource/Wisconsin_Badgers_women's_ice_hockey, http://dbpedia.org/resource/Calgary_Oval_X-Treme, http://dbpedia.org/resource/St._John's_IceCaps, http://dbpedia.org/resource/Buffalo_Sabres, http://dbpedia.org/resource/SC_Bietigheim-Bissingen, http://dbpedia.org/resource/HK_Nitra, http://dbpedia.org/resource/Ottawa_Senators, http://dbpedia.org/resource/EHC_Basel, http://dbpedia.org/resource/Adler_Mannheim, http://dbpedia.org/resource/EHC_München, http://dbpedia.org/resource/Dinamo_Riga, http://dbpedia.org/resource/HC_Vítkovice_Steel, http://dbpedia.org/resource/Piráti_Chomutov, http://dbpedia.org/resource/Metallurg_Magnitogorsk, http://dbpedia.org/resource/HC_Plzeň, http://dbpedia.org/resource/Guildford_Flames, http://dbpedia.org/resource/KHL_Medveščak_Zagreb, http://dbpedia.org/resource/HC_Dynamo_Moscow, http://dbpedia.org/resource/KooKoo_(Liiga), http://dbpedia.org/resource/Tappara, http://dbpedia.org/resource/Pelican, http://dbpedia.org/resource/Dallas_Stars, http://dbpedia.org/resource/Motor_České_Budějovice, http://dbpedia.org/resource/HC_Sochi, http://dbpedia.org/resource/HC_Slovan_Bratislava, http://dbpedia.org/resource/Ässät, http://dbpedia.org/resource/STS_Sanok, http://dbpedia.org/resource/HDD_Olimpija_Ljubljana, http://dbpedia.org/resource/Espoo_Blues, http://dbpedia.org/resource/Wichita_Thunder, http://dbpedia.org/resource/Pittsburgh_Penguins, http://dbpedia.org/resource/HC_Sibir_Novosibirsk, http://dbpedia.org/resource/Metallurg_Novokuznetsk, http://dbpedia.org/resource/Belfast_Giants, http://dbpedia.org/resource/Nottingham_Panthers, http://dbpedia.org/resource/HC_Ugra, http://dbpedia.org/resource/Düsseldorfer_EG, http://dbpedia.org/resource/Augsburger_Panther, http://dbpedia.org/resource/Stavanger_Oilers, http://dbpedia.org/resource/Djurgårdens_IF_Hockey, http://dbpedia.org/resource/Traktor_Chelyabinsk, http://dbpedia.org/resource/Severstal_Cherepovets, http://dbpedia.org/resource/KooKoo, http://dbpedia.org/resource/PSG_Zlín, http://dbpedia.org/resource/Jokerit, http://dbpedia.org/resource/Amur_Khabarovsk, http://dbpedia.org/resource/HC_Lugano, http://dbpedia.org/resource/Ilves, http://dbpedia.org/resource/Team_Kuortane, http://dbpedia.org/resource/JYP_Jyväskylä, http://dbpedia.org/resource/SKIF_Nizhni_Novgorod, http://dbpedia.org/resource/KS_Cracovia_(ice_hockey), http://dbpedia.org/resource/Luleå_HF, http://dbpedia.org/resource/Atlant_Moscow_Oblast, http://dbpedia.org/resource/Iowa_Wild, http://dbpedia.org/resource/SCL_Tigers, http://dbpedia.org/resource/Salavat_Yulaev_Ufa, http://dbpedia.org/resource/EHC_Wolfsburg, http://dbpedia.org/resource/SC_Reinach, http://dbpedia.org/resource/Minnesota–Duluth_Bulldogs_women's_ice_hockey, http://dbpedia.org/resource/ZSC_Lions, http://dbpedia.org/resource/HC_Davos, http://dbpedia.org/resource/HC_Pustertal_Wölfe, http://dbpedia.org/resource/SG_Cortina, http://dbpedia.org/resource/Barys_Astana, http://dbpedia.org/resource/Missouri_Mavericks, http://dbpedia.org/resource/Slough_Phantoms, http://dbpedia.org/resource/HC_Ajoie, http://dbpedia.org/resource/Kingston_Diamonds, http://dbpedia.org/resource/Perth_Thunder, http://dbpedia.org/resource/Toledo_Walleye, http://dbpedia.org/resource/CBR_Brave, http://dbpedia.org/resource/Grand_Rapids_Griffins, http://dbpedia.org/resource/HC_Donbass, http://dbpedia.org/resource/Hannover_Scorpions, http://dbpedia.org/resource/Vaasan_Sport, http://dbpedia.org/resource/Leksands_IF, http://dbpedia.org/resource/Straubing_Tigers, http://dbpedia.org/resource/HC_Vityaz, http://dbpedia.org/resource/Orli_Znojmo, http://dbpedia.org/resource/Örebro_HK, http://dbpedia.org/resource/Malmö_Redhawks, http://dbpedia.org/resource/HC_Kladno, http://dbpedia.org/resource/HC_Sparta_Praha, http://dbpedia.org/resource/HC_Pardubice, http://dbpedia.org/resource/Herning_Blue_Fox, http://dbpedia.org/resource/Eisbären_Berlin, http://dbpedia.org/resource/HPK, http://dbpedia.org/resource/BK_Mladá_Boleslav, http://dbpedia.org/resource/EC_Red_Bull_Salzburg, http://dbpedia.org/resource/HC_Slavia_Praha, http://dbpedia.org/resource/Genève-Servette_HC, http://dbpedia.org/resource/Columbus_Blue_Jackets, http://dbpedia.org/resource/HC_Spartak_Moscow, http://dbpedia.org/resource/HC_CSKA_Moscow, http://dbpedia.org/resource/Arystan_Temirtau, http://dbpedia.org/resource/HC_Red_Ice, http://dbpedia.org/resource/St._Louis_Blues, http://dbpedia.org/resource/Sydney_Ice_Dogs, http://dbpedia.org/resource/Yertis_Pavlodar, http://dbpedia.org/resource/Tohoku_Free_Blades, http://dbpedia.org/resource/Rapid_City_Rush, http://dbpedia.org/resource/Milwaukee_Admirals, http://dbpedia.org/resource/HC_Oceláři_Třinec, http://dbpedia.org/resource/SC_Bern, http://dbpedia.org/resource/Atlanta_Gladiators, http://dbpedia.org/resource/Étoile_Noire_de_Strasbourg, http://dbpedia.org/resource/Adelaide_Adrenaline, http://dbpedia.org/resource/Timrå_IK, http://dbpedia.org/resource/HC_Lada_Togliatti, http://dbpedia.org/resource/Admiral_Vladivostok, http://dbpedia.org/resource/Torpedo_Nizhny_Novgorod, http://dbpedia.org/resource/Lempäälän_Kisa, http://dbpedia.org/resource/Södertälje_SK, http://dbpedia.org/resource/HCB_South_Tyrol, http://dbpedia.org/resource/Ontario_Reign_(ECHL), http://dbpedia.org/resource/AIK_IF, http://dbpedia.org/resource/Asplöven_HC, http://dbpedia.org/resource/HC_Bílí_Tygři_Liberec, http://dbpedia.org/resource/Ice_Knights_Tbilisi, http://dbpedia.org/resource/ERC_Ingolstadt, http://dbpedia.org/resource/Esbjerg_Energy, http://dbpedia.org/resource/Greenville_Swamp_Rabbits, http://dbpedia.org/resource/Boston_Bruins, http://dbpedia.org/resource/Free_agent, http://dbpedia.org/resource/Sokol_Krasnoyarsk, http://dbpedia.org/resource/Sarnia_Sting, http://dbpedia.org/resource/Ducs_d'Angers, http://dbpedia.org/resource/Wilkes-Barre/Scranton_Penguins, http://dbpedia.org/resource/EC_KAC, http://dbpedia.org/resource/Lawrenceville_School, http://dbpedia.org/resource/Rögle_BK, http://dbpedia.org/resource/New_York_Islanders, http://dbpedia.org/resource/Lake_Erie_Monsters, http://dbpedia.org/resource/Springfield_Falcons, http://dbpedia.org/resource/Boston_Pride, http://dbpedia.org/resource/Edinburgh_Capitals, http://dbpedia.org/resource/Lokomotiv_Yaroslavl, http://dbpedia.org/resource/Dauphins_d'Épinal, http://dbpedia.org/resource/Sparta_Warriors, http://dbpedia.org/resource/Thetford_Mines_Isothermic, http://dbpedia.org/resource/HC_Neftekhimik_Nizhnekamsk, http://dbpedia.org/resource/Akwesasne_Warriors, http://dbpedia.org/resource/Laval_Predators, http://dbpedia.org/resource/Allen_Americans, http://dbpedia.org/resource/Sheffield_Steelers, http://dbpedia.org/resource/Sokil_Kiev, http://dbpedia.org/resource/Odessa_Jackalopes, http://dbpedia.org/resource/HC_Fribourg-Gottéron, http://dbpedia.org/resource/Vålerenga_Ishockey, http://dbpedia.org/resource/Friesland_Flyers, http://dbpedia.org/resource/Alba_Volán_Székesfehérvár, http://dbpedia.org/resource/Iserlohn_Roosters, http://dbpedia.org/resource/Lausanne_HC, http://dbpedia.org/resource/Dragons_de_Rouen, http://dbpedia.org/resource/Fort_Worth_Brahmas, http://dbpedia.org/resource/Buran_Voronezh, http://dbpedia.org/resource/Université_du_Québec_à_Trois-Rivières, http://dbpedia.org/resource/Calgary_Inferno, http://dbpedia.org/resource/North_Dakota_Fighting_Hawks_women's_ice_hockey, http://dbpedia.org/resource/Brampton_Thunder, http://dbpedia.org/resource/Les_Canadiennes, http://dbpedia.org/resource/IF_Sundsvall_Hockey, http://dbpedia.org/resource/Modo_Hockey, http://dbpedia.org/resource/Brynäs_IF, http://dbpedia.org/resource/Anyang_Halla, http://dbpedia.org/resource/İstanbul_Paten_S.K._-_Canterbury_Red_Devils, http://dbpedia.org/resource/İzmir_Büyükşehir_Belediyesi_GSK, http://dbpedia.org/resource/HC_Gherdëina, http://dbpedia.org/resource/Mississippi_RiverKings, http://dbpedia.org/resource/Steaua_Rangers, http://dbpedia.org/resource/HC_Karlovy_Vary, http://dbpedia.org/resource/EV_Zug, http://dbpedia.org/resource/SaiPa, http://dbpedia.org/resource/Las_Vegas_Wranglers, http://dbpedia.org/resource/CSKA_Moscow, http://dbpedia.org/resource/Idaho_Steelheads, http://dbpedia.org/resource/HSC_Csíkszereda, http://dbpedia.org/resource/HC_Dinamo_Minsk, http://dbpedia.org/resource/Tri-City_Americans, http://dbpedia.org/resource/Cardiff_Devils, http://dbpedia.org/resource/Orlando_Solar_Bears_(ECHL), http://dbpedia.org/resource/Ak_Bars_Kazan, http://dbpedia.org/resource/LHC_Les_Lions, http://dbpedia.org/resource/GCK_Lions, http://dbpedia.org/resource/ESV_Kaufbeuren, http://dbpedia.org/resource/Hamburg_Freezers, http://dbpedia.org/resource/Destil_Trappers, http://dbpedia.org/resource/HKm_Zvolen, http://dbpedia.org/resource/Companion_Kyiv, http://dbpedia.org/resource/Providence_Bruins, http://dbpedia.org/resource/Minnesota_Whitecaps, http://dbpedia.org/resource/Minnesota_Golden_Gophers_women's_ice_hockey, http://dbpedia.org/resource/Mora_IK, http://dbpedia.org/resource/South_Carolina_Stingrays, http://dbpedia.org/resource/HC_Košice, http://dbpedia.org/resource/Avtomobilist_Yekaterinburg, http://dbpedia.org/resource/HC_Thurgau, http://dbpedia.org/resource/Almtuna_IS, http://dbpedia.org/resource/Evansville_IceMen, http://dbpedia.org/resource/Budapest_Stars, http://dbpedia.org/resource/China_Dragon, http://dbpedia.org/resource/HK_Poprad, http://dbpedia.org/resource/HC_’05_Banská_Bystrica, http://dbpedia.org/resource/SKA_Saint_Petersburg, http://dbpedia.org/resource/Peliitat_Heinola, http://dbpedia.org/resource/Skellefteå_AIK, http://dbpedia.org/resource/Tulsa_Oilers, http://dbpedia.org/resource/Schwenninger_Wild_Wings, http://dbpedia.org/resource/EHC_Lustenau, http://dbpedia.org/resource/HC_TWK_Innsbruck, http://dbpedia.org/resource/Rochester_Americans, http://dbpedia.org/resource/Graz_99ers, http://dbpedia.org/resource/Kloten_Flyers, http://dbpedia.org/resource/Milenyum_Paten_SK, http://dbpedia.org/resource/HC_Eppan_Pirates, http://dbpedia.org/resource/EC_VSV, http://dbpedia.org/resource/Sheffield_Scimitars, http://dbpedia.org/resource/EHC_Black_Wings_Linz, http://dbpedia.org/resource/Boston_University_Terriers_men's_ice_hockey, http://dbpedia.org/resource/Brampton_Beast, http://dbpedia.org/resource/UNB_Varsity_Reds, http://dbpedia.org/resource/Wheeling_Nailers, http://dbpedia.org/resource/Flint_Generals, http://dbpedia.org/resource/Lørenskog_IK, http://dbpedia.org/resource/HC_Valpellice, http://dbpedia.org/resource/Rockford_IceHogs, http://dbpedia.org/resource/San_Francisco_Bulls, http://dbpedia.org/resource/Frankfurt_Lions, http://dbpedia.org/resource/EHC_Chur, http://dbpedia.org/resource/HC_Villars, http://dbpedia.org/resource/EHC_Olten, http://dbpedia.org/resource/Ritten_Sport, http://dbpedia.org/resource/Manitoba_Moose, http://dbpedia.org/resource/Seibu_Princess_Rabbits, http://dbpedia.org/resource/IF_Björklöven, http://dbpedia.org/resource/Toyota_Cygnus, http://dbpedia.org/resource/Mitsuboshi_Daito_Perigrine, http://dbpedia.org/resource/Mikage_Gretz, http://dbpedia.org/resource/Toronto_Furies, http://dbpedia.org/resource/Indy_Fuel, http://dbpedia.org/resource/HK_Dukla_Trenčín, http://dbpedia.org/resource/ESC_Planegg, http://dbpedia.org/resource/Newcastle_Vipers, http://dbpedia.org/resource/Brandon_Wheat_Kings, http://dbpedia.org/resource/Colorado_Eagles, http://dbpedia.org/resource/Trenton_Titans, http://dbpedia.org/resource/HK_Rīga, http://dbpedia.org/resource/Chicoutimi_Saguenéens, http://dbpedia.org/resource/EC_Oilers_Salzburg, http://dbpedia.org/resource/Utica_Comets, http://dbpedia.org/resource/Lahti_Pelicans, http://dbpedia.org/resource/TUTO_Hockey, http://dbpedia.org/resource/MHC_Martin, http://dbpedia.org/resource/Aalborg_Pirates, http://dbpedia.org/resource/Odense_Bulldogs, http://dbpedia.org/resource/Rapaces_de_Gap, http://dbpedia.org/resource/DAB-Docler, http://dbpedia.org/resource/Prince_George_Cougars, http://dbpedia.org/resource/Nordsjælland_Cobras, http://dbpedia.org/resource/Syracuse_Crunch, http://dbpedia.org/resource/Diables_Rouges_de_Briançon, http://dbpedia.org/resource/Louisiana_IceGators_(SPHL), http://dbpedia.org/resource/Cincinnati_Cyclones, http://dbpedia.org/resource/Nikkō_Ice_Bucks, http://dbpedia.org/resource/Tingsryds_AIF, http://dbpedia.org/resource/Karlskrona_HK, http://dbpedia.org/resource/San_Antonio_Rampage, http://dbpedia.org/resource/HC_Ambrì-Piotta, http://dbpedia.org/resource/Rødovre_Mighty_Bulls, http://dbpedia.org/resource/University_of_Ottawa, http://dbpedia.org/resource/IF_Troja/Ljungby, http://dbpedia.org/resource/Hockey_Milano_Rossoblu, http://dbpedia.org/resource/BIK_Karlskoga, http://dbpedia.org/resource/SønderjyskE_Ishockey, http://dbpedia.org/resource/Melbourne_Ice, http://dbpedia.org/resource/HC_Shakhtyor_Soligorsk, http://dbpedia.org/resource/Minnesota_Golden_Gophers_men's_ice_hockey, http://dbpedia.org/resource/Yunost_Minsk, http://dbpedia.org/resource/EV_Landshut, http://dbpedia.org/resource/Morinville_Kings, http://dbpedia.org/resource/Alleghe_Hockey, http://dbpedia.org/resource/Bakersfield_Condors_(1998–2015), http://dbpedia.org/resource/Binghamton_Senators, http://dbpedia.org/resource/Braehead_Clan, http://dbpedia.org/resource/London_Raiders, http://dbpedia.org/resource/Kazzinc-Torpedo, http://dbpedia.org/resource/Mountfield_HK, http://dbpedia.org/resource/Burlington_Barracudas, http://dbpedia.org/resource/High1, http://dbpedia.org/resource/Mississauga_Steelheads, http://dbpedia.org/resource/Bakersfield_Condors_(AHL), http://dbpedia.org/resource/Stockton_Heat, http://dbpedia.org/resource/Fischtown_Pinguins, http://dbpedia.org/resource/Rapperswil-Jona_Lakers, http://dbpedia.org/resource/New_Hampshire_Wildcats_women's_ice_hockey, http://dbpedia.org/resource/Connecticut_Whale_(NWHL), http://dbpedia.org/resource/Fort_Wayne_Komets, http://dbpedia.org/resource/Toronto_Marlies, http://dbpedia.org/resource/Barrie_Colts, http://dbpedia.org/resource/Storhamar_Ishockey, http://dbpedia.org/resource/Elmira_Jackals, http://dbpedia.org/resource/Florida_Everblades, http://dbpedia.org/resource/Brunflo_IK, http://dbpedia.org/resource/Chicago_Wolves, http://dbpedia.org/resource/Buffalo_Beauts, http://dbpedia.org/resource/Boston_College_Eagles_women's_ice_hockey, http://dbpedia.org/resource/Harvard_Crimson_women's_ice_hockey, http://dbpedia.org/resource/Milton_Keynes_Lightning, http://dbpedia.org/resource/Saskatoon_Blades, http://dbpedia.org/resource/Manchester_Phoenix, http://dbpedia.org/resource/Amstel_Tijgers, http://dbpedia.org/resource/Ravensburg_Towerstars, http://dbpedia.org/resource/Adirondack_Thunder, http://dbpedia.org/resource/Dundas_Real_McCoys, http://dbpedia.org/resource/Melbourne_Mustangs, http://dbpedia.org/resource/EHC_Biel, http://dbpedia.org/resource/Oji_Eagles, http://dbpedia.org/resource/Kongsvinger_Knights, http://dbpedia.org/resource/Yale_Bulldogs_men's_ice_hockey, http://dbpedia.org/resource/Gold_Coast_Blue_Tongues, http://dbpedia.org/resource/Hull_Stingrays, http://dbpedia.org/resource/Boxers_de_Bordeaux, http://dbpedia.org/resource/SC_Riessersee, http://dbpedia.org/resource/HK_DUKLA_Trenčín, http://dbpedia.org/resource/St._Lawrence_Saints_men's_ice_hockey, http://dbpedia.org/resource/IK_Oskarshamn, http://dbpedia.org/resource/Manchester_Storm_(2015–), http://dbpedia.org/resource/Basingstoke_Bison, http://dbpedia.org/resource/SG_Pontebba, http://dbpedia.org/resource/Bossier-Shreveport_Mudbugs, http://dbpedia.org/resource/Boston_University_Terriers_women's_ice_hockey, http://dbpedia.org/resource/McGill_Martlets_ice_hockey, http://dbpedia.org/resource/Saint-Hyacinthe_Chiefs, http://dbpedia.org/resource/Texas_Stars, http://dbpedia.org/resource/Windsor_Spitfires, http://dbpedia.org/resource/VIK_Västerås_HK, http://dbpedia.org/resource/Tornado_Moscow_Region, http://dbpedia.org/resource/HC_Agidel_Ufa, http://dbpedia.org/resource/OSC_Berlin, http://dbpedia.org/resource/ECDC_Memmingen, http://dbpedia.org/resource/EHC_Klostersee, http://dbpedia.org/resource/EC_Bergkamen, http://dbpedia.org/resource/Kalamazoo_Wings, http://dbpedia.org/resource/Hamilton_Bulldogs_(AHL), http://dbpedia.org/resource/Kuban_Krasnodar_(ice_hockey), http://dbpedia.org/resource/Hannover_Indians, http://dbpedia.org/resource/Ohio_State_Buckeyes_women's_ice_hockey, http://dbpedia.org/resource/Miskolci_JJSE, http://dbpedia.org/resource/Charlotte_Checkers, http://dbpedia.org/resource/HC_Lev_Praha, http://dbpedia.org/resource/IF_Frisk_Asker, http://dbpedia.org/resource/EHC_Winterthur, http://dbpedia.org/resource/VEU_Feldkirch, http://dbpedia.org/resource/Pingouins_de_Morzine-Avoriaz, http://dbpedia.org/resource/Stjernen_Hockey, http://dbpedia.org/resource/HK_Liepājas_Metalurgs, http://dbpedia.org/resource/ŠHK_37_Piešťany, http://dbpedia.org/resource/Hudiksvalls_HC, http://dbpedia.org/resource/Yermak_Angarsk, http://dbpedia.org/resource/HC_Dukla_Jihlava, http://dbpedia.org/resource/Notre_Dame_Fighting_Irish_men's_ice_hockey, http://dbpedia.org/resource/Gatineau_Olympiques, http://dbpedia.org/resource/Arizona_Sundogs, http://dbpedia.org/resource/Providence_Friars_women's_ice_hockey, http://dbpedia.org/resource/HYS_The_Hague, http://dbpedia.org/resource/Bilyi_Bars, http://dbpedia.org/resource/Abbotsford_Heat, http://dbpedia.org/resource/Ottawa_Senators_(original), http://dbpedia.org/resource/Královští_Lvi, http://dbpedia.org/resource/Cornwall_River_Kings, http://dbpedia.org/resource/Mississauga_Chiefs, http://dbpedia.org/resource/Rivière-du-Loup_3L, http://dbpedia.org/resource/Narmanspor, http://dbpedia.org/resource/Molot-Prikamye_Perm, http://dbpedia.org/resource/HC_Prešov_07, http://dbpedia.org/resource/Chicago_Express, http://dbpedia.org/resource/HK_Neman_Grodno, http://dbpedia.org/resource/EV_Bozen_84, http://dbpedia.org/resource/HC_Budivelnyk, http://dbpedia.org/resource/Portland_Pirates, http://dbpedia.org/resource/HK_Spišská_Nová_Ves, http://dbpedia.org/resource/Victoria_Salmon_Kings, http://dbpedia.org/resource/Newcastle_North_Stars, http://dbpedia.org/resource/Rote_Teufel_Bad_Nauheim, http://dbpedia.org/resource/Michigan_Wolverines_men's_ice_hockey, http://dbpedia.org/resource/Norfolk_Admirals_(ECHL), http://dbpedia.org/resource/Saryarka_Karagandy, http://dbpedia.org/resource/HC_Slovan_Ústečtí_Lvi, http://dbpedia.org/resource/HC_Olomouc, http://dbpedia.org/resource/Penticton_Vees, http://dbpedia.org/resource/HC_La_Chaux-de-Fonds, http://dbpedia.org/resource/Jokipojat, http://dbpedia.org/resource/San_Diego_Gulls, http://dbpedia.org/resource/London_Knights, http://dbpedia.org/resource/ODTÜ_SK, http://dbpedia.org/resource/ETC_Crimmitschau, http://dbpedia.org/resource/Lehigh_Valley_Phantoms, http://dbpedia.org/resource/Minnesota–Duluth_Bulldogs_men's_ice_hockey, http://dbpedia.org/resource/University_of_North_Dakota, http://dbpedia.org/resource/Cornell_Big_Red_men's_ice_hockey, http://dbpedia.org/resource/Minnesota_State_Mavericks_women's_ice_hockey, http://dbpedia.org/resource/Beibarys_Atyrau, http://dbpedia.org/resource/Başkent_Yıldızları, http://dbpedia.org/resource/Kocaeli_Büyükşehir_Belediyesi_Kağıt_Spor_Kulübü, http://dbpedia.org/resource/Manchester_Monarchs_(ECHL), http://dbpedia.org/resource/Alaska_Aces_(ECHL), http://dbpedia.org/resource/Brock_Badgers, http://dbpedia.org/resource/Sheffield_Steeldogs, http://dbpedia.org/resource/Erzurum_Gençlik_SK, http://dbpedia.org/resource/Mörrums_GoIS_IK, http://dbpedia.org/resource/Brûleurs_de_Loups, http://dbpedia.org/resource/HK_Partizan, http://dbpedia.org/resource/HC_Astana, http://dbpedia.org/resource/Varberg_Vipers, http://dbpedia.org/resource/Bruleurs_de_Loups_de_Grenoble, http://dbpedia.org/resource/HK_Gomel, http://dbpedia.org/resource/Everett_Silvertips, http://dbpedia.org/resource/Boston_College_Eagles_men's_ice_hockey, http://dbpedia.org/resource/St._Lawrence_Saints_women's_ice_hockey, http://dbpedia.org/resource/Fayetteville_FireAntz, http://dbpedia.org/resource/Kocaeli_Büyükşehir_Belediyesi_Kağıt_S.K._Men's_Ice_Hockey, http://dbpedia.org/resource/Frederikshavn_White_Hawks, http://dbpedia.org/resource/HK_Brest] [INFO] Predicted Answer: [] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 695, P: 0.511, R: 0.562, macro F1: 0.517, macro F1*: 0.535 -[INFO] Current time: 460 ms, average: 2805.57 ms/ques, entity detection: 448.47 ms/ques (linking tool: 356.03 ms/ques), relation detection: 915.90 ms/ques (similarity: 705.26 ms/ques), EDG generation: 47.53 ms/ques +[INFO] Cumulative metrics, sample: 695, P: 0.511, R: 0.562, macro F1: 0.535 +[INFO] Current question consumed time: 410 ms, average: 2532.61 ms/ques, entity detection: 397.39 ms/ques (linking tool: 319.00 ms/ques), relation detection: 847.09 ms/ques (similarity: 695.47 ms/ques), EDG generation: 47.44 ms/ques [INFO] Entity Linking P: 0.7489208633093525, R: 0.7, micro F1: 0.7155875299760203, macro F1: 0.7236345580933465 [INFO] Relation Linking P: 0.600599520383693, R: 0.5474820143884892, micro F1: 0.5623980815347728, macro F1: 0.5728119916599357 [INFO] Type Linking P: 0.5496402877697841, R: 0.5496402877697841, micro F1: 0.5496402877697841, macro F1: 0.5496402877697841 @@ -9761,8 +9761,8 @@ Question 695: Which rivers start from the Provinces of Zambia? [INFO] Golden Answer: [http://dbpedia.org/resource/Zambezi, http://dbpedia.org/resource/Kafue_River] [INFO] Predicted Answer: [http://dbpedia.org/resource/Central_Province,_Zambia, http://dbpedia.org/resource/Copperbelt_Province, http://dbpedia.org/resource/Eastern_Province,_Zambia, http://dbpedia.org/resource/Luapula_Province, http://dbpedia.org/resource/Lusaka_Province, http://dbpedia.org/resource/North-Western_Province,_Zambia, http://dbpedia.org/resource/Northern_Province,_Zambia, http://dbpedia.org/resource/Southern_Province,_Zambia, http://dbpedia.org/resource/Western_Province,_Zambia, http://dbpedia.org/resource/Muchinga_Province] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 696, P: 0.510, R: 0.561, macro F1: 0.516, macro F1*: 0.535 -[INFO] Current time: 1069 ms, average: 2803.08 ms/ques, entity detection: 448.38 ms/ques (linking tool: 355.97 ms/ques), relation detection: 914.77 ms/ques (similarity: 704.40 ms/ques), EDG generation: 47.57 ms/ques +[INFO] Cumulative metrics, sample: 696, P: 0.510, R: 0.561, macro F1: 0.535 +[INFO] Current question consumed time: 885 ms, average: 2530.25 ms/ques, entity detection: 397.25 ms/ques (linking tool: 318.91 ms/ques), relation detection: 846.04 ms/ques (similarity: 694.62 ms/ques), EDG generation: 47.47 ms/ques [INFO] Entity Linking P: 0.7492816091954023, R: 0.7004310344827587, micro F1: 0.7159961685823765, macro F1: 0.7240332695396604 [INFO] Relation Linking P: 0.6011733716475095, R: 0.5474137931034483, micro F1: 0.5625478927203071, macro F1: 0.5730354748613392 [INFO] Type Linking P: 0.5488505747126436, R: 0.5488505747126436, micro F1: 0.5488505747126436, macro F1: 0.5488505747126436 @@ -9775,8 +9775,8 @@ Question 696: Who are the scientists known for the Manhattan Project? [INFO] Golden Answer: [http://dbpedia.org/resource/Ernest_Lawrence, http://dbpedia.org/resource/James_Chadwick, http://dbpedia.org/resource/Philip_Burton_Moon, http://dbpedia.org/resource/Robert_Bacher, http://dbpedia.org/resource/Alvin_M._Weinberg, http://dbpedia.org/resource/Vannevar_Bush, http://dbpedia.org/resource/Chien-Shiung_Wu, http://dbpedia.org/resource/Charles_Critchfield, http://dbpedia.org/resource/George_Kistiakowsky, http://dbpedia.org/resource/William_G._Pollard, http://dbpedia.org/resource/Samuel_King_Allison, http://dbpedia.org/resource/Robert_E._Sheriff] [INFO] Predicted Answer: [http://dbpedia.org/resource/Leona_Woods, http://dbpedia.org/resource/Dorothy_McKibbin, http://dbpedia.org/resource/Ernest_Lawrence, http://dbpedia.org/resource/James_Chadwick, http://dbpedia.org/resource/Philip_Burton_Moon, http://dbpedia.org/resource/Joseph_Rotblat, http://dbpedia.org/resource/Robert_Bacher, http://dbpedia.org/resource/Alvin_M._Weinberg, http://dbpedia.org/resource/Edgar_Sengier, http://dbpedia.org/resource/George_Cowan, http://dbpedia.org/resource/Vannevar_Bush, http://dbpedia.org/resource/Leon_Davidson, http://dbpedia.org/resource/Rubby_Sherr, http://dbpedia.org/resource/Chien-Shiung_Wu, http://dbpedia.org/resource/Charles_Critchfield, http://dbpedia.org/resource/George_Kistiakowsky, http://dbpedia.org/resource/William_Penney,_Baron_Penney, http://dbpedia.org/resource/Ed_Westcott, http://dbpedia.org/resource/William_G._Pollard, http://dbpedia.org/resource/Priscilla_Duffield, http://dbpedia.org/resource/Piara_Singh_Gill, http://dbpedia.org/resource/Samuel_King_Allison, http://dbpedia.org/resource/Robert_E._Sheriff] [INFO] P: 0.5217391304347826, R: 1.0, F: 0.6857142857142856 -[INFO] Cumulative metrics, sample: 697, P: 0.510, R: 0.562, macro F1: 0.516, macro F1*: 0.535 -[INFO] Current time: 1699 ms, average: 2801.49 ms/ques, entity detection: 448.38 ms/ques (linking tool: 356.00 ms/ques), relation detection: 914.14 ms/ques (similarity: 703.76 ms/ques), EDG generation: 47.52 ms/ques +[INFO] Cumulative metrics, sample: 697, P: 0.510, R: 0.562, macro F1: 0.535 +[INFO] Current question consumed time: 1452 ms, average: 2528.70 ms/ques, entity detection: 397.23 ms/ques (linking tool: 318.92 ms/ques), relation detection: 845.44 ms/ques (similarity: 694.00 ms/ques), EDG generation: 47.42 ms/ques [INFO] Entity Linking P: 0.7496413199426112, R: 0.7008608321377331, micro F1: 0.7164036346245826, macro F1: 0.7244308304489923 [INFO] Relation Linking P: 0.6017455762792921, R: 0.5480631276901005, micro F1: 0.5631755141080829, macro F1: 0.5736511847071384 [INFO] Type Linking P: 0.5480631276901005, R: 0.5480631276901005, micro F1: 0.5480631276901005, macro F1: 0.5480631276901005 @@ -9789,8 +9789,8 @@ Question 697: What are some dishes from the family of dishes which primarily use [INFO] Golden Answer: [http://dbpedia.org/resource/Taralli, http://dbpedia.org/resource/Injera, http://dbpedia.org/resource/Shaobing, http://dbpedia.org/resource/Focaccia, http://dbpedia.org/resource/Pane_carasau, http://dbpedia.org/resource/Piadina, http://dbpedia.org/resource/Laobing, http://dbpedia.org/resource/Bazlama, http://dbpedia.org/resource/Lazarakia, http://dbpedia.org/resource/Mantou, http://dbpedia.org/resource/Paska_(bread), http://dbpedia.org/resource/Tsoureki, http://dbpedia.org/resource/Bread_roll, http://dbpedia.org/resource/Daktyla, http://dbpedia.org/resource/Guernsey_Gâche, http://dbpedia.org/resource/Rye_bread, http://dbpedia.org/resource/Tapalapa_bread, http://dbpedia.org/resource/Vasilopita, http://dbpedia.org/resource/Breadstick, http://dbpedia.org/resource/Challah, http://dbpedia.org/resource/Coppia_Ferrarese, http://dbpedia.org/resource/Filone, http://dbpedia.org/resource/Michetta, http://dbpedia.org/resource/Pane_di_Altamura, http://dbpedia.org/resource/Penia_(bread), http://dbpedia.org/resource/Wotou, http://dbpedia.org/resource/Ptitim, http://dbpedia.org/resource/Giouvetsi, http://dbpedia.org/resource/Pastitsio, http://dbpedia.org/resource/Bavette_(pasta), http://dbpedia.org/resource/Bigoli, http://dbpedia.org/resource/Bucatini, http://dbpedia.org/resource/Ciceri_e_Tria, http://dbpedia.org/resource/Conchiglie, http://dbpedia.org/resource/Farfalle, http://dbpedia.org/resource/Garganelli, http://dbpedia.org/resource/Lasagnette, http://dbpedia.org/resource/Linguine, http://dbpedia.org/resource/Macaroni, http://dbpedia.org/resource/Orecchiette, http://dbpedia.org/resource/Pappardelle, http://dbpedia.org/resource/Passatelli, http://dbpedia.org/resource/Penne, http://dbpedia.org/resource/Pizzoccheri, http://dbpedia.org/resource/Rigatoni, http://dbpedia.org/resource/Rigatoni_con_la_Pajata, http://dbpedia.org/resource/Spaghetti_alle_vongole, http://dbpedia.org/resource/Tortellini, http://dbpedia.org/resource/Trenette, http://dbpedia.org/resource/Vermicelli, http://dbpedia.org/resource/Madeira_cake, http://dbpedia.org/resource/Castella, http://dbpedia.org/resource/Roti_canai, http://dbpedia.org/resource/Afghan_bread, http://dbpedia.org/resource/Bolani, http://dbpedia.org/resource/Simit, http://dbpedia.org/resource/English_muffin, http://dbpedia.org/resource/Bread_bowl, http://dbpedia.org/resource/Cougnou, http://dbpedia.org/resource/Pepperoni_roll, http://dbpedia.org/resource/Vienna_bread, http://dbpedia.org/resource/Taftan_(bread), http://dbpedia.org/resource/Boule_(bread), http://dbpedia.org/resource/Faluche_(bread), http://dbpedia.org/resource/Gata_(food), http://dbpedia.org/resource/Mutschel, http://dbpedia.org/resource/Pasta_al_pomodoro, http://dbpedia.org/resource/Acini_di_pepe, http://dbpedia.org/resource/Alphabet_pasta, http://dbpedia.org/resource/Marry_girl_cake, http://dbpedia.org/resource/Flapjack_(oat_bar), http://dbpedia.org/resource/Tonis_puri, http://dbpedia.org/resource/Tongyeong_Kkulbbang, http://dbpedia.org/resource/Brioche, http://dbpedia.org/resource/Fougasse_(bread), http://dbpedia.org/resource/Bhakri, http://dbpedia.org/resource/Tortello_amaro_di_Castel_Goffredo, http://dbpedia.org/resource/Cavatelli, http://dbpedia.org/resource/Premium_Plus, http://dbpedia.org/resource/Lefse, http://dbpedia.org/resource/Lahoh, http://dbpedia.org/resource/Sangak, http://dbpedia.org/resource/Taboon_bread, http://dbpedia.org/resource/Amish_friendship_bread, http://dbpedia.org/resource/Desem, http://dbpedia.org/resource/National_loaf, http://dbpedia.org/resource/Farl, http://dbpedia.org/resource/Pogača, http://dbpedia.org/resource/Buccellato_(di_Lucca), http://dbpedia.org/resource/Plain_loaf, http://dbpedia.org/resource/Korovai, http://dbpedia.org/resource/Colaci, http://dbpedia.org/resource/Hefekranz, http://dbpedia.org/resource/Kalács, http://dbpedia.org/resource/Vánočka, http://dbpedia.org/resource/Huangqiao_Sesame_Cake, http://dbpedia.org/resource/Toasted_ravioli, http://dbpedia.org/resource/Pasta_alla_Norma, http://dbpedia.org/resource/Pici, http://dbpedia.org/resource/Lamington, http://dbpedia.org/resource/Scone, http://dbpedia.org/resource/Paški_baškotin, http://dbpedia.org/resource/Hot_dog_bun, http://dbpedia.org/resource/Animal_cracker, http://dbpedia.org/resource/Graham_cracker, http://dbpedia.org/resource/Saltine_cracker, http://dbpedia.org/resource/Hardtack, http://dbpedia.org/resource/Ajdov_Kruh, http://dbpedia.org/resource/Tortilla, http://dbpedia.org/resource/Kitcha, http://dbpedia.org/resource/Fatoot, http://dbpedia.org/resource/Kubaneh, http://dbpedia.org/resource/Malawach, http://dbpedia.org/resource/Chickpea_bread, http://dbpedia.org/resource/Himbasha, http://dbpedia.org/resource/Pane_ticinese, http://dbpedia.org/resource/Ruisreikäleipä, http://dbpedia.org/resource/Basler_Brot, http://dbpedia.org/resource/Sweet_roll, http://dbpedia.org/resource/Bun, http://dbpedia.org/resource/Fit-fit, http://dbpedia.org/resource/Flatbread, http://dbpedia.org/resource/Pretzel, http://dbpedia.org/resource/Viennoiserie, http://dbpedia.org/resource/Lagane_e_cicciari, http://dbpedia.org/resource/Orzo_(pasta), http://dbpedia.org/resource/Capellini, http://dbpedia.org/resource/Rotelle, http://dbpedia.org/resource/Maultasche, http://dbpedia.org/resource/Blondie_(confection), http://dbpedia.org/resource/Traou_Mad, http://dbpedia.org/resource/Tunnbröd, http://dbpedia.org/resource/Corn_tortilla, http://dbpedia.org/resource/Bammy, http://dbpedia.org/resource/Laufabrauð, http://dbpedia.org/resource/Obi_non, http://dbpedia.org/resource/Proja, http://dbpedia.org/resource/Garlic_bread, http://dbpedia.org/resource/Pan_de_yuca, http://dbpedia.org/resource/Fried_bread, http://dbpedia.org/resource/Güirila, http://dbpedia.org/resource/Piki, http://dbpedia.org/resource/Butterbrot, http://dbpedia.org/resource/Česnica, http://dbpedia.org/resource/Coco_bread, http://dbpedia.org/resource/Hard_dough_bread, http://dbpedia.org/resource/Tortilla_de_rescoldo, http://dbpedia.org/resource/Adobe_bread, http://dbpedia.org/resource/Pastel_de_Camiguín, http://dbpedia.org/resource/Spaghetti_aglio_e_olio, http://dbpedia.org/resource/Spaghetti_alla_chitarra, http://dbpedia.org/resource/Linguettine, http://dbpedia.org/resource/Angel_food_cake, http://dbpedia.org/resource/Hedgehog_slice, http://dbpedia.org/resource/Pizza, http://dbpedia.org/resource/Torta, http://dbpedia.org/resource/Bing_(bread), http://dbpedia.org/resource/Hot_water_corn_bread, http://dbpedia.org/resource/Anadama_bread, http://dbpedia.org/resource/Biscuit_(bread), http://dbpedia.org/resource/Bulkie_roll, http://dbpedia.org/resource/Sourdough, http://dbpedia.org/resource/Toast, http://dbpedia.org/resource/White_bread, http://dbpedia.org/resource/Almojábana, http://dbpedia.org/resource/Danish_pastry, http://dbpedia.org/resource/Kolach, http://dbpedia.org/resource/Pandebono, http://dbpedia.org/resource/Baozi, http://dbpedia.org/resource/Cannelloni, http://dbpedia.org/resource/Fettuccine, http://dbpedia.org/resource/Fusilli, http://dbpedia.org/resource/Lasagne, http://dbpedia.org/resource/Ravioli, http://dbpedia.org/resource/Spaghetti, http://dbpedia.org/resource/Tagliatelle, http://dbpedia.org/resource/Agnolotti, http://dbpedia.org/resource/Tres_leches_cake, http://dbpedia.org/resource/Chocolate_brownie, http://dbpedia.org/resource/Cornbread, http://dbpedia.org/resource/Muffin, http://dbpedia.org/resource/Cream_cracker, http://dbpedia.org/resource/Scallion_pancake, http://dbpedia.org/resource/Parotta, http://dbpedia.org/resource/Kaak, http://dbpedia.org/resource/Bollo, http://dbpedia.org/resource/Colston_bun, http://dbpedia.org/resource/Lady_Arundel's_Manchet, http://dbpedia.org/resource/London_bun, http://dbpedia.org/resource/Manchet, http://dbpedia.org/resource/Mandarin_roll, http://dbpedia.org/resource/Penny_bun, http://dbpedia.org/resource/Raisin_bread, http://dbpedia.org/resource/Dripping_cake, http://dbpedia.org/resource/Kandil_simidi, http://dbpedia.org/resource/Kattama, http://dbpedia.org/resource/Paris_buns, http://dbpedia.org/resource/Baguette, http://dbpedia.org/resource/Strozzapreti, http://dbpedia.org/resource/Campanelle, http://dbpedia.org/resource/Pastina, http://dbpedia.org/resource/Pissaladière, http://dbpedia.org/resource/Luchi, http://dbpedia.org/resource/Pashti, http://dbpedia.org/resource/Manakish, http://dbpedia.org/resource/Aish_merahrah, http://dbpedia.org/resource/Chakhchoukha, http://dbpedia.org/resource/Mlinci, http://dbpedia.org/resource/Pão_de_queijo, http://dbpedia.org/resource/Barmbrack, http://dbpedia.org/resource/No-knead_bread, http://dbpedia.org/resource/Sliced_bread, http://dbpedia.org/resource/Kulich, http://dbpedia.org/resource/Sprouted_bread, http://dbpedia.org/resource/Unleavened_bread, http://dbpedia.org/resource/Banana_bread, http://dbpedia.org/resource/Barley_bread, http://dbpedia.org/resource/Beer_bread, http://dbpedia.org/resource/Bolillo, http://dbpedia.org/resource/Brown_bread, http://dbpedia.org/resource/Bush_bread, http://dbpedia.org/resource/Cardamom_bread, http://dbpedia.org/resource/Cottage_loaf, http://dbpedia.org/resource/Crumpet, http://dbpedia.org/resource/Horsebread, http://dbpedia.org/resource/Kompyang, http://dbpedia.org/resource/Makki_di_roti, http://dbpedia.org/resource/Monkey_bread, http://dbpedia.org/resource/Pandesal, http://dbpedia.org/resource/Potato_bread, http://dbpedia.org/resource/Quick_bread, http://dbpedia.org/resource/Rice_bread, http://dbpedia.org/resource/Salt-rising_bread, http://dbpedia.org/resource/Scali_bread, http://dbpedia.org/resource/Shotis_puri, http://dbpedia.org/resource/Sop, http://dbpedia.org/resource/Steamed_bread, http://dbpedia.org/resource/Stottie_cake, http://dbpedia.org/resource/Texas_toast, http://dbpedia.org/resource/Tiger_bread, http://dbpedia.org/resource/Zopf, http://dbpedia.org/resource/Bialy_(pastry), http://dbpedia.org/resource/Marraqueta, http://dbpedia.org/resource/Antidoron, http://dbpedia.org/resource/Prosphora, http://dbpedia.org/resource/Pullman_loaf, http://dbpedia.org/resource/Farfel, http://dbpedia.org/resource/Fideuà, http://dbpedia.org/resource/Tortelloni, http://dbpedia.org/resource/Chicken_riggies, http://dbpedia.org/resource/Penne_alla_vodka, http://dbpedia.org/resource/Pumpkin_bread, http://dbpedia.org/resource/Pain_d'épices, http://dbpedia.org/resource/Damper_(food), http://dbpedia.org/resource/Crown_Pilot_Crackers, http://dbpedia.org/resource/Maltose_crackers, http://dbpedia.org/resource/Nantong_Xiting_Cracker, http://dbpedia.org/resource/Kulcha, http://dbpedia.org/resource/Flatkaka, http://dbpedia.org/resource/Malooga, http://dbpedia.org/resource/Ngome_(bread), http://dbpedia.org/resource/Podpłomyk, http://dbpedia.org/resource/Sanchuisanda, http://dbpedia.org/resource/Matzo, http://dbpedia.org/resource/Kneippbrød, http://dbpedia.org/resource/Borodinsky_bread, http://dbpedia.org/resource/Tostada_(toast), http://dbpedia.org/resource/Bappir, http://dbpedia.org/resource/Pain_de_mie, http://dbpedia.org/resource/Kerststol, http://dbpedia.org/resource/Suikerbrood, http://dbpedia.org/resource/Lardy_cake, http://dbpedia.org/resource/Singing_hinny, http://dbpedia.org/resource/Easter_bread, http://dbpedia.org/resource/Lipie_(bread), http://dbpedia.org/resource/Burebrot, http://dbpedia.org/resource/Peg_bread, http://dbpedia.org/resource/Slow_Bread, http://dbpedia.org/resource/Gemelli_(pasta), http://dbpedia.org/resource/Bannock_(food), http://dbpedia.org/resource/Oatcake, http://dbpedia.org/resource/Oyster_cracker, http://dbpedia.org/resource/Jolada_rotti, http://dbpedia.org/resource/Rupjmaize, http://dbpedia.org/resource/Bánh_mì, http://dbpedia.org/resource/Bánh, http://dbpedia.org/resource/Cheesymite_scroll, http://dbpedia.org/resource/Dampfnudel, http://dbpedia.org/resource/Germknödel, http://dbpedia.org/resource/Barm_cake, http://dbpedia.org/resource/Baati, http://dbpedia.org/resource/Covrigi, http://dbpedia.org/resource/Kalach_(food), http://dbpedia.org/resource/Sushki, http://dbpedia.org/resource/Twist_bread, http://dbpedia.org/resource/Bánh_bò, http://dbpedia.org/resource/Swiss_roll, http://dbpedia.org/resource/Soda_bread, http://dbpedia.org/resource/Ry-Krisp, http://dbpedia.org/resource/Christmas_wafer, http://dbpedia.org/resource/Torta_de_gazpacho, http://dbpedia.org/resource/Khubz, http://dbpedia.org/resource/Frybread, http://dbpedia.org/resource/Zwieback, http://dbpedia.org/resource/Potbrood, http://dbpedia.org/resource/Cockle_bread, http://dbpedia.org/resource/Biskotso, http://dbpedia.org/resource/Ramazan_pidesi, http://dbpedia.org/resource/Finnish_bread, http://dbpedia.org/resource/Mandoca, http://dbpedia.org/resource/Pan_loaf, http://dbpedia.org/resource/Mealie_bread, http://dbpedia.org/resource/Hallulla, http://dbpedia.org/resource/Canadian_white_bread, http://dbpedia.org/resource/Bublik, http://dbpedia.org/resource/Chipa, http://dbpedia.org/resource/Bark_bread, http://dbpedia.org/resource/Pasta_primavera, http://dbpedia.org/resource/Anelli, http://dbpedia.org/resource/Fusi_(pasta), http://dbpedia.org/resource/Troccoli, http://dbpedia.org/resource/Mein_gon, http://dbpedia.org/resource/Water_biscuit, http://dbpedia.org/resource/Jacob's_Oddities, http://dbpedia.org/resource/Kichel, http://dbpedia.org/resource/Pletzel, http://dbpedia.org/resource/Flatbrød, http://dbpedia.org/resource/Markook, http://dbpedia.org/resource/Mekitsa, http://dbpedia.org/resource/Krotekake, http://dbpedia.org/resource/Balep_korkun, http://dbpedia.org/resource/Sabaayad, http://dbpedia.org/resource/Mantecadas, http://dbpedia.org/resource/Sel_roti, http://dbpedia.org/resource/Tequeño, http://dbpedia.org/resource/Sorullos, http://dbpedia.org/resource/Pain_brié, http://dbpedia.org/resource/Panbrioche, http://dbpedia.org/resource/Soldiers_(food), http://dbpedia.org/resource/Samoon, http://dbpedia.org/resource/Sha_Phaley, http://dbpedia.org/resource/Corzetti, http://dbpedia.org/resource/Trofie, http://dbpedia.org/resource/Anellini, http://dbpedia.org/resource/Calamarata, http://dbpedia.org/resource/Capunti, http://dbpedia.org/resource/Casarecce, http://dbpedia.org/resource/Casoncelli, http://dbpedia.org/resource/Casunziei, http://dbpedia.org/resource/Cavatappi, http://dbpedia.org/resource/Cencioni, http://dbpedia.org/resource/Croxetti, http://dbpedia.org/resource/Fagottini, http://dbpedia.org/resource/Fiori_(pasta), http://dbpedia.org/resource/Foglie_d'ulivo, http://dbpedia.org/resource/Fregula, http://dbpedia.org/resource/Lanterne_(pasta), http://dbpedia.org/resource/Maccheroni_alla_molinara, http://dbpedia.org/resource/Mafalde, http://dbpedia.org/resource/Mafaldine, http://dbpedia.org/resource/Occhi_di_Lupo, http://dbpedia.org/resource/Pillus, http://dbpedia.org/resource/Radiatori, http://dbpedia.org/resource/Rotini, http://dbpedia.org/resource/Sacchettoni, http://dbpedia.org/resource/Sagnarelli, http://dbpedia.org/resource/Scialatelli, http://dbpedia.org/resource/Stringozzi, http://dbpedia.org/resource/Taglierini, http://dbpedia.org/resource/Tripoline, http://dbpedia.org/resource/Genoise, http://dbpedia.org/resource/Mekupelet, http://dbpedia.org/resource/Scotcheroos, http://dbpedia.org/resource/Crispbread, http://dbpedia.org/resource/Rempeyek, http://dbpedia.org/resource/Lángos, http://dbpedia.org/resource/Mjukkaka, http://dbpedia.org/resource/Tohax, http://dbpedia.org/resource/Youtazi, http://dbpedia.org/resource/Sally_Lunn_bun, http://dbpedia.org/resource/Bara_brith, http://dbpedia.org/resource/Maltese_bread, http://dbpedia.org/resource/Matnakash, http://dbpedia.org/resource/Pită_de_Pecica, http://dbpedia.org/resource/Folar, http://dbpedia.org/resource/Graham_bread, http://dbpedia.org/resource/Mollete, http://dbpedia.org/resource/Veda_bread, http://dbpedia.org/resource/Panfocaccia, http://dbpedia.org/resource/Barbine, http://dbpedia.org/resource/Macaroni_casserole, http://dbpedia.org/resource/Mezzelune, http://dbpedia.org/resource/Battenberg_cake, http://dbpedia.org/resource/Talo_(food), http://dbpedia.org/resource/Rewena_bread, http://dbpedia.org/resource/Grant_loaf, http://dbpedia.org/resource/Trenza, http://dbpedia.org/resource/Chikkolee, http://dbpedia.org/resource/Vakfıkebir_bread, http://dbpedia.org/resource/Halguane, http://dbpedia.org/resource/Goro_(sweet_bread), http://dbpedia.org/resource/Cornell_bread, http://dbpedia.org/resource/Picatostes, http://dbpedia.org/resource/Crackling_bread, http://dbpedia.org/resource/Hoyha, http://dbpedia.org/resource/Wagafi_bread, http://dbpedia.org/resource/Cocol, http://dbpedia.org/resource/Sabounee, http://dbpedia.org/resource/Amandine_(dessert)] [INFO] Predicted Answer: [] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 698, P: 0.510, R: 0.561, macro F1: 0.516, macro F1*: 0.534 -[INFO] Current time: 1990 ms, average: 2800.33 ms/ques, entity detection: 447.90 ms/ques (linking tool: 355.58 ms/ques), relation detection: 914.07 ms/ques (similarity: 703.50 ms/ques), EDG generation: 47.50 ms/ques +[INFO] Cumulative metrics, sample: 698, P: 0.510, R: 0.561, macro F1: 0.534 +[INFO] Current question consumed time: 1785 ms, average: 2527.63 ms/ques, entity detection: 396.81 ms/ques (linking tool: 318.54 ms/ques), relation detection: 845.37 ms/ques (similarity: 693.77 ms/ques), EDG generation: 47.39 ms/ques [INFO] Entity Linking P: 0.748567335243553, R: 0.6998567335243553, micro F1: 0.7153772683858655, macro F1: 0.7233929639297243 [INFO] Relation Linking P: 0.6008834765998089, R: 0.5472779369627507, micro F1: 0.5623686723973264, macro F1: 0.5728293348723145 [INFO] Type Linking P: 0.5472779369627507, R: 0.5472779369627507, micro F1: 0.5472779369627507, macro F1: 0.5472779369627507 @@ -9803,8 +9803,8 @@ Question 698: Which religion do the people follow, whose constituency is Jadavpu [INFO] Golden Answer: [http://dbpedia.org/resource/Hinduism, http://dbpedia.org/resource/Islam] [INFO] Predicted Answer: [http://dbpedia.org/resource/Hinduism, http://dbpedia.org/resource/Islam] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 699, P: 0.510, R: 0.562, macro F1: 0.516, macro F1*: 0.535 -[INFO] Current time: 2998 ms, average: 2800.61 ms/ques, entity detection: 447.73 ms/ques (linking tool: 355.37 ms/ques), relation detection: 915.00 ms/ques (similarity: 704.22 ms/ques), EDG generation: 47.46 ms/ques +[INFO] Cumulative metrics, sample: 699, P: 0.510, R: 0.562, macro F1: 0.535 +[INFO] Current question consumed time: 2628 ms, average: 2527.78 ms/ques, entity detection: 396.61 ms/ques (linking tool: 318.31 ms/ques), relation detection: 846.18 ms/ques (similarity: 694.48 ms/ques), EDG generation: 47.36 ms/ques [INFO] Entity Linking P: 0.7489270386266095, R: 0.7002861230329042, micro F1: 0.71578445398188, macro F1: 0.7237902969550353 [INFO] Relation Linking P: 0.601454458750596, R: 0.547925608011445, micro F1: 0.5629947544110641, macro F1: 0.5734435623727289 [INFO] Type Linking P: 0.5464949928469242, R: 0.5464949928469242, micro F1: 0.5464949928469242, macro F1: 0.5464949928469242 @@ -9817,8 +9817,8 @@ Question 699: In which region is the rival school of Somerset, Massachusetts loc [INFO] Golden Answer: [http://dbpedia.org/resource/New_England] [INFO] Predicted Answer: [http://dbpedia.org/resource/Henry_Cabot_Lodge, http://dbpedia.org/resource/John_W._Weeks, http://dbpedia.org/resource/Barney_Frank, http://dbpedia.org/resource/Marty_Meehan, http://dbpedia.org/resource/Richard_Neal, http://dbpedia.org/resource/John_Davis_(Massachusetts_governor), http://dbpedia.org/resource/William_B._Calhoun, http://dbpedia.org/resource/Levi_Lincoln,_Sr., http://dbpedia.org/resource/William_Claflin, http://dbpedia.org/resource/Niki_Tsongas, http://dbpedia.org/resource/Barbara_L'Italien, http://dbpedia.org/resource/Michael_F._Flaherty,_Sr., http://dbpedia.org/resource/Daniel_Webster, http://dbpedia.org/resource/John_F._Kennedy, http://dbpedia.org/resource/Nathaniel_P._Banks, http://dbpedia.org/resource/Timothy_Pickering, http://dbpedia.org/resource/Jane_Swift, http://dbpedia.org/resource/Edith_Nourse_Rogers, http://dbpedia.org/resource/James_Michael_Curley, http://dbpedia.org/resource/Thomson_J._Skinner, http://dbpedia.org/resource/Nathaniel_Silsbee, http://dbpedia.org/resource/Alvah_Crocker, http://dbpedia.org/resource/Barry_Finegold, http://dbpedia.org/resource/Bill_Keating_(politician), http://dbpedia.org/resource/Bruce_Tarr, http://dbpedia.org/resource/Enoch_Lincoln, http://dbpedia.org/resource/John_Velis, http://dbpedia.org/resource/John_W._Hulbert, http://dbpedia.org/resource/John_Wilson_(Massachusetts), http://dbpedia.org/resource/Joshua_Cushman, http://dbpedia.org/resource/Karyn_Polito, http://dbpedia.org/resource/Levi_Hubbard, http://dbpedia.org/resource/Patricia_D._Jehlen, http://dbpedia.org/resource/Samuel_Clesson_Allen, http://dbpedia.org/resource/Tom_Birmingham, http://dbpedia.org/resource/Torbert_Macdonald, http://dbpedia.org/resource/George_H._Tinkham, http://dbpedia.org/resource/Laurence_Curtis, http://dbpedia.org/resource/Joseph_H._Walker, http://dbpedia.org/resource/Anthony_Petruccelli, http://dbpedia.org/resource/Steven_C._Panagiotakos, http://dbpedia.org/resource/George_S._Boutwell, http://dbpedia.org/resource/Henry_Wilson, http://dbpedia.org/resource/John_Kerry, http://dbpedia.org/resource/John_Quincy_Adams, http://dbpedia.org/resource/Shearjashub_Bourne, http://dbpedia.org/resource/Gerry_Studds, http://dbpedia.org/resource/Henry_Dearborn, http://dbpedia.org/resource/Richard_Tisei, http://dbpedia.org/resource/Robert_DeLeo_(politician), http://dbpedia.org/resource/Paul_J._Gannon, http://dbpedia.org/resource/Brian_J._Donnelly, http://dbpedia.org/resource/William_Cogswell, http://dbpedia.org/resource/Michael_Knapik, http://dbpedia.org/resource/Michael_W._Morrissey, http://dbpedia.org/resource/Boston_cream_pie, http://dbpedia.org/resource/Chocolate_chip_cookie, http://dbpedia.org/resource/Ed_Markey, http://dbpedia.org/resource/Katherine_Clark, http://dbpedia.org/resource/Mike_Capuano, http://dbpedia.org/resource/Fisher_Ames, http://dbpedia.org/resource/Samuel_Dexter, http://dbpedia.org/resource/Benjamin_Butler_(politician), http://dbpedia.org/resource/Aaron_Hobart, http://dbpedia.org/resource/Barker_Burnell, http://dbpedia.org/resource/Elijah_Brigham, http://dbpedia.org/resource/Francis_Baylies, http://dbpedia.org/resource/George_Grennell,_Jr., http://dbpedia.org/resource/George_Leonard_(Congressman), http://dbpedia.org/resource/Harold_Donohue, http://dbpedia.org/resource/Henry_Williams_(Massachusetts), http://dbpedia.org/resource/Isaiah_L._Green, http://dbpedia.org/resource/Joshua_Gage, http://dbpedia.org/resource/Laban_Wheaton, http://dbpedia.org/resource/Nahum_Mitchell, http://dbpedia.org/resource/Nathaniel_B._Borden, http://dbpedia.org/resource/Phanuel_Bishop, http://dbpedia.org/resource/Richard_B._Wigglesworth, http://dbpedia.org/resource/Samuel_Dana, http://dbpedia.org/resource/Samuel_Lathrop, http://dbpedia.org/resource/Samuel_S._Conner, http://dbpedia.org/resource/William_Baylies, http://dbpedia.org/resource/Therese_Murray, http://dbpedia.org/resource/Joseph_Story, http://dbpedia.org/resource/William_Henry_Moody, http://dbpedia.org/resource/Margaret_Heckler, http://dbpedia.org/resource/Robert_Drinan, http://dbpedia.org/resource/Artemas_Ward, http://dbpedia.org/resource/Scott_Brown, http://dbpedia.org/resource/Robert_Luce, http://dbpedia.org/resource/Susan_Tucker, http://dbpedia.org/resource/Hastings_Keith, http://dbpedia.org/resource/James_A._Burke_(Massachusetts_politician), http://dbpedia.org/resource/Stephen_F._Lynch, http://dbpedia.org/resource/Jose_Tosado, http://dbpedia.org/resource/Michael_Finn, http://dbpedia.org/resource/John_W._Candler, http://dbpedia.org/resource/Henri_S._Rauschenbach, http://dbpedia.org/resource/John_F._Fitzgerald, http://dbpedia.org/resource/Caleb_Cushing, http://dbpedia.org/resource/Jim_McGovern_(U.S._politician), http://dbpedia.org/resource/Matthew_Beaton, http://dbpedia.org/resource/John_Olver, http://dbpedia.org/resource/Marty_Walsh_(politician), http://dbpedia.org/resource/William_W._Rice, http://dbpedia.org/resource/Thomas_Rice_(1768), http://dbpedia.org/resource/William_S._Greene, http://dbpedia.org/resource/George_T._Davis, http://dbpedia.org/resource/Norwood_Hospital, http://dbpedia.org/resource/Charles_E._Shannon,_Jr., http://dbpedia.org/resource/Elisha_Capen_Monk, http://dbpedia.org/resource/Harrison_Gray_Otis_(politician), http://dbpedia.org/resource/John_B._Alley, http://dbpedia.org/resource/Henry_L._Dawes, http://dbpedia.org/resource/James_Shannon, http://dbpedia.org/resource/Michael_Sullivan_(U.S._Attorney), http://dbpedia.org/resource/John_Joseph_Mitchell, http://dbpedia.org/resource/Suzanne_M._Bump, http://dbpedia.org/resource/William_F._Galvin, http://dbpedia.org/resource/Shannon_O'Brien, http://dbpedia.org/resource/Stan_Rosenberg, http://dbpedia.org/resource/Charles_F._Swift, http://dbpedia.org/resource/Adams_political_family, http://dbpedia.org/resource/Martha's_Vineyard_Sign_Language, http://dbpedia.org/resource/Loup_language, http://dbpedia.org/resource/Massachusetts_State_Collegiate_Athletic_Conference, http://dbpedia.org/resource/Eastern_Collegiate_Women's_Hockey_League] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 700, P: 0.510, R: 0.561, macro F1: 0.516, macro F1*: 0.534 -[INFO] Current time: 9436 ms, average: 2810.09 ms/ques, entity detection: 449.57 ms/ques (linking tool: 357.01 ms/ques), relation detection: 917.76 ms/ques (similarity: 707.18 ms/ques), EDG generation: 47.43 ms/ques +[INFO] Cumulative metrics, sample: 700, P: 0.510, R: 0.561, macro F1: 0.534 +[INFO] Current question consumed time: 8518 ms, average: 2536.34 ms/ques, entity detection: 398.24 ms/ques (linking tool: 319.80 ms/ques), relation detection: 848.67 ms/ques (similarity: 697.48 ms/ques), EDG generation: 47.34 ms/ques [INFO] Entity Linking P: 0.7478571428571429, R: 0.6992857142857143, micro F1: 0.7147619047619058, macro F1: 0.722756310816528 [INFO] Relation Linking P: 0.6020238095238095, R: 0.5478571428571428, micro F1: 0.5631428571428577, macro F1: 0.5736646798692558 [INFO] Type Linking P: 0.5457142857142857, R: 0.5457142857142857, micro F1: 0.5457142857142857, macro F1: 0.5457142857142857 @@ -9831,8 +9831,8 @@ Question 700: Which key person of Clinton Foundation is also the president of Ji [INFO] Golden Answer: [http://dbpedia.org/resource/Bill_Clinton] [INFO] Predicted Answer: [http://dbpedia.org/resource/Bill_Clinton] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 701, P: 0.510, R: 0.561, macro F1: 0.516, macro F1*: 0.535 -[INFO] Current time: 2079 ms, average: 2809.05 ms/ques, entity detection: 449.86 ms/ques (linking tool: 357.03 ms/ques), relation detection: 917.23 ms/ques (similarity: 706.80 ms/ques), EDG generation: 47.50 ms/ques +[INFO] Cumulative metrics, sample: 701, P: 0.510, R: 0.561, macro F1: 0.535 +[INFO] Current question consumed time: 1731 ms, average: 2535.19 ms/ques, entity detection: 398.47 ms/ques (linking tool: 319.81 ms/ques), relation detection: 848.08 ms/ques (similarity: 696.88 ms/ques), EDG generation: 47.44 ms/ques [INFO] Entity Linking P: 0.7482168330955777, R: 0.6997146932952925, micro F1: 0.715168806466953, macro F1: 0.7231534120854796 [INFO] Relation Linking P: 0.6018782691393247, R: 0.5477888730385164, micro F1: 0.5630527817403715, macro F1: 0.5735611755131856 [INFO] Type Linking P: 0.5463623395149786, R: 0.5463623395149786, micro F1: 0.5463623395149786, macro F1: 0.5463623395149786 @@ -9845,8 +9845,8 @@ Question 701: Where did the athlete start his career, who did his highschool in [INFO] Golden Answer: [http://dbpedia.org/resource/New_York_Yankees] [INFO] Predicted Answer: [http://dbpedia.org/resource/New_York_Yankees] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 702, P: 0.511, R: 0.562, macro F1: 0.517, macro F1*: 0.535 -[INFO] Current time: 1593 ms, average: 2807.32 ms/ques, entity detection: 449.47 ms/ques (linking tool: 356.65 ms/ques), relation detection: 916.83 ms/ques (similarity: 706.54 ms/ques), EDG generation: 47.48 ms/ques +[INFO] Cumulative metrics, sample: 702, P: 0.511, R: 0.562, macro F1: 0.535 +[INFO] Current question consumed time: 1387 ms, average: 2533.55 ms/ques, entity detection: 398.10 ms/ques (linking tool: 319.45 ms/ques), relation detection: 847.70 ms/ques (similarity: 696.65 ms/ques), EDG generation: 47.42 ms/ques [INFO] Entity Linking P: 0.7485754985754985, R: 0.7001424501424501, micro F1: 0.7155745489078833, macro F1: 0.7235493757126008 [INFO] Relation Linking P: 0.6024453941120608, R: 0.5484330484330484, micro F1: 0.5636752136752143, macro F1: 0.5741717835580652 [INFO] Type Linking P: 0.5455840455840456, R: 0.5455840455840456, micro F1: 0.5455840455840456, macro F1: 0.5455840455840456 @@ -9859,8 +9859,8 @@ Question 702: Name a person who works in Open Society Foundation and has wife na [INFO] Golden Answer: [http://dbpedia.org/resource/George_Soros] [INFO] Predicted Answer: [http://dbpedia.org/resource/London_School_of_Economics] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 703, P: 0.510, R: 0.561, macro F1: 0.516, macro F1*: 0.535 -[INFO] Current time: 2741 ms, average: 2807.22 ms/ques, entity detection: 449.36 ms/ques (linking tool: 356.44 ms/ques), relation detection: 916.66 ms/ques (similarity: 706.66 ms/ques), EDG generation: 47.45 ms/ques +[INFO] Cumulative metrics, sample: 703, P: 0.510, R: 0.561, macro F1: 0.535 +[INFO] Current question consumed time: 2351 ms, average: 2533.29 ms/ques, entity detection: 397.93 ms/ques (linking tool: 319.23 ms/ques), relation detection: 847.52 ms/ques (similarity: 696.77 ms/ques), EDG generation: 47.39 ms/ques [INFO] Entity Linking P: 0.7489331436699858, R: 0.6998577524893315, micro F1: 0.7155049786628744, macro F1: 0.7235642742967757 [INFO] Relation Linking P: 0.6022996680891417, R: 0.5483641536273115, micro F1: 0.5635846372688484, macro F1: 0.5740678406470319 [INFO] Type Linking P: 0.5448079658605974, R: 0.5448079658605974, micro F1: 0.5448079658605974, macro F1: 0.5448079658605974 @@ -9873,8 +9873,8 @@ Question 703: How many things are crossed over by bridges which can carry cars? [INFO] Golden Answer: [58] [INFO] Predicted Answer: [] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 704, P: 0.510, R: 0.560, macro F1: 0.516, macro F1*: 0.534 -[INFO] Current time: 791 ms, average: 2804.36 ms/ques, entity detection: 448.72 ms/ques (linking tool: 355.93 ms/ques), relation detection: 916.06 ms/ques (similarity: 706.11 ms/ques), EDG generation: 47.41 ms/ques +[INFO] Cumulative metrics, sample: 704, P: 0.510, R: 0.560, macro F1: 0.534 +[INFO] Current question consumed time: 687 ms, average: 2530.67 ms/ques, entity detection: 397.37 ms/ques (linking tool: 318.78 ms/ques), relation detection: 846.95 ms/ques (similarity: 696.22 ms/ques), EDG generation: 47.36 ms/ques [INFO] Entity Linking P: 0.7478693181818182, R: 0.6988636363636364, micro F1: 0.7144886363636375, macro F1: 0.7225364841344223 [INFO] Relation Linking P: 0.6014441287878788, R: 0.5475852272727273, micro F1: 0.5627840909090915, macro F1: 0.5732524033733857 [INFO] Type Linking P: 0.5454545454545454, R: 0.5454545454545454, micro F1: 0.5454545454545454, macro F1: 0.5454545454545454 @@ -9887,8 +9887,8 @@ Question 704: Which fictional character sculpted by Josh Friedmann was portrayed [INFO] Golden Answer: [http://dbpedia.org/resource/John_Connor] [INFO] Predicted Answer: [http://dbpedia.org/resource/Minnie_Mouse, http://dbpedia.org/resource/Papa_Lazarou, http://dbpedia.org/resource/Phil_Coulson, http://dbpedia.org/resource/Saul_Goodman, http://dbpedia.org/resource/Mr._Gold, http://dbpedia.org/resource/Mother_Gothel, http://dbpedia.org/resource/Oliver_Queen_(Arrow), http://dbpedia.org/resource/Ace_Ventura, http://dbpedia.org/resource/Miley_Stewart, http://dbpedia.org/resource/Penny_(The_Big_Bang_Theory), http://dbpedia.org/resource/Clarke_Griffin, http://dbpedia.org/resource/Lexa_(The_100), http://dbpedia.org/resource/Godzilla, http://dbpedia.org/resource/Simba, http://dbpedia.org/resource/Biollante, http://dbpedia.org/resource/Father_Ted_Crilly, http://dbpedia.org/resource/Monica_Geller, http://dbpedia.org/resource/Mothra, http://dbpedia.org/resource/Ludwig_Von_Drake, http://dbpedia.org/resource/Aladdin_(Disney_character), http://dbpedia.org/resource/Franco_(General_Hospital), http://dbpedia.org/resource/Anna_(Disney), http://dbpedia.org/resource/Elsa_(Disney), http://dbpedia.org/resource/List_of_General_Hospital_characters_(2000s)] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 705, P: 0.509, R: 0.560, macro F1: 0.515, macro F1*: 0.533 -[INFO] Current time: 4314 ms, average: 2806.50 ms/ques, entity detection: 448.65 ms/ques (linking tool: 355.74 ms/ques), relation detection: 917.08 ms/ques (similarity: 707.12 ms/ques), EDG generation: 47.39 ms/ques +[INFO] Cumulative metrics, sample: 705, P: 0.509, R: 0.560, macro F1: 0.533 +[INFO] Current question consumed time: 3698 ms, average: 2532.33 ms/ques, entity detection: 397.25 ms/ques (linking tool: 318.58 ms/ques), relation detection: 847.83 ms/ques (similarity: 697.24 ms/ques), EDG generation: 47.33 ms/ques [INFO] Entity Linking P: 0.74822695035461, R: 0.6985815602836879, micro F1: 0.7144208037825069, macro F1: 0.7225524961757752 [INFO] Relation Linking P: 0.6013002364066193, R: 0.5475177304964539, micro F1: 0.5626950354609935, macro F1: 0.5731500555685689 [INFO] Type Linking P: 0.5446808510638298, R: 0.5446808510638298, micro F1: 0.5446808510638298, macro F1: 0.5446808510638298 @@ -9901,8 +9901,8 @@ Question 705: Which team won the American Football League? [INFO] Golden Answer: [http://dbpedia.org/resource/Kansas_City_Chiefs] [INFO] Predicted Answer: [http://dbpedia.org/resource/Tommy_O'Connell, http://dbpedia.org/resource/Alvin_Reed, http://dbpedia.org/resource/Bob_McLeod_(American_football), http://dbpedia.org/resource/Charley_Frazier, http://dbpedia.org/resource/Dave_Smith_(fullback), http://dbpedia.org/resource/Don_Trull, http://dbpedia.org/resource/Willard_Dewveall, http://dbpedia.org/resource/Woody_Campbell_(American_football), http://dbpedia.org/resource/Al_Romine, http://dbpedia.org/resource/Bill_Atkins_(American_football), http://dbpedia.org/resource/Brad_Hubbert, http://dbpedia.org/resource/Charlie_Long, http://dbpedia.org/resource/Jim_Tolbert, http://dbpedia.org/resource/Terry_Owens_(American_football), http://dbpedia.org/resource/Don_Norton, http://dbpedia.org/resource/Palmer_Pyle, http://dbpedia.org/resource/Johnny_Green_(American_football), http://dbpedia.org/resource/Jacque_MacKinnon, http://dbpedia.org/resource/Jim_McCusker, http://dbpedia.org/resource/Wayne_Patrick, http://dbpedia.org/resource/Bill_Miller_(wide_receiver), http://dbpedia.org/resource/Tom_Beier, http://dbpedia.org/resource/Bobby_Maples, http://dbpedia.org/resource/Ray_Jacobs, http://dbpedia.org/resource/Bob_Zeman, http://dbpedia.org/resource/Dick_McCabe_(American_football), http://dbpedia.org/resource/Don_Joyce_(American_football), http://dbpedia.org/resource/John_McGeever, http://dbpedia.org/resource/Len_St._Jean, http://dbpedia.org/resource/Howard_Kindig, http://dbpedia.org/resource/Paul_Miller_(defensive_end), http://dbpedia.org/resource/Bob_Svihus, http://dbpedia.org/resource/Bobby_Hunt, http://dbpedia.org/resource/Carl_Garrett, http://dbpedia.org/resource/Chuck_Hurston, http://dbpedia.org/resource/Curt_Merz, http://dbpedia.org/resource/E._J._Holub, http://dbpedia.org/resource/Frank_Pitts, http://dbpedia.org/resource/Jim_Kearney, http://dbpedia.org/resource/Jim_Riley_(American_football), http://dbpedia.org/resource/Mike_Mercer_(American_football), http://dbpedia.org/resource/Pete_Lammons, http://dbpedia.org/resource/Robert_Holmes_(American_football), http://dbpedia.org/resource/Willie_Lanier, http://dbpedia.org/resource/Don_Webb_(American_football), http://dbpedia.org/resource/Charlie_Flowers, http://dbpedia.org/resource/Gary_Garrison, http://dbpedia.org/resource/Mike_Taliaferro, http://dbpedia.org/resource/Sherman_Plunkett, http://dbpedia.org/resource/Steve_Tensi, http://dbpedia.org/resource/Frank_Tripucka, http://dbpedia.org/resource/George_Saimes, http://dbpedia.org/resource/Jack_Spikes, http://dbpedia.org/resource/Keith_Lincoln, http://dbpedia.org/resource/Sam_DeLuca, http://dbpedia.org/resource/Tobin_Rote, http://dbpedia.org/resource/Haven_Moses, http://dbpedia.org/resource/Dave_Webster, http://dbpedia.org/resource/Ron_McDole, http://dbpedia.org/resource/Daryl_Johnson, http://dbpedia.org/resource/Jimmy_Jones_(wide_receiver), http://dbpedia.org/resource/Jackie_Allen_(American_football), http://dbpedia.org/resource/Mark_Johnston_(American_football), http://dbpedia.org/resource/Jimmy_Warren, http://dbpedia.org/resource/Buddy_Alliston, http://dbpedia.org/resource/Dick_Hudson, http://dbpedia.org/resource/Al_Frazier, http://dbpedia.org/resource/Al_Jamison, http://dbpedia.org/resource/Art_Graham, http://dbpedia.org/resource/Bill_Hudson_(American_football), http://dbpedia.org/resource/Bill_Hull, http://dbpedia.org/resource/Bill_Roehnelt, http://dbpedia.org/resource/Billy_Lott, http://dbpedia.org/resource/Billy_Reynolds_(American_football), http://dbpedia.org/resource/Bob_Dougherty, http://dbpedia.org/resource/Bob_Hudson_(offensive_lineman), http://dbpedia.org/resource/Bob_Jackson_(American_football), http://dbpedia.org/resource/Bruce_Alford, http://dbpedia.org/resource/Bud_Whitehead, http://dbpedia.org/resource/Buzz_Guy, http://dbpedia.org/resource/Carl_Larpenter, http://dbpedia.org/resource/Charlie_McNeil, http://dbpedia.org/resource/Chuck_Leo, http://dbpedia.org/resource/Chuck_McMurtry, http://dbpedia.org/resource/Chuck_Shonta, http://dbpedia.org/resource/Cornell_Gordon, http://dbpedia.org/resource/Dainard_Paulson, http://dbpedia.org/resource/Dale_Livingston, http://dbpedia.org/resource/Dalva_Allen, http://dbpedia.org/resource/Dan_Darragh, http://dbpedia.org/resource/Dee_Mackey, http://dbpedia.org/resource/Dennis_Partee, http://dbpedia.org/resource/Dick_Harris_(American_football), http://dbpedia.org/resource/Dick_Westmoreland, http://dbpedia.org/resource/Dudley_Meredith, http://dbpedia.org/resource/Ed_Meyer_(American_football), http://dbpedia.org/resource/Eldridge_Dickey, http://dbpedia.org/resource/Eric_Crabtree, http://dbpedia.org/resource/Fran_Lynch, http://dbpedia.org/resource/Fred_Ford_(American_football), http://dbpedia.org/resource/Garrett_Ford,_Sr., http://dbpedia.org/resource/Gene_Sykes, http://dbpedia.org/resource/Gene_Trosch, http://dbpedia.org/resource/George_Goeddeke, http://dbpedia.org/resource/George_Gross_(American_football), http://dbpedia.org/resource/George_Nock, http://dbpedia.org/resource/Goldie_Sellers, http://dbpedia.org/resource/Harold_Akin, http://dbpedia.org/resource/Harold_Olson, http://dbpedia.org/resource/Howard_Fest, http://dbpedia.org/resource/Jeff_Staggs, http://dbpedia.org/resource/Jerry_Cornelison, http://dbpedia.org/resource/Jerry_Tarr, http://dbpedia.org/resource/Jesse_Murdock, http://dbpedia.org/resource/Jim_Allison, http://dbpedia.org/resource/Jim_Fraser_(American_football), http://dbpedia.org/resource/Jim_Sears, http://dbpedia.org/resource/Jim_Urbanek, http://dbpedia.org/resource/Jim_Waskiewicz, http://dbpedia.org/resource/Joe_O'Donnell_(American_football), http://dbpedia.org/resource/Jon_Gilliam, http://dbpedia.org/resource/Junior_Wren, http://dbpedia.org/resource/Justin_Canale, http://dbpedia.org/resource/Karl_Noonan, http://dbpedia.org/resource/Kenny_Graham_(American_football), http://dbpedia.org/resource/Lee_White, http://dbpedia.org/resource/Leroy_Mitchell, http://dbpedia.org/resource/Marshall_Starks, http://dbpedia.org/resource/Marvin_Terrell, http://dbpedia.org/resource/Mike_Dukes, http://dbpedia.org/resource/Noland_Smith, http://dbpedia.org/resource/Orville_Trask, http://dbpedia.org/resource/Pat_Holmes, http://dbpedia.org/resource/Pat_Matson, http://dbpedia.org/resource/Pete_Jaquess, http://dbpedia.org/resource/Pete_Perreault, http://dbpedia.org/resource/Ray_Abruzzese, http://dbpedia.org/resource/Rich_Michael, http://dbpedia.org/resource/Royce_Womble, http://dbpedia.org/resource/Sam_Gruneisen, http://dbpedia.org/resource/Scotty_Glacken, http://dbpedia.org/resource/Sid_Fournet, http://dbpedia.org/resource/Speedy_Thomas, http://dbpedia.org/resource/Steve_DeLong, http://dbpedia.org/resource/Steve_Quinn, http://dbpedia.org/resource/Ted_Wegert, http://dbpedia.org/resource/Tom_Janik, http://dbpedia.org/resource/Tom_Pennington, http://dbpedia.org/resource/Tom_Rychlec, http://dbpedia.org/resource/Tony_Teresa, http://dbpedia.org/resource/Whit_Canale, http://dbpedia.org/resource/Willie_Frazier, http://dbpedia.org/resource/Randy_Edmunds, http://dbpedia.org/resource/Mike_D'Amato, http://dbpedia.org/resource/Ed_Rutkowski, http://dbpedia.org/resource/Alan_Miller_(American_football), http://dbpedia.org/resource/Dave_Costa, http://dbpedia.org/resource/Tom_Keating_(American_football), http://dbpedia.org/resource/Tom_Morrow, http://dbpedia.org/resource/Warren_Wells, http://dbpedia.org/resource/Ollie_Spencer, http://dbpedia.org/resource/George_Daney, http://dbpedia.org/resource/Ralph_Felton, http://dbpedia.org/resource/Bob_Laraba, http://dbpedia.org/resource/Hunter_Enis, http://dbpedia.org/resource/W._K._Hicks, http://dbpedia.org/resource/Dave_Hill_(American_football), http://dbpedia.org/resource/Bobby_Smith_(running_back), http://dbpedia.org/resource/Al_Reynolds, http://dbpedia.org/resource/Stewart_Stover, http://dbpedia.org/resource/Jack_Kemp__6] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 706, P: 0.508, R: 0.559, macro F1: 0.514, macro F1*: 0.532 -[INFO] Current time: 1250 ms, average: 2804.30 ms/ques, entity detection: 448.31 ms/ques (linking tool: 355.44 ms/ques), relation detection: 916.25 ms/ques (similarity: 706.56 ms/ques), EDG generation: 47.33 ms/ques +[INFO] Cumulative metrics, sample: 706, P: 0.508, R: 0.559, macro F1: 0.532 +[INFO] Current question consumed time: 1109 ms, average: 2530.31 ms/ques, entity detection: 396.92 ms/ques (linking tool: 318.30 ms/ques), relation detection: 847.06 ms/ques (similarity: 696.70 ms/ques), EDG generation: 47.32 ms/ques [INFO] Entity Linking P: 0.7485835694050992, R: 0.6990084985835694, micro F1: 0.7148253068932966, macro F1: 0.7229471457953355 [INFO] Relation Linking P: 0.6004485363550519, R: 0.546742209631728, micro F1: 0.5618980169971678, macro F1: 0.5723382282943925 [INFO] Type Linking P: 0.5453257790368272, R: 0.5453257790368272, micro F1: 0.5453257790368272, macro F1: 0.5453257790368272 @@ -9915,8 +9915,8 @@ Question 706: Which currency can I use both in Kerguelen islands and French sout [INFO] Golden Answer: [http://dbpedia.org/resource/Euro] [INFO] Predicted Answer: [http://dbpedia.org/resource/Euro] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 707, P: 0.509, R: 0.559, macro F1: 0.515, macro F1*: 0.533 -[INFO] Current time: 3932 ms, average: 2805.89 ms/ques, entity detection: 448.02 ms/ques (linking tool: 355.19 ms/ques), relation detection: 916.86 ms/ques (similarity: 707.77 ms/ques), EDG generation: 47.33 ms/ques +[INFO] Cumulative metrics, sample: 707, P: 0.509, R: 0.559, macro F1: 0.533 +[INFO] Current question consumed time: 3895 ms, average: 2532.24 ms/ques, entity detection: 396.65 ms/ques (linking tool: 318.05 ms/ques), relation detection: 847.58 ms/ques (similarity: 697.93 ms/ques), EDG generation: 47.34 ms/ques [INFO] Entity Linking P: 0.748939179632249, R: 0.6994342291371994, micro F1: 0.7152286657237164, macro F1: 0.7233406725159124 [INFO] Relation Linking P: 0.6010136727958509, R: 0.5473833097595474, micro F1: 0.5625176803394631, macro F1: 0.5729462170715246 [INFO] Type Linking P: 0.5445544554455446, R: 0.5445544554455446, micro F1: 0.5445544554455446, macro F1: 0.5445544554455446 @@ -9929,8 +9929,8 @@ Question 707: Who is the founder of the trade union which affiliates Amalgamated [INFO] Golden Answer: [1935-11-14, http://dbpedia.org/resource/Pittsburgh] [INFO] Predicted Answer: [http://dbpedia.org/resource/Daniel_De_Leon, http://dbpedia.org/resource/David_Triesman,_Baron_Triesman, http://dbpedia.org/resource/Mikhail_Tomsky, http://dbpedia.org/resource/Vincent_Saint_John, http://dbpedia.org/resource/Andy_Gilchrist, http://dbpedia.org/resource/Tefere_Gebre, http://dbpedia.org/resource/Fritz_Kater, http://dbpedia.org/resource/Savumiamoorthy_Thondaman, http://dbpedia.org/resource/Bill_Routley, http://dbpedia.org/resource/Charles_Collins_(Queensland_politician), http://dbpedia.org/resource/Herbert_Hardacre, http://dbpedia.org/resource/Johnno_Mann, http://dbpedia.org/resource/Maurice_Hynes, http://dbpedia.org/resource/Samuel_Brassington, http://dbpedia.org/resource/Rodney_Bickerstaffe, http://dbpedia.org/resource/J._J._Kenneally, http://dbpedia.org/resource/Thomas_J._O'Connell, http://dbpedia.org/resource/William_X._O'Brien, http://dbpedia.org/resource/Wong_Kwok-hing, http://dbpedia.org/resource/Alhaji_Alieu_Ebrima_Cham_Joof, http://dbpedia.org/resource/Mick_Gardner, http://dbpedia.org/resource/Daniel_Santalla, http://dbpedia.org/resource/Anderson_Dawson, http://dbpedia.org/resource/Brenda_Dean,_Baroness_Dean_of_Thornton-le-Fylde, http://dbpedia.org/resource/Cesar_Chavez, http://dbpedia.org/resource/David_Bowman_(Australian_politician), http://dbpedia.org/resource/George_Meany, http://dbpedia.org/resource/George_Wright_(trade_unionist), http://dbpedia.org/resource/Ted_Theodore, http://dbpedia.org/resource/Thomas_Johnson_(Irish_politician), http://dbpedia.org/resource/Farrell_Dobbs, http://dbpedia.org/resource/Wim_Kok, http://dbpedia.org/resource/Gus_Hall, http://dbpedia.org/resource/Willem_Aantjes, http://dbpedia.org/resource/César_Andreu_Iglesias, http://dbpedia.org/resource/Tam_Yiu-chung, http://dbpedia.org/resource/Mark_Furner, http://dbpedia.org/resource/Tom_Mooney_(educator), http://dbpedia.org/resource/Solly_Sachs, http://dbpedia.org/resource/Arlene_Holt_Baker, http://dbpedia.org/resource/David_Rolf, http://dbpedia.org/resource/William_Carron,_Baron_Carron, http://dbpedia.org/resource/Elsa_Cladera_de_Bravo, http://dbpedia.org/resource/Anna_Walentynowicz, http://dbpedia.org/resource/Barend_Biesheuvel, http://dbpedia.org/resource/Dolores_Huerta, http://dbpedia.org/resource/Tom_Sawyer,_Baron_Sawyer, http://dbpedia.org/resource/Tony_Mazzocchi, http://dbpedia.org/resource/Pepita_Carpeña, http://dbpedia.org/resource/David_Holmes_(trade_unionist), http://dbpedia.org/resource/Hassan_Yussuff, http://dbpedia.org/resource/Michael_Williamson_(Australian_unionist), http://dbpedia.org/resource/Karen_Lewis_(labor_leader), http://dbpedia.org/resource/Alec_Campbell, http://dbpedia.org/resource/Joe_Morris_(trade_unionist), http://dbpedia.org/resource/Michael_Sata, http://dbpedia.org/resource/Earl_Browder, http://dbpedia.org/resource/Frank_Cooper_(Australian_politician), http://dbpedia.org/resource/William_Forgan_Smith, http://dbpedia.org/resource/William_McCormack, http://dbpedia.org/resource/Anil_Moonesinghe, http://dbpedia.org/resource/Missak_Manouchian, http://dbpedia.org/resource/Bob_White_(trade_unionist), http://dbpedia.org/resource/Evilastus_Kaaronda, http://dbpedia.org/resource/John_Kwedhi, http://dbpedia.org/resource/Brian_Davis_(politician), http://dbpedia.org/resource/Josephine_Casey, http://dbpedia.org/resource/Richard_Briginshaw,_Baron_Briginshaw, http://dbpedia.org/resource/Michael_Mulgrew, http://dbpedia.org/resource/Isaac_Myers, http://dbpedia.org/resource/Tom_Proctor_(trade_unionist), http://dbpedia.org/resource/Albert_Risso, http://dbpedia.org/resource/Joe_Hill, http://dbpedia.org/resource/Timothy_D._Murphy, http://dbpedia.org/resource/Julia_Varley, http://dbpedia.org/resource/Dennis_Van_Roekel, http://dbpedia.org/resource/C._W._Bowerman, http://dbpedia.org/resource/Makhan_Singh_(Kenyan_trade_unionist), http://dbpedia.org/resource/Patrick_Lynch_(Patrolmen's_Benevolent_Association), http://dbpedia.org/resource/Bob_Gibbs_(Australian_politician), http://dbpedia.org/resource/James_Duncan_(union_leader), http://dbpedia.org/resource/Ken_Vaughan, http://dbpedia.org/resource/Lyle_Schuntner, http://dbpedia.org/resource/Ned_Davis, http://dbpedia.org/resource/Jack_Melloy, http://dbpedia.org/resource/R._Sugathan, http://dbpedia.org/resource/Louise_O'Reilly, http://dbpedia.org/resource/Daryl_Reid, http://dbpedia.org/resource/Sam_Church, http://dbpedia.org/resource/Solomon_Lozovsky, http://dbpedia.org/resource/Randi_Weingarten, http://dbpedia.org/resource/Keith_Farnham, http://dbpedia.org/resource/Gladys_Bustamante, http://dbpedia.org/resource/Ken_Georgetti, http://dbpedia.org/resource/Michael_James_MacDonald, http://dbpedia.org/resource/Russell_Cunningham_(Canadian_politician), http://dbpedia.org/resource/Tony_Dubbins, http://dbpedia.org/resource/Grace_Hartman_(trade_unionist), http://dbpedia.org/resource/Jeff_Rose, http://dbpedia.org/resource/George_Lawson_(Australian_politician), http://dbpedia.org/resource/Neil_Roberts_(politician), http://dbpedia.org/resource/Pat_Purcell, http://dbpedia.org/resource/Quintin_O'Connor, http://dbpedia.org/resource/Cecilio_Salazar, http://dbpedia.org/resource/Harry_Coyne, http://dbpedia.org/resource/John_Dufficy, http://dbpedia.org/resource/Rudy_Fernandez_(labor_leader), http://dbpedia.org/resource/Bob_Katter,_Sr., http://dbpedia.org/resource/Darshan_Singh_Canadian, http://dbpedia.org/resource/Mei_Li_Vos, http://dbpedia.org/resource/Jesse_Klaver, http://dbpedia.org/resource/Liz_Shuler, http://dbpedia.org/resource/Bernhardt_Esau, http://dbpedia.org/resource/Irving_Brown, http://dbpedia.org/resource/John_F._Henning, http://dbpedia.org/resource/Rose_Schneiderman, http://dbpedia.org/resource/Lee_Cheuk-yan, http://dbpedia.org/resource/Iain_McNicol, http://dbpedia.org/resource/Luis_Eduardo_Garzón, http://dbpedia.org/resource/Wanja_Lundby-Wedin, http://dbpedia.org/resource/P._T._Daly, http://dbpedia.org/resource/Mary_Fitzgerald_(trade_unionist), http://dbpedia.org/resource/Fred_Newton_(politician), http://dbpedia.org/resource/Jacqueline_B._Vaughn, http://dbpedia.org/resource/William_Power_(Australian_politician), http://dbpedia.org/resource/William_Henley_Rawlings, http://dbpedia.org/resource/Les_Yewdale, http://dbpedia.org/resource/Leo_Crawford, http://dbpedia.org/resource/John_Taylor_(Australian_politician), http://dbpedia.org/resource/Ernest_Purnell, http://dbpedia.org/resource/Henry_Llewellyn_(Queensland_politician), http://dbpedia.org/resource/Robert_Mulvey, http://dbpedia.org/resource/Harsh_Vardhan_(Uttar_Pradesh_politician), http://dbpedia.org/resource/John_Mullan_(Australian_politician), http://dbpedia.org/resource/Lewis_McDonald, http://dbpedia.org/resource/Ted_Hanson, http://dbpedia.org/resource/George_Addes] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 708, P: 0.508, R: 0.559, macro F1: 0.514, macro F1*: 0.532 -[INFO] Current time: 3468 ms, average: 2806.83 ms/ques, entity detection: 448.75 ms/ques (linking tool: 355.79 ms/ques), relation detection: 916.54 ms/ques (similarity: 707.21 ms/ques), EDG generation: 47.36 ms/ques +[INFO] Cumulative metrics, sample: 708, P: 0.508, R: 0.559, macro F1: 0.532 +[INFO] Current question consumed time: 2937 ms, average: 2532.81 ms/ques, entity detection: 397.21 ms/ques (linking tool: 318.54 ms/ques), relation detection: 847.29 ms/ques (similarity: 697.41 ms/ques), EDG generation: 47.36 ms/ques [INFO] Entity Linking P: 0.7478813559322034, R: 0.6984463276836158, micro F1: 0.7142184557438805, macro F1: 0.7223190048993644 [INFO] Relation Linking P: 0.6001647834274952, R: 0.5466101694915254, micro F1: 0.5617231638418085, macro F1: 0.5721369710022145 [INFO] Type Linking P: 0.5437853107344632, R: 0.5437853107344632, micro F1: 0.5437853107344632, macro F1: 0.5437853107344632 @@ -9943,8 +9943,8 @@ Question 708: Which leader of United States House of Representatives was the app [INFO] Golden Answer: [http://dbpedia.org/resource/James_Madison] [INFO] Predicted Answer: [http://dbpedia.org/resource/Paul_Ryan, http://dbpedia.org/resource/Steve_Scalise, http://dbpedia.org/resource/Nancy_Pelosi, http://dbpedia.org/resource/Steny_Hoyer, http://dbpedia.org/resource/Kevin_McCarthy_(California_politician)] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 709, P: 0.507, R: 0.558, macro F1: 0.513, macro F1*: 0.531 -[INFO] Current time: 5976 ms, average: 2811.30 ms/ques, entity detection: 449.93 ms/ques (linking tool: 356.88 ms/ques), relation detection: 917.87 ms/ques (similarity: 707.46 ms/ques), EDG generation: 47.33 ms/ques +[INFO] Cumulative metrics, sample: 709, P: 0.507, R: 0.558, macro F1: 0.531 +[INFO] Current question consumed time: 5102 ms, average: 2536.44 ms/ques, entity detection: 398.20 ms/ques (linking tool: 319.47 ms/ques), relation detection: 848.43 ms/ques (similarity: 697.59 ms/ques), EDG generation: 47.33 ms/ques [INFO] Entity Linking P: 0.7468265162200282, R: 0.6974612129760226, micro F1: 0.7132110954395874, macro F1: 0.721300219278914 [INFO] Relation Linking P: 0.6007287259050305, R: 0.5465444287729196, micro F1: 0.561871180065821, macro F1: 0.5723570485520729 [INFO] Type Linking P: 0.5444287729196051, R: 0.5444287729196051, micro F1: 0.5444287729196051, macro F1: 0.5444287729196051 @@ -9957,8 +9957,8 @@ Question 709: Is fox belong to the phylum of Chordate? [INFO] Golden Answer: [true] [INFO] Predicted Answer: [true] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 710, P: 0.508, R: 0.558, macro F1: 0.514, macro F1*: 0.532 -[INFO] Current time: 2236 ms, average: 2810.49 ms/ques, entity detection: 449.30 ms/ques (linking tool: 356.38 ms/ques), relation detection: 916.58 ms/ques (similarity: 706.47 ms/ques), EDG generation: 47.38 ms/ques +[INFO] Cumulative metrics, sample: 710, P: 0.508, R: 0.558, macro F1: 0.532 +[INFO] Current question consumed time: 2028 ms, average: 2535.72 ms/ques, entity detection: 397.64 ms/ques (linking tool: 319.02 ms/ques), relation detection: 847.23 ms/ques (similarity: 696.61 ms/ques), EDG generation: 47.41 ms/ques [INFO] Entity Linking P: 0.7471830985915493, R: 0.6978873239436619, micro F1: 0.7136150234741795, macro F1: 0.7216943963978806 [INFO] Relation Linking P: 0.6012910798122065, R: 0.5471830985915493, micro F1: 0.5624882629107988, macro F1: 0.5729624964914679 [INFO] Type Linking P: 0.5450704225352113, R: 0.5450704225352113, micro F1: 0.5450704225352113, macro F1: 0.5450704225352113 @@ -9971,8 +9971,8 @@ Question 710: Which researchers received fellowship from Royal Society? [INFO] Golden Answer: [http://dbpedia.org/resource/Arthur_Holmes, http://dbpedia.org/resource/Christopher_Kelk_Ingold, http://dbpedia.org/resource/Cyrus_Chothia, http://dbpedia.org/resource/Denis_Parsons_Burkitt, http://dbpedia.org/resource/Frederic_Bartlett, http://dbpedia.org/resource/George_Emil_Palade, http://dbpedia.org/resource/Hans_Bethe, http://dbpedia.org/resource/Henry_Edward_Armstrong, http://dbpedia.org/resource/Henry_Nicholas_Ridley, http://dbpedia.org/resource/Ian_Stewart_(mathematician), http://dbpedia.org/resource/Jerzy_Neyman, http://dbpedia.org/resource/John_Graham_Kerr, http://dbpedia.org/resource/John_Lennard-Jones, http://dbpedia.org/resource/John_Pople, http://dbpedia.org/resource/Martin_Karplus, http://dbpedia.org/resource/Maurice_Wilkes, http://dbpedia.org/resource/Michael_Levitt, http://dbpedia.org/resource/Peter_D._Mitchell, http://dbpedia.org/resource/Richard_Axel, http://dbpedia.org/resource/Roald_Hoffmann, http://dbpedia.org/resource/Vladimir_Prelog, http://dbpedia.org/resource/Andrew_Mackenzie_(businessman), http://dbpedia.org/resource/Autar_Singh_Paintal, http://dbpedia.org/resource/Christopher_Cockerell, http://dbpedia.org/resource/David_Chilton_Phillips, http://dbpedia.org/resource/David_Wheeler_(British_computer_scientist), http://dbpedia.org/resource/Geoffrey_Grimmett, http://dbpedia.org/resource/George_Nares, http://dbpedia.org/resource/Howard_Martin_Temin, http://dbpedia.org/resource/Julia_Slingo, http://dbpedia.org/resource/Julius_Axelrod, http://dbpedia.org/resource/Leslie_Orgel, http://dbpedia.org/resource/Martin_Hume_Johnson, http://dbpedia.org/resource/Paul_Langevin, http://dbpedia.org/resource/Richard_M._Durbin, http://dbpedia.org/resource/Barry_Marshall, http://dbpedia.org/resource/David_Cox_(statistician), http://dbpedia.org/resource/G._I._Taylor, http://dbpedia.org/resource/George_Henry_Richards, http://dbpedia.org/resource/Linda_B._Buck, http://dbpedia.org/resource/Michael_W._Bevan, http://dbpedia.org/resource/Oliver_Smithies, http://dbpedia.org/resource/Otto_Fritz_Meyerhof, http://dbpedia.org/resource/Reginald_Punnett, http://dbpedia.org/resource/Richard_B._Flavell, http://dbpedia.org/resource/Robert_W._Wood, http://dbpedia.org/resource/Alick_Isaacs, http://dbpedia.org/resource/Andre_Geim, http://dbpedia.org/resource/Bert_Sakmann, http://dbpedia.org/resource/Erwin_Neher, http://dbpedia.org/resource/Harry_Smith_(microbiologist), http://dbpedia.org/resource/James_Collip, http://dbpedia.org/resource/K._VijayRaghavan, http://dbpedia.org/resource/Nikolai_Vavilov, http://dbpedia.org/resource/Roy_J._Glauber, http://dbpedia.org/resource/Anthony_A._Hyman, http://dbpedia.org/resource/Ara_Darzi,_Baron_Darzi_of_Denham, http://dbpedia.org/resource/Arthur_Kornberg, http://dbpedia.org/resource/Athene_Donald, http://dbpedia.org/resource/Bennett_Lewis, http://dbpedia.org/resource/Derek_Barton, http://dbpedia.org/resource/George_Andrew_Olah, http://dbpedia.org/resource/Heinrich_Kayser, http://dbpedia.org/resource/Ian_Horrocks, http://dbpedia.org/resource/Janet_Thornton, http://dbpedia.org/resource/Joanna_Haigh, http://dbpedia.org/resource/Lionel_Penrose, http://dbpedia.org/resource/Louis_Néel, http://dbpedia.org/resource/Mary_F._Lyon, http://dbpedia.org/resource/Michael_Ashburner, http://dbpedia.org/resource/Michael_Dexter, http://dbpedia.org/resource/Michael_Green_(physicist), http://dbpedia.org/resource/William_Hume-Rothery, http://dbpedia.org/resource/George_Martin_Lees, http://dbpedia.org/resource/Thomas_A._Steitz, http://dbpedia.org/resource/Venkatraman_Ramakrishnan, http://dbpedia.org/resource/Arthur_Thomas_Doodson, http://dbpedia.org/resource/Steve_Jones_(biologist), http://dbpedia.org/resource/William_Bertram_Turrill, http://dbpedia.org/resource/Edward_Collingwood, http://dbpedia.org/resource/Nicholas_Kemmer, http://dbpedia.org/resource/Trevor_Wooley, http://dbpedia.org/resource/William_Edward_Ayrton, http://dbpedia.org/resource/Enrico_Coen, http://dbpedia.org/resource/George_D._W._Smith, http://dbpedia.org/resource/H._E._Hinton, http://dbpedia.org/resource/Jeremiah_P._Ostriker, http://dbpedia.org/resource/Louise_Johnson, http://dbpedia.org/resource/Robert_W._Cahn, http://dbpedia.org/resource/Vincent_Wigglesworth, http://dbpedia.org/resource/David_Spiegelhalter, http://dbpedia.org/resource/Francis_Simon, http://dbpedia.org/resource/Kurt_Mendelssohn, http://dbpedia.org/resource/Martin_Green_(professor), http://dbpedia.org/resource/Maurice_Hill_(geophysicist), http://dbpedia.org/resource/Pehr_Victor_Edman, http://dbpedia.org/resource/Kenneth_B._Storey, http://dbpedia.org/resource/Harry_Godwin, http://dbpedia.org/resource/James_Julian_Bennett_Jack, http://dbpedia.org/resource/William_Noel_Benson, http://dbpedia.org/resource/Eleanor_Dodson, http://dbpedia.org/resource/Tom_Blundell, http://dbpedia.org/resource/Alexander_Ogston, http://dbpedia.org/resource/Joshua_Harold_Burn, http://dbpedia.org/resource/John_Allan_Broun, http://dbpedia.org/resource/Gareth_Roberts_(statistician), http://dbpedia.org/resource/M._S._Bartlett, http://dbpedia.org/resource/Terry_Speed, http://dbpedia.org/resource/Leslie_H._Martin, http://dbpedia.org/resource/Hans_Heilbronn, http://dbpedia.org/resource/Leonard_Bairstow, http://dbpedia.org/resource/Arthur_Mourant, http://dbpedia.org/resource/Graham_Liggins, http://dbpedia.org/resource/Peter_Gluckman, http://dbpedia.org/resource/Derek_Jackson, http://dbpedia.org/resource/Andrew_Robertson_(engineer), http://dbpedia.org/resource/Bas_Pease, http://dbpedia.org/resource/Nicholas_Lydon, http://dbpedia.org/resource/Denis_Noble, http://dbpedia.org/resource/W._B._R._King, http://dbpedia.org/resource/Francis_Cole, http://dbpedia.org/resource/Philip_Saffman, http://dbpedia.org/resource/M._G._K._Menon, http://dbpedia.org/resource/Michele_Dougherty, http://dbpedia.org/resource/Terri_Attwood, http://dbpedia.org/resource/Hubert_Markl, http://dbpedia.org/resource/Richard_V._Southwell, http://dbpedia.org/resource/Arthur_Felix, http://dbpedia.org/resource/Bob_Vaughan, http://dbpedia.org/resource/Cresswell_Shearer, http://dbpedia.org/resource/John_Frederick_Clarke, http://dbpedia.org/resource/Elizabeth_Robertson, http://dbpedia.org/resource/Rory_Collins, http://dbpedia.org/resource/Richard_Hynes, http://dbpedia.org/resource/Raphael_Weldon, http://dbpedia.org/resource/Frederick_Gugenheim_Gregory, http://dbpedia.org/resource/Herbert_Turnbull, http://dbpedia.org/resource/Amanda_Fisher, http://dbpedia.org/resource/Jenny_Nelson, http://dbpedia.org/resource/P._A._Buxton, http://dbpedia.org/resource/John_Dick_(biologist), http://dbpedia.org/resource/John_Alfred_Valentine_Butler, http://dbpedia.org/resource/Bruce_Bilby, http://dbpedia.org/resource/Chris_Hunter_(chemist), http://dbpedia.org/resource/Abdus_Salam, http://dbpedia.org/resource/Alain_Aspect, http://dbpedia.org/resource/Alan_Turing, http://dbpedia.org/resource/Albert_Einstein, http://dbpedia.org/resource/Alexander_Fleming, http://dbpedia.org/resource/Andrew_Huxley, http://dbpedia.org/resource/André_Weil, http://dbpedia.org/resource/Arnold_Sommerfeld, http://dbpedia.org/resource/Arthur_Eddington, http://dbpedia.org/resource/Béla_Bollobás, http://dbpedia.org/resource/C._V._Raman, http://dbpedia.org/resource/Charles_Darwin, http://dbpedia.org/resource/David_Hilbert, http://dbpedia.org/resource/Dmitri_Mendeleev, http://dbpedia.org/resource/Donald_Knuth, http://dbpedia.org/resource/Douglas_Hartree, http://dbpedia.org/resource/Edmond_H._Fischer, http://dbpedia.org/resource/Enrico_Fermi, http://dbpedia.org/resource/Ernest_Barnes, http://dbpedia.org/resource/Ernest_Marsden, http://dbpedia.org/resource/Ernst_Mayr, http://dbpedia.org/resource/Feodor_Felix_Konrad_Lynen, http://dbpedia.org/resource/Frank_Whittle, http://dbpedia.org/resource/Frederick_Sanger, http://dbpedia.org/resource/Frederick_Soddy, http://dbpedia.org/resource/Freeman_Dyson, http://dbpedia.org/resource/Geoffrey_Hinton, http://dbpedia.org/resource/George_de_Hevesy, http://dbpedia.org/resource/Gilbert_N._Lewis, http://dbpedia.org/resource/Glenn_T._Seaborg, http://dbpedia.org/resource/Henri_Lebesgue, http://dbpedia.org/resource/Henry_Drysdale_Dakin, http://dbpedia.org/resource/Irving_Langmuir, http://dbpedia.org/resource/Isaac_Newton, http://dbpedia.org/resource/Ivan_Pavlov, http://dbpedia.org/resource/Jacques_Friedel, http://dbpedia.org/resource/James_Chadwick, http://dbpedia.org/resource/James_George_Frazer, http://dbpedia.org/resource/James_Watson, http://dbpedia.org/resource/Johannes_Rydberg, http://dbpedia.org/resource/John_Tuzo_Wilson, http://dbpedia.org/resource/John_Venn, http://dbpedia.org/resource/Jules_Bordet, http://dbpedia.org/resource/Kurt_Gödel, http://dbpedia.org/resource/Lise_Meitner, http://dbpedia.org/resource/Louis_Pasteur, http://dbpedia.org/resource/M._J._Seaton, http://dbpedia.org/resource/Martin_Fleischmann, http://dbpedia.org/resource/Max_Born, http://dbpedia.org/resource/Max_Newman, http://dbpedia.org/resource/Michael_Mingos, http://dbpedia.org/resource/Michael_Smith_(chemist), http://dbpedia.org/resource/Niels_Bohr, http://dbpedia.org/resource/Nikolaas_Tinbergen, http://dbpedia.org/resource/Norman_Borlaug, http://dbpedia.org/resource/Otto_Hahn, http://dbpedia.org/resource/Otto_Heinrich_Warburg, http://dbpedia.org/resource/Otto_Struve, http://dbpedia.org/resource/Owen_Willans_Richardson, http://dbpedia.org/resource/Paul_J._Crutzen, http://dbpedia.org/resource/Percy_Williams_Bridgman, http://dbpedia.org/resource/Rajesh_Thakker, http://dbpedia.org/resource/Richard_Dawkins, http://dbpedia.org/resource/Robert_Boyle, http://dbpedia.org/resource/Roger_Penrose, http://dbpedia.org/resource/Satyendra_Nath_Bose, http://dbpedia.org/resource/Sigmund_Freud, http://dbpedia.org/resource/Stephen_Hawking, http://dbpedia.org/resource/Stephen_O'Rahilly, http://dbpedia.org/resource/Subrahmanyan_Chandrasekhar, http://dbpedia.org/resource/Svante_Arrhenius, http://dbpedia.org/resource/Tim_Berners-Lee, http://dbpedia.org/resource/Werner_Heisenberg, http://dbpedia.org/resource/Allan_Wilson, http://dbpedia.org/resource/Antony_Hewish, http://dbpedia.org/resource/Brian_Schmidt, http://dbpedia.org/resource/David_Tabor, http://dbpedia.org/resource/Gertrude_B._Elion, http://dbpedia.org/resource/Gilbert_Walker, http://dbpedia.org/resource/Harold_Davenport, http://dbpedia.org/resource/J._Michael_Bishop, http://dbpedia.org/resource/John_Maddox, http://dbpedia.org/resource/Marian_Dawkins, http://dbpedia.org/resource/Martin_Hairer, http://dbpedia.org/resource/Renato_Dulbecco, http://dbpedia.org/resource/Roderick_Oliver_Redman, http://dbpedia.org/resource/Sergei_Winogradsky, http://dbpedia.org/resource/Shanti_Swaroop_Bhatnagar, http://dbpedia.org/resource/Tim_Hunt, http://dbpedia.org/resource/César_Milstein, http://dbpedia.org/resource/David_Weatherall, http://dbpedia.org/resource/Dominic_Joyce, http://dbpedia.org/resource/Edward_Augustus_Inglefield, http://dbpedia.org/resource/J._A._Todd, http://dbpedia.org/resource/John_A._Peacock, http://dbpedia.org/resource/Linda_Partridge, http://dbpedia.org/resource/Paul_Midgley, http://dbpedia.org/resource/Richard_S._Ward, http://dbpedia.org/resource/Robin_Hill_(biochemist), http://dbpedia.org/resource/Samuel_Tolansky, http://dbpedia.org/resource/Frédéric_Cuvier, http://dbpedia.org/resource/John_Ziman, http://dbpedia.org/resource/Russell_Lande, http://dbpedia.org/resource/Alfred_Ubbelohde, http://dbpedia.org/resource/Archibald_Garrod, http://dbpedia.org/resource/John_Charnley, http://dbpedia.org/resource/P.A._Sheppard, http://dbpedia.org/resource/Philip_Burton_Moon, http://dbpedia.org/resource/Ronnie_Bell, http://dbpedia.org/resource/Rosalind_Pitt-Rivers, http://dbpedia.org/resource/Arthur_Philemon_Coleman, http://dbpedia.org/resource/David_Chapman_(chemist), http://dbpedia.org/resource/Edward_Hodges_Baily, http://dbpedia.org/resource/Howard_Douglas, http://dbpedia.org/resource/Wilfred_Trotter, http://dbpedia.org/resource/Joseph_Wedderburn, http://dbpedia.org/resource/Max_Carl_Wilhelm_Weber, http://dbpedia.org/resource/Sheena_Radford, http://dbpedia.org/resource/Arthur_Lee_Dixon, http://dbpedia.org/resource/Bryan_Webber, http://dbpedia.org/resource/Dorothy_Bishop_(psychologist), http://dbpedia.org/resource/Georg_Gottlob, http://dbpedia.org/resource/Hagan_Bayley, http://dbpedia.org/resource/Henning_Sirringhaus, http://dbpedia.org/resource/Henry_Snaith, http://dbpedia.org/resource/Lorna_Casselton, http://dbpedia.org/resource/Martin_J._Taylor, http://dbpedia.org/resource/Nevil_Sidgwick, http://dbpedia.org/resource/Paul_H._Harvey, http://dbpedia.org/resource/Richard_Friend, http://dbpedia.org/resource/Robin_Saxby, http://dbpedia.org/resource/Warrington_Yorke, http://dbpedia.org/resource/Frank_Edward_Smith, http://dbpedia.org/resource/Frederick_Stewart_(geologist), http://dbpedia.org/resource/Georgina_Mace, http://dbpedia.org/resource/Nicholas_Higham, http://dbpedia.org/resource/Eric_Burhop, http://dbpedia.org/resource/David_Forbes_Martyn, http://dbpedia.org/resource/Geoffrey_Eglinton, http://dbpedia.org/resource/Maurice_Lugeon, http://dbpedia.org/resource/Owen_Thomas_Jones, http://dbpedia.org/resource/Richard_Alley, http://dbpedia.org/resource/Alexander_Pines, http://dbpedia.org/resource/Heinrich_Gerhard_Kuhn, http://dbpedia.org/resource/Samuel_Milner, http://dbpedia.org/resource/John_Hardy_(geneticist), http://dbpedia.org/resource/Leslie_Reginald_Cox, http://dbpedia.org/resource/Petrus_Johann_du_Toit, http://dbpedia.org/resource/Janet_Hemingway, http://dbpedia.org/resource/Govind_Swarup, http://dbpedia.org/resource/Jens_Marklof, http://dbpedia.org/resource/John_Gaddum, http://dbpedia.org/resource/Otto_Maass, http://dbpedia.org/resource/Léo_Marion, http://dbpedia.org/resource/David_Delpy, http://dbpedia.org/resource/Frank_Ewart_Smith, http://dbpedia.org/resource/Timothy_Bliss, http://dbpedia.org/resource/Winifred_Pennington, http://dbpedia.org/resource/Gordon_Elliott_Fogg, http://dbpedia.org/resource/Andrew_F._Read, http://dbpedia.org/resource/Walter_Frederick_Whittard, http://dbpedia.org/resource/William_George_Fearnsides, http://dbpedia.org/resource/Leonard_Frank_Spath, http://dbpedia.org/resource/William_Dickson_Lang, http://dbpedia.org/resource/Douglas_Fearon, http://dbpedia.org/resource/Margaret_Buckingham, http://dbpedia.org/resource/Quentin_Gibson, http://dbpedia.org/resource/Timothy_Eglinton, http://dbpedia.org/resource/Nicholas_Hastie, http://dbpedia.org/resource/Bernard_Keen, http://dbpedia.org/resource/Adrian_Bird, http://dbpedia.org/resource/Alan_M._Roberts, http://dbpedia.org/resource/Alan_MacDiarmid, http://dbpedia.org/resource/Almroth_Wright, http://dbpedia.org/resource/Andrey_Kolmogorov, http://dbpedia.org/resource/Arnold_Wolfendale, http://dbpedia.org/resource/Arthur_B._McDonald, http://dbpedia.org/resource/Arthur_Robert_Hinks, http://dbpedia.org/resource/Arthur_Schuster, http://dbpedia.org/resource/Ben_G._Davis, http://dbpedia.org/resource/Bengt_I._Samuelsson, http://dbpedia.org/resource/Brian_Charlesworth, http://dbpedia.org/resource/Caroline_Dean, http://dbpedia.org/resource/David_Colquhoun, http://dbpedia.org/resource/David_Hopwood, http://dbpedia.org/resource/David_J._C._MacKay, http://dbpedia.org/resource/David_King_(chemist), http://dbpedia.org/resource/David_Rees_(mathematician), http://dbpedia.org/resource/E._B._Ford, http://dbpedia.org/resource/Edgeworth_David, http://dbpedia.org/resource/Edward_Abraham, http://dbpedia.org/resource/Ernst_Boris_Chain, http://dbpedia.org/resource/F._Albert_Cotton, http://dbpedia.org/resource/Frédéric_Joliot-Curie, http://dbpedia.org/resource/George_H._Hitchings, http://dbpedia.org/resource/Gerhard_Domagk, http://dbpedia.org/resource/Godfrey_Hounsfield, http://dbpedia.org/resource/Hannes_Alfvén, http://dbpedia.org/resource/Hans_Sloane, http://dbpedia.org/resource/Harold_Jeffreys, http://dbpedia.org/resource/Heinrich_Otto_Wieland, http://dbpedia.org/resource/Henri_Becquerel, http://dbpedia.org/resource/Hermann_Emil_Fischer, http://dbpedia.org/resource/Hermann_Weyl, http://dbpedia.org/resource/Howard_Florey, http://dbpedia.org/resource/Jacobus_Henricus_van_'t_Hoff, http://dbpedia.org/resource/Jane_A._Langdale, http://dbpedia.org/resource/Jean_le_Rond_d'Alembert, http://dbpedia.org/resource/John_Argyris, http://dbpedia.org/resource/John_Nelder, http://dbpedia.org/resource/Joseph_Dalton_Hooker, http://dbpedia.org/resource/Joseph_L._Goldstein, http://dbpedia.org/resource/Joseph_Rotblat, http://dbpedia.org/resource/Kenneth_Arrow, http://dbpedia.org/resource/Konstantin_Novoselov, http://dbpedia.org/resource/Leslie_Comrie, http://dbpedia.org/resource/Mark_Oliphant, http://dbpedia.org/resource/Martin_Ryle, http://dbpedia.org/resource/Matthew_Boulton, http://dbpedia.org/resource/Michael_Benton, http://dbpedia.org/resource/Michael_Rutter, http://dbpedia.org/resource/Michael_Stuart_Brown, http://dbpedia.org/resource/Nevill_Francis_Mott, http://dbpedia.org/resource/Paul_Dirac, http://dbpedia.org/resource/Peter_Morris_(surgeon), http://dbpedia.org/resource/Reginald_Victor_Jones, http://dbpedia.org/resource/Richard_Feynman, http://dbpedia.org/resource/Richard_Laws, http://dbpedia.org/resource/Richard_R._Ernst, http://dbpedia.org/resource/Richard_Sykes_(biochemist), http://dbpedia.org/resource/Richard_van_der_Riet_Woolley, http://dbpedia.org/resource/Rita_Levi-Montalcini, http://dbpedia.org/resource/Robert_Koch, http://dbpedia.org/resource/Robert_S._Mulliken, http://dbpedia.org/resource/Robert_Watson-Watt, http://dbpedia.org/resource/Salimuzzaman_Siddiqui, http://dbpedia.org/resource/Steven_Weinberg, http://dbpedia.org/resource/Theodor_Svedberg, http://dbpedia.org/resource/Vilhelm_Bjerknes, http://dbpedia.org/resource/William_Eccles, http://dbpedia.org/resource/William_Parks_(paleontologist), http://dbpedia.org/resource/Clifford_Cocks, http://dbpedia.org/resource/David_Keilin, http://dbpedia.org/resource/Donald_O._Hebb, http://dbpedia.org/resource/Edmund_Clifton_Stoner, http://dbpedia.org/resource/Gilles_Brassard, http://dbpedia.org/resource/Gordon_Plotkin, http://dbpedia.org/resource/Ian_Wilmut, http://dbpedia.org/resource/James_H._Wilkinson, http://dbpedia.org/resource/James_Lovelock, http://dbpedia.org/resource/John_Desmond_Bernal, http://dbpedia.org/resource/John_Tukey, http://dbpedia.org/resource/Jon_Crowcroft, http://dbpedia.org/resource/Jonathan_Pila, http://dbpedia.org/resource/Kathleen_Lonsdale, http://dbpedia.org/resource/Keith_Glover, http://dbpedia.org/resource/Leslie_Valiant, http://dbpedia.org/resource/Luca_Cardelli, http://dbpedia.org/resource/Max_Perutz, http://dbpedia.org/resource/Ralph_H._Fowler, http://dbpedia.org/resource/Richard_J._Roberts, http://dbpedia.org/resource/Robin_Milner, http://dbpedia.org/resource/Ross_J._Anderson, http://dbpedia.org/resource/Samson_Abramsky, http://dbpedia.org/resource/Vito_Volterra, http://dbpedia.org/resource/Zoubin_Ghahramani, http://dbpedia.org/resource/Andrew_Lyne, http://dbpedia.org/resource/Barbara_Pearse, http://dbpedia.org/resource/Charles_Thomson_Rees_Wilson, http://dbpedia.org/resource/Dennis_Gabor, http://dbpedia.org/resource/Douglas_Mawson, http://dbpedia.org/resource/Ernest_William_Brown, http://dbpedia.org/resource/Evan_James_Williams, http://dbpedia.org/resource/George_E._P._Box, http://dbpedia.org/resource/Harold_A._Wilson_(physicist), http://dbpedia.org/resource/James_Franck, http://dbpedia.org/resource/John_Jacob_Abel, http://dbpedia.org/resource/Kenichi_Fukui, http://dbpedia.org/resource/Patrick_Blackett, http://dbpedia.org/resource/Peter_Higgs, http://dbpedia.org/resource/Richard_Borcherds, http://dbpedia.org/resource/Tony_Hoare, http://dbpedia.org/resource/Ulf_von_Euler, http://dbpedia.org/resource/Walter_Bradford_Cannon, http://dbpedia.org/resource/Walter_Sydney_Adams, http://dbpedia.org/resource/A._W._F._Edwards, http://dbpedia.org/resource/Arthur_Keith, http://dbpedia.org/resource/Frederick_Thomas_Trouton, http://dbpedia.org/resource/Harish-Chandra, http://dbpedia.org/resource/Joseph_Larmor, http://dbpedia.org/resource/Richard_E._Taylor, http://dbpedia.org/resource/Ronald_Ross, http://dbpedia.org/resource/Wilfrid_Le_Gros_Clark, http://dbpedia.org/resource/G._M._B._Dobson, http://dbpedia.org/resource/Henry_Gilman, http://dbpedia.org/resource/Henry_Louis_Le_Châtelier, http://dbpedia.org/resource/John_Hasbrouck_Van_Vleck, http://dbpedia.org/resource/Karl_Lashley, http://dbpedia.org/resource/Kurt_Wüthrich, http://dbpedia.org/resource/Mark_Welland, http://dbpedia.org/resource/Nancy_Rothwell, http://dbpedia.org/resource/Paul_Cohn, http://dbpedia.org/resource/Peter_Bruce, http://dbpedia.org/resource/Peter_C._Doherty, http://dbpedia.org/resource/Philip_Maini, http://dbpedia.org/resource/Rodney_Robert_Porter, http://dbpedia.org/resource/Roger_Heath-Brown, http://dbpedia.org/resource/Timothy_Leighton, http://dbpedia.org/resource/Wallace_L._W._Sargent, http://dbpedia.org/resource/Jean_de_Hautefeuille, http://dbpedia.org/resource/Michael_J._S._Dewar, http://dbpedia.org/resource/David_George_Kendall, http://dbpedia.org/resource/János_Szentágothai, http://dbpedia.org/resource/Trevor_Robbins, http://dbpedia.org/resource/Claude_Ambrose_Rogers, http://dbpedia.org/resource/David_B._A._Epstein, http://dbpedia.org/resource/Harold_Hopkins_(physicist), http://dbpedia.org/resource/John_Rarity, http://dbpedia.org/resource/Steven_Cowley, http://dbpedia.org/resource/Jack_Drummond, http://dbpedia.org/resource/Richard_Rado, http://dbpedia.org/resource/David_Rivett, http://dbpedia.org/resource/George_Simpson_(meteorologist), http://dbpedia.org/resource/Robert_Thomson_Leiper, http://dbpedia.org/resource/Reinhard_Genzel, http://dbpedia.org/resource/Bai_Chunli, http://dbpedia.org/resource/William_Symonds, http://dbpedia.org/resource/S._Francis_Boys, http://dbpedia.org/resource/William_Stewart_(biologist), http://dbpedia.org/resource/Karen_Vousden, http://dbpedia.org/resource/Anthony_Cheetham, http://dbpedia.org/resource/Charles_Kemball, http://dbpedia.org/resource/Margaret_Gowing, http://dbpedia.org/resource/Raymond_Baker_(chemist), http://dbpedia.org/resource/Stephen_Jackson_(scientist), http://dbpedia.org/resource/Joseph_Proudman, http://dbpedia.org/resource/William_Michael_Herbert_Greaves, http://dbpedia.org/resource/Ian_Heilbron, http://dbpedia.org/resource/Simon_Lilly, http://dbpedia.org/resource/Hugh_Pelham, http://dbpedia.org/resource/Mark_Child, http://dbpedia.org/resource/Frederick_Stratten_Russell, http://dbpedia.org/resource/James_Peter_Hill, http://dbpedia.org/resource/Nigel_Weiss, http://dbpedia.org/resource/Tom_McLeish, http://dbpedia.org/resource/Clifford_H._Mortimer, http://dbpedia.org/resource/Aaron_Klug, http://dbpedia.org/resource/Alan_Bundy, http://dbpedia.org/resource/Alan_Fersht, http://dbpedia.org/resource/Alison_Etheridge, http://dbpedia.org/resource/Annette_Dolphin, http://dbpedia.org/resource/Arne_Tiselius, http://dbpedia.org/resource/Bernard_Katz, http://dbpedia.org/resource/Bruce_Ponder, http://dbpedia.org/resource/Carol_V._Robinson, http://dbpedia.org/resource/David_Baulcombe, http://dbpedia.org/resource/E._T._Whittaker, http://dbpedia.org/resource/Edmund_Hirst, http://dbpedia.org/resource/Edward_Victor_Appleton, http://dbpedia.org/resource/Eustace_Tennyson_d'Eyncourt, http://dbpedia.org/resource/Flinders_Petrie, http://dbpedia.org/resource/Francis_Crick, http://dbpedia.org/resource/Frederick_W._Lanchester, http://dbpedia.org/resource/Friedrich_Hirzebruch, http://dbpedia.org/resource/Gavin_de_Beer, http://dbpedia.org/resource/H._Robert_Horvitz, http://dbpedia.org/resource/Hendrik_Lorentz, http://dbpedia.org/resource/Hermann_Bondi, http://dbpedia.org/resource/Hermann_Hauser, http://dbpedia.org/resource/Hermann_Joseph_Muller, http://dbpedia.org/resource/John_H._Coates, http://dbpedia.org/resource/John_Krebs,_Baron_Krebs, http://dbpedia.org/resource/John_Maynard_Smith, http://dbpedia.org/resource/John_Sulston, http://dbpedia.org/resource/John_Vane, http://dbpedia.org/resource/Joseph_Whitworth, http://dbpedia.org/resource/Mary_Cartwright, http://dbpedia.org/resource/Oliver_Heaviside, http://dbpedia.org/resource/Oswald_Avery, http://dbpedia.org/resource/Peter_Colman, http://dbpedia.org/resource/Peter_Goodfellow, http://dbpedia.org/resource/Philip_Donoghue, http://dbpedia.org/resource/Pyotr_Kapitsa, http://dbpedia.org/resource/Robert_Edwards_(physiologist), http://dbpedia.org/resource/Shiing-Shen_Chern, http://dbpedia.org/resource/Solomon_Lefschetz, http://dbpedia.org/resource/Sophie_Wilson, http://dbpedia.org/resource/Steve_Furber, http://dbpedia.org/resource/Sydney_Brenner, http://dbpedia.org/resource/Victor_Goldschmidt, http://dbpedia.org/resource/Wilder_Penfield, http://dbpedia.org/resource/William_Derham, http://dbpedia.org/resource/William_Hudson_(engineer), http://dbpedia.org/resource/Wolfgang_Pauli, http://dbpedia.org/resource/Alfred_Romer, http://dbpedia.org/resource/David_Bohm, http://dbpedia.org/resource/Edward_B._Lewis, http://dbpedia.org/resource/Grafton_Elliot_Smith, http://dbpedia.org/resource/H._Christopher_Longuet-Higgins, http://dbpedia.org/resource/Jaroslav_Heyrovský, http://dbpedia.org/resource/Jocelyn_Bell_Burnell, http://dbpedia.org/resource/John_O'Keefe_(neuroscientist), http://dbpedia.org/resource/John_Polkinghorne, http://dbpedia.org/resource/Klaus_von_Klitzing, http://dbpedia.org/resource/Louis_Essen, http://dbpedia.org/resource/Peter_Sneath, http://dbpedia.org/resource/Philip_Warren_Anderson, http://dbpedia.org/resource/Roger_Y._Tsien, http://dbpedia.org/resource/Stanley_B._Prusiner, http://dbpedia.org/resource/Alec_Jeffreys, http://dbpedia.org/resource/Alexander_Aitken, http://dbpedia.org/resource/Carl_Ferdinand_Cori, http://dbpedia.org/resource/Charles_Galton_Darwin, http://dbpedia.org/resource/Cyril_Norman_Hinshelwood, http://dbpedia.org/resource/David_Phillips_(chemist), http://dbpedia.org/resource/Gerhard_Herzberg, http://dbpedia.org/resource/Har_Gobind_Khorana, http://dbpedia.org/resource/Harold_E._Varmus, http://dbpedia.org/resource/Harry_Ricardo, http://dbpedia.org/resource/John_Cornforth, http://dbpedia.org/resource/John_Meurig_Thomas, http://dbpedia.org/resource/Keith_Martin_Ball, http://dbpedia.org/resource/L._E._J._Brouwer, http://dbpedia.org/resource/Luis_Federico_Leloir, http://dbpedia.org/resource/Manne_Siegbahn, http://dbpedia.org/resource/Martyn_Poliakoff, http://dbpedia.org/resource/Michael_Burrows, http://dbpedia.org/resource/Otto_Loewi, http://dbpedia.org/resource/Pieter_Zeeman, http://dbpedia.org/resource/Raymond_E._Goldstein, http://dbpedia.org/resource/Robert_Whytlaw-Gray, http://dbpedia.org/resource/Bertram_Brockhouse, http://dbpedia.org/resource/Brigitte_Askonas, http://dbpedia.org/resource/Colin_J._N._Wilson, http://dbpedia.org/resource/Gero_Miesenböck, http://dbpedia.org/resource/Guy_Dodson, http://dbpedia.org/resource/Hans_Kornberg, http://dbpedia.org/resource/Hartmut_Michel, http://dbpedia.org/resource/Irwin_McLean, http://dbpedia.org/resource/John_Alexander_MacWilliam, http://dbpedia.org/resource/Ross_Granville_Harrison, http://dbpedia.org/resource/Rudolph_A._Marcus, http://dbpedia.org/resource/Rudolph_Peters, http://dbpedia.org/resource/Walter_Bodmer, http://dbpedia.org/resource/William_G._Schneider, http://dbpedia.org/resource/Yakov_Sinai, http://dbpedia.org/resource/Chris_J._Leaver, http://dbpedia.org/resource/Peter_Lachmann, http://dbpedia.org/resource/William_Wright_Smith, http://dbpedia.org/resource/Arthur_Rowledge, http://dbpedia.org/resource/Michael_S._Longuet-Higgins, http://dbpedia.org/resource/Bryan_Clarke, http://dbpedia.org/resource/John_M._Edmond, http://dbpedia.org/resource/Thomas_Graham_Brown, http://dbpedia.org/resource/Colin_Nichols, http://dbpedia.org/resource/H.J._Gough, http://dbpedia.org/resource/John_Hilton_Grace, http://dbpedia.org/resource/Ann_Dowling, http://dbpedia.org/resource/Avelino_Corma_Canos, http://dbpedia.org/resource/Charles_Dodds, http://dbpedia.org/resource/Peter_Lawrence_(biologist), http://dbpedia.org/resource/Bernard_Mills, http://dbpedia.org/resource/Jack_Mackenzie, http://dbpedia.org/resource/John_Stuart_Foster, http://dbpedia.org/resource/Francis_Charles_Fraser, http://dbpedia.org/resource/Peter_Buneman, http://dbpedia.org/resource/Kim_Nasmyth, http://dbpedia.org/resource/Yoshio_Masui, http://dbpedia.org/resource/Julian_Besag, http://dbpedia.org/resource/Louis_J._Mordell, http://dbpedia.org/resource/Mark_Pagel, http://dbpedia.org/resource/Olaus_Henrici, http://dbpedia.org/resource/Terry_Lyons_(mathematician), http://dbpedia.org/resource/Harold_Munro_Fox, http://dbpedia.org/resource/John_Savill, http://dbpedia.org/resource/Allan_Bradley, http://dbpedia.org/resource/Keith_Moffatt, http://dbpedia.org/resource/William_H._Miller_(chemistry), http://dbpedia.org/resource/Frank_Pasquill, http://dbpedia.org/resource/John_William_Nicholson, http://dbpedia.org/resource/Janet_Rossant, http://dbpedia.org/resource/Frank_Philip_Bowden, http://dbpedia.org/resource/Frances_Ashcroft, http://dbpedia.org/resource/Neil_Kensington_Adam, http://dbpedia.org/resource/Rodney_Hill, http://dbpedia.org/resource/Cecil_Hoare, http://dbpedia.org/resource/Rosa_Beddington, http://dbpedia.org/resource/Patrick_Linstead, http://dbpedia.org/resource/Alan_Martin_(physicist), http://dbpedia.org/resource/Bill_Johnson_(scientist), http://dbpedia.org/resource/G._H._Cunningham, http://dbpedia.org/resource/Andrew_Ian_Cooper, http://dbpedia.org/resource/Percival_Hartley, http://dbpedia.org/resource/Dalbir_Bindra, http://dbpedia.org/resource/James_Barber_(biochemist), http://dbpedia.org/resource/Mervyn_Bibb, http://dbpedia.org/resource/N._H._Ashton, http://dbpedia.org/resource/Ahmed_Zewail, http://dbpedia.org/resource/Charles_K._Kao, http://dbpedia.org/resource/Charles_Scott_Sherrington, http://dbpedia.org/resource/Charles_Spearman, http://dbpedia.org/resource/Christian_de_Duve, http://dbpedia.org/resource/David_H._Hubel, http://dbpedia.org/resource/Derek_Briggs, http://dbpedia.org/resource/Harry_Kroto, http://dbpedia.org/resource/Hugh_Stott_Taylor, http://dbpedia.org/resource/John_Bardeen, http://dbpedia.org/resource/John_Horton_Conway, http://dbpedia.org/resource/Josiah_Willard_Gibbs, http://dbpedia.org/resource/Karl_Landsteiner, http://dbpedia.org/resource/Lewis_Fry_Richardson, http://dbpedia.org/resource/Ludwig_Boltzmann, http://dbpedia.org/resource/Martin_Lowry, http://dbpedia.org/resource/Robert_Huber, http://dbpedia.org/resource/Robert_Stephenson, http://dbpedia.org/resource/Timothy_Gowers, http://dbpedia.org/resource/Torsten_Wiesel, http://dbpedia.org/resource/Wendy_Hall, http://dbpedia.org/resource/William_Astbury, http://dbpedia.org/resource/William_Lawrence_Bragg, http://dbpedia.org/resource/Detlev_Bronk, http://dbpedia.org/resource/Elizabeth_Blackburn, http://dbpedia.org/resource/Frederic_Calland_Williams, http://dbpedia.org/resource/Gilbert_Smithson_Adair, http://dbpedia.org/resource/Robert_Watson_(scientist), http://dbpedia.org/resource/Tom_Kibble, http://dbpedia.org/resource/Tom_Kilburn, http://dbpedia.org/resource/Vitaly_Ginzburg, http://dbpedia.org/resource/Ajay_K._Sood, http://dbpedia.org/resource/Alan_Arnold_Griffith, http://dbpedia.org/resource/Alan_Soper, http://dbpedia.org/resource/Alexei_Alexeyevich_Abrikosov, http://dbpedia.org/resource/Anne_Cutler, http://dbpedia.org/resource/Anne_Mills, http://dbpedia.org/resource/Anthony_Michell, http://dbpedia.org/resource/Anthony_Watts_(geologist), http://dbpedia.org/resource/Basil_John_Mason, http://dbpedia.org/resource/Brian_Josephson, http://dbpedia.org/resource/Clifford_Tabin, http://dbpedia.org/resource/Daniel_J._Drucker, http://dbpedia.org/resource/David_Beerling, http://dbpedia.org/resource/Edward_Charles_Titchmarsh, http://dbpedia.org/resource/Elias_James_Corey, http://dbpedia.org/resource/Frank_Sherwood_Rowland, http://dbpedia.org/resource/Geoffrey_Wilkinson, http://dbpedia.org/resource/Ghillean_Prance, http://dbpedia.org/resource/Gideon_Henderson, http://dbpedia.org/resource/Harry_B._Whittington, http://dbpedia.org/resource/Herbert_Spencer_Gasser, http://dbpedia.org/resource/James_D._Murray, http://dbpedia.org/resource/Jim_Naismith, http://dbpedia.org/resource/Joan_A._Steitz, http://dbpedia.org/resource/John_Boyd_Orr,_1st_Baron_Boyd-Orr, http://dbpedia.org/resource/John_E._Walker, http://dbpedia.org/resource/John_Enderby, http://dbpedia.org/resource/John_Polanyi, http://dbpedia.org/resource/John_Postgate_(microbiologist), http://dbpedia.org/resource/Julia_Higgins, http://dbpedia.org/resource/Karl_J._Friston, http://dbpedia.org/resource/Laurence_Hurst, http://dbpedia.org/resource/Martin_Evans, http://dbpedia.org/resource/Michael_Richard_Lynch, http://dbpedia.org/resource/Michael_Stratton, http://dbpedia.org/resource/Miles_J._Padgett, http://dbpedia.org/resource/Miles_Reid, http://dbpedia.org/resource/Morris_Travers, http://dbpedia.org/resource/Nicholas_C._Handy, http://dbpedia.org/resource/Norman_L._Bowen, http://dbpedia.org/resource/Olaf_Holtedahl, http://dbpedia.org/resource/Paul_Attfield, http://dbpedia.org/resource/Peter_Hirsch, http://dbpedia.org/resource/Peter_McCullagh, http://dbpedia.org/resource/Prasanta_Chandra_Mahalanobis, http://dbpedia.org/resource/Randy_Read, http://dbpedia.org/resource/Richard_R._Schrock, http://dbpedia.org/resource/Roger_Powell_(scientist), http://dbpedia.org/resource/Rolf_M._Zinkernagel, http://dbpedia.org/resource/Saul_Adler, http://dbpedia.org/resource/Terry_Wyatt, http://dbpedia.org/resource/Tomas_Lindahl, http://dbpedia.org/resource/Tony_Kouzarides, http://dbpedia.org/resource/Cliff_Addison, http://dbpedia.org/resource/Elmer_McCollum, http://dbpedia.org/resource/Henry_Tizard, http://dbpedia.org/resource/Philip_Hall, http://dbpedia.org/resource/Samuel_Baker, http://dbpedia.org/resource/Stephen_Kuffler, http://dbpedia.org/resource/Carl_Størmer, http://dbpedia.org/resource/Henry_Head, http://dbpedia.org/resource/Lawrence_Wager, http://dbpedia.org/resource/Albert_Ingham, http://dbpedia.org/resource/Antoine_Lacroix, http://dbpedia.org/resource/Carlyle_Smith_Beals, http://dbpedia.org/resource/E._J._Bowen, http://dbpedia.org/resource/Ernst_Cohen, http://dbpedia.org/resource/Henry_Horatio_Dixon, http://dbpedia.org/resource/Herbert_McLean_Evans, http://dbpedia.org/resource/James_Wimshurst, http://dbpedia.org/resource/Joseph_Lade_Pawsey, http://dbpedia.org/resource/Neil_Hamilton_Fairley, http://dbpedia.org/resource/Richard_Tecwyn_Williams, http://dbpedia.org/resource/Roy_Cameron, http://dbpedia.org/resource/Solly_Zuckerman,_Baron_Zuckerman, http://dbpedia.org/resource/Alwyn_Jones_(biophysicist), http://dbpedia.org/resource/H._F._Baker, http://dbpedia.org/resource/Meredith_Gwynne_Evans, http://dbpedia.org/resource/Hans_Kronberger_(physicist), http://dbpedia.org/resource/Harold_Ridley_(ophthalmologist), http://dbpedia.org/resource/Martin_Hinton, http://dbpedia.org/resource/George_Frederick_Charles_Searle, http://dbpedia.org/resource/Albert_Heim, http://dbpedia.org/resource/John_Kuriyan, http://dbpedia.org/resource/Lewis_Leigh_Fermor, http://dbpedia.org/resource/Sisir_Kumar_Mitra, http://dbpedia.org/resource/Hedley_Marston, http://dbpedia.org/resource/John_Lightfoot_(biologist), http://dbpedia.org/resource/Avrion_Mitchison, http://dbpedia.org/resource/Ian_Axford, http://dbpedia.org/resource/Rupert_E._Billingham, http://dbpedia.org/resource/Alan_R._Katritzky, http://dbpedia.org/resource/Brian_Hemmings, http://dbpedia.org/resource/Christopher_John_Lamb, http://dbpedia.org/resource/John_Alwyne_Kitching, http://dbpedia.org/resource/Jonathan_D._G._Jones, http://dbpedia.org/resource/Kariamanickam_Srinivasa_Krishnan, http://dbpedia.org/resource/Nick_Barton, http://dbpedia.org/resource/Peter_Liss, http://dbpedia.org/resource/Thomas_Bennet-Clark, http://dbpedia.org/resource/Albrecht_Fröhlich, http://dbpedia.org/resource/Ernest_Kennaway, http://dbpedia.org/resource/Ian_Graham_Gass, http://dbpedia.org/resource/Vladimir_Markovic, http://dbpedia.org/resource/Bonnie_Bassler, http://dbpedia.org/resource/Judy_Armitage, http://dbpedia.org/resource/Thomas_Spencer_Cobbold, http://dbpedia.org/resource/John_Robertson_(physicist), http://dbpedia.org/resource/Jeremy_Sanders, http://dbpedia.org/resource/Tim_Birkhead, http://dbpedia.org/resource/James_Sykes_Gamble, http://dbpedia.org/resource/Frederick_Tom_Brooks, http://dbpedia.org/resource/Gerard_Evan, http://dbpedia.org/resource/Ray_Freeman, http://dbpedia.org/resource/Iain_Campbell_(biophysicist), http://dbpedia.org/resource/Edwin_Dunkin, http://dbpedia.org/resource/Fritz_Ursell, http://dbpedia.org/resource/Gail_R._Martin, http://dbpedia.org/resource/William_Hutcheon_Hall, http://dbpedia.org/resource/Hugh_Ford_(engineer), http://dbpedia.org/resource/Mark_Kisin, http://dbpedia.org/resource/Andrew_Bruce_Holmes, http://dbpedia.org/resource/Arthur_Birch_(organic_chemist), http://dbpedia.org/resource/Joseph_Sambrook, http://dbpedia.org/resource/John_Walter_Guerrier_Lund, http://dbpedia.org/resource/Gillian_Griffiths, http://dbpedia.org/resource/Patricia_H._Clarke, http://dbpedia.org/resource/Bridget_Ogilvie, http://dbpedia.org/resource/Walter_Eric_Spear, http://dbpedia.org/resource/Frank_Kelly_(mathematician), http://dbpedia.org/resource/John_Christopher_Willis, http://dbpedia.org/resource/Ezer_Griffiths, http://dbpedia.org/resource/Malcolm_Clarke_(zoologist), http://dbpedia.org/resource/Bryan_M._Turner, http://dbpedia.org/resource/Adolphe_Quetelet, http://dbpedia.org/resource/Arthur_Evans, http://dbpedia.org/resource/Arthur_Tansley, http://dbpedia.org/resource/Brian_Pippard, http://dbpedia.org/resource/Edmund_Beecher_Wilson, http://dbpedia.org/resource/Edward_Mellanby, http://dbpedia.org/resource/Frederick_Twort, http://dbpedia.org/resource/G._H._Hardy, http://dbpedia.org/resource/G._N._Watson, http://dbpedia.org/resource/Hideki_Yukawa, http://dbpedia.org/resource/Jacques_Monod, http://dbpedia.org/resource/Karl_von_Frisch, http://dbpedia.org/resource/Konrad_Lorenz, http://dbpedia.org/resource/Lloyd_N._Trefethen, http://dbpedia.org/resource/Murray_Gell-Mann, http://dbpedia.org/resource/Paul_Fildes, http://dbpedia.org/resource/Pierre-Gilles_de_Gennes, http://dbpedia.org/resource/Theodosius_Dobzhansky, http://dbpedia.org/resource/William_Wilson_(English_academic), http://dbpedia.org/resource/Émile_Picard, http://dbpedia.org/resource/Abram_Samoilovitch_Besicovitch, http://dbpedia.org/resource/Daniel_Wolpert, http://dbpedia.org/resource/Ewan_Birney, http://dbpedia.org/resource/Jack_D._Dunitz, http://dbpedia.org/resource/James_Augustus_Grant, http://dbpedia.org/resource/James_Black_(pharmacologist), http://dbpedia.org/resource/John_Randall_(physicist), http://dbpedia.org/resource/Peter_Medawar, http://dbpedia.org/resource/Phillip_V._Tobias, http://dbpedia.org/resource/Seymour_Benzer, http://dbpedia.org/resource/Thomas_Little_Heath, http://dbpedia.org/resource/Waldemar_Christofer_Brøgger_(geologist), http://dbpedia.org/resource/William_Molyneux, http://dbpedia.org/resource/Anthony_James_Leggett, http://dbpedia.org/resource/Carlo_Rubbia, http://dbpedia.org/resource/David_Wallace_(physicist), http://dbpedia.org/resource/John_Beddington, http://dbpedia.org/resource/Mark_Walport, http://dbpedia.org/resource/Max_Delbrück, http://dbpedia.org/resource/Otto_Robert_Frisch, http://dbpedia.org/resource/Russell_Brain,_1st_Baron_Brain, http://dbpedia.org/resource/Arthur_Stoll, http://dbpedia.org/resource/John_Westcott, http://dbpedia.org/resource/Stephen_Joseph_Perry, http://dbpedia.org/resource/Élie_Cartan, http://dbpedia.org/resource/Gerard_F._Gilmore, http://dbpedia.org/resource/Karalyn_Patterson, http://dbpedia.org/resource/Niels_Kaj_Jerne, http://dbpedia.org/resource/Stephen_R._Bloom, http://dbpedia.org/resource/Heinz_London, http://dbpedia.org/resource/Thomas_Alan_Stephenson, http://dbpedia.org/resource/John_Alexander_Sinton, http://dbpedia.org/resource/Melvyn_A._Goodale, http://dbpedia.org/resource/Randy_Schekman, http://dbpedia.org/resource/Richard_Friedrich_Johannes_Pfeiffer, http://dbpedia.org/resource/Uta_Frith, http://dbpedia.org/resource/David_Lack, http://dbpedia.org/resource/Frederick_G._Donnan, http://dbpedia.org/resource/Hayne_Constant, http://dbpedia.org/resource/Alfred_Newton_Richards, http://dbpedia.org/resource/Frederic_Wood_Jones, http://dbpedia.org/resource/Margaret_Robinson, http://dbpedia.org/resource/Edward_A._Irving, http://dbpedia.org/resource/Henri-Alexandre_Deslandres, http://dbpedia.org/resource/John_Arthur_Phillips, http://dbpedia.org/resource/Thomas_Summers_West, http://dbpedia.org/resource/Richard_John_Harrison, http://dbpedia.org/resource/Charles_Tomlinson_(scientist), http://dbpedia.org/resource/Richard_Southwood, http://dbpedia.org/resource/Edith_Heard, http://dbpedia.org/resource/John_William_Heslop-Harrison, http://dbpedia.org/resource/Oscar_Werner_Tiegs, http://dbpedia.org/resource/J._Norman_Collie, http://dbpedia.org/resource/Herbert_Fröhlich, http://dbpedia.org/resource/John_Hammond_(physiologist), http://dbpedia.org/resource/Daniel_Frank_Walls, http://dbpedia.org/resource/Michele_Parrinello, http://dbpedia.org/resource/C._S._Seshadri, http://dbpedia.org/resource/Faheem_Hussain, http://dbpedia.org/resource/Harold_Roper_Robinson, http://dbpedia.org/resource/Paul_Taunton_Matthews, http://dbpedia.org/resource/Grenville_Turner, http://dbpedia.org/resource/Irvine_Masson, http://dbpedia.org/resource/Willis_Jackson,_Baron_Jackson_of_Burnley, http://dbpedia.org/resource/Chris_Frith, http://dbpedia.org/resource/Charles_Kellaway, http://dbpedia.org/resource/Allan_Balmain, http://dbpedia.org/resource/Colin_P._Smith, http://dbpedia.org/resource/David_Charlton, http://dbpedia.org/resource/John_Evershed, http://dbpedia.org/resource/Robert_Cyril_Layton_Perkins, http://dbpedia.org/resource/Reginald_W._James, http://dbpedia.org/resource/Sidney_William_Wooldridge, http://dbpedia.org/resource/Michael_Neuberger, http://dbpedia.org/resource/Gordon_Morgan_Holmes, http://dbpedia.org/resource/Ruth_Lynden-Bell, http://dbpedia.org/resource/Leonard_Rogers, http://dbpedia.org/resource/Leon_Simon, http://dbpedia.org/resource/Tom_Bridgeland, http://dbpedia.org/resource/David_Klenerman, http://dbpedia.org/resource/David_Cushing, http://dbpedia.org/resource/Donald_Devereux_Woods, http://dbpedia.org/resource/Daan_Frenkel, http://dbpedia.org/resource/Gordon_Dougan, http://dbpedia.org/resource/William_Wharton_(Royal_Navy_officer), http://dbpedia.org/resource/Jon_Blundy, http://dbpedia.org/resource/Alexander_Rankine, http://dbpedia.org/resource/David_Barford, http://dbpedia.org/resource/William_Joscelyn_Arkell, http://dbpedia.org/resource/Frank_Uhlmann, http://dbpedia.org/resource/Ali_Alavi, http://dbpedia.org/resource/Charles_Ellington, http://dbpedia.org/resource/Oliver_Bulman, http://dbpedia.org/resource/John_C._H._Spence, http://dbpedia.org/resource/Patrik_Rorsman, http://dbpedia.org/resource/Alan_Frederick_Williams_(immunologist), http://dbpedia.org/resource/Francis_Brambell, http://dbpedia.org/resource/Gareth_A._Morris, http://dbpedia.org/resource/Rudi_Lemberg, http://dbpedia.org/resource/Henry_Lewis_Guy, http://dbpedia.org/resource/Kenneth_Bray, http://dbpedia.org/resource/Scott_W._Sloan, http://dbpedia.org/resource/Brigitta_Stockinger, http://dbpedia.org/resource/David_Ron, http://dbpedia.org/resource/Ian_Calman_Muir_MacLennan, http://dbpedia.org/resource/John_Mollon, http://dbpedia.org/resource/Michael_Fasham, http://dbpedia.org/resource/Peter_Guy_Wolynes, http://dbpedia.org/resource/Peter_Keightley, http://dbpedia.org/resource/William_Bullerwell, http://dbpedia.org/resource/Edwin_H._Land, http://dbpedia.org/resource/Edwin_Southern, http://dbpedia.org/resource/J._H._C._Whitehead, http://dbpedia.org/resource/Keith_Gull, http://dbpedia.org/resource/Malcolm_Longair, http://dbpedia.org/resource/Peter_Mansfield, http://dbpedia.org/resource/Stephen_D._M._Brown, http://dbpedia.org/resource/Stephen_Timoshenko, http://dbpedia.org/resource/Arthur_Smith_Woodward, http://dbpedia.org/resource/Davidson_Black, http://dbpedia.org/resource/Kaj_Ulrik_Linderstrøm-Lang, http://dbpedia.org/resource/Susan_Lindquist, http://dbpedia.org/resource/Valentine_Telegdi, http://dbpedia.org/resource/William_Henry_Lang, http://dbpedia.org/resource/Edward_A._Guggenheim, http://dbpedia.org/resource/Thomas_Eckersley, http://dbpedia.org/resource/Alan_Cottrell, http://dbpedia.org/resource/David_Crighton, http://dbpedia.org/resource/Charles_Inglis_(engineer), http://dbpedia.org/resource/Janet_Vaughan, http://dbpedia.org/resource/Albert_Neuberger, http://dbpedia.org/resource/Frederick_Keeble, http://dbpedia.org/resource/George_Brownlee, http://dbpedia.org/resource/Malcolm_Dixon, http://dbpedia.org/resource/Boris_Uvarov, http://dbpedia.org/resource/Emmanuel_Fauré-Fremiet, http://dbpedia.org/resource/Gilbert_Roberts, http://dbpedia.org/resource/Patrick_Steptoe, http://dbpedia.org/resource/John_Monteith, http://dbpedia.org/resource/Barry_Everitt_(scientist), http://dbpedia.org/resource/Frank_Horton_(physicist), http://dbpedia.org/resource/Kenneth_Murray_(biologist), http://dbpedia.org/resource/Martin_Bott, http://dbpedia.org/resource/Noreen_Murray, http://dbpedia.org/resource/Cheryll_Tickle, http://dbpedia.org/resource/Louis_Harold_Gray, http://dbpedia.org/resource/Alexander_Fleck,_1st_Baron_Fleck, http://dbpedia.org/resource/E._J._Conway, http://dbpedia.org/resource/Gary_Gibbons, http://dbpedia.org/resource/George_Barker_Jeffery, http://dbpedia.org/resource/Ofer_Lahav, http://dbpedia.org/resource/Frederick_Orpen_Bower, http://dbpedia.org/resource/James_Gray_(zoologist), http://dbpedia.org/resource/John_Baker,_Baron_Baker, http://dbpedia.org/resource/Joseph_Hutchinson, http://dbpedia.org/resource/Robert_Mair,_Baron_Mair, http://dbpedia.org/resource/Demetri_Terzopoulos, http://dbpedia.org/resource/Lord_Amelius_Beauclerk, http://dbpedia.org/resource/Mark_Pepys, http://dbpedia.org/resource/Ray_Beverton, http://dbpedia.org/resource/Albert_Frey-Wyssling, http://dbpedia.org/resource/Dale_Sanders, http://dbpedia.org/resource/Liam_Dolan, http://dbpedia.org/resource/Nicholas_Harberd, http://dbpedia.org/resource/Alexander_Watt, http://dbpedia.org/resource/Harry_Harris_(geneticist), http://dbpedia.org/resource/Peter_Marler, http://dbpedia.org/resource/Timothy_Holland, http://dbpedia.org/resource/Henry_Harold_Welch_Pearson, http://dbpedia.org/resource/Edgar_William_Richard_Steacie, http://dbpedia.org/resource/Dalziel_Hammick, http://dbpedia.org/resource/Leonard_Colebrook, http://dbpedia.org/resource/Jocelyn_Field_Thorpe, http://dbpedia.org/resource/Panchanan_Maheshwari]... [INFO] Predicted Answer: [http://dbpedia.org/resource/Howell_Peregrine] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 711, P: 0.507, R: 0.558, macro F1: 0.513, macro F1*: 0.531 -[INFO] Current time: 2315 ms, average: 2809.79 ms/ques, entity detection: 449.36 ms/ques (linking tool: 356.41 ms/ques), relation detection: 916.15 ms/ques (similarity: 706.16 ms/ques), EDG generation: 47.33 ms/ques +[INFO] Cumulative metrics, sample: 711, P: 0.507, R: 0.558, macro F1: 0.531 +[INFO] Current question consumed time: 2047 ms, average: 2535.03 ms/ques, entity detection: 397.68 ms/ques (linking tool: 319.04 ms/ques), relation detection: 846.82 ms/ques (similarity: 696.34 ms/ques), EDG generation: 47.34 ms/ques [INFO] Entity Linking P: 0.7475386779184248, R: 0.6983122362869199, micro F1: 0.7140178152836392, macro F1: 0.7220874583394901 [INFO] Relation Linking P: 0.6004453820909517, R: 0.5464135021097046, micro F1: 0.5616971401781535, macro F1: 0.5721566420660229 [INFO] Type Linking P: 0.5443037974683544, R: 0.5443037974683544, micro F1: 0.5443037974683544, macro F1: 0.5443037974683544 @@ -9985,8 +9985,8 @@ Question 711: Which president of United States had vice president as John Adams? [INFO] Golden Answer: [http://dbpedia.org/resource/George_Washington] [INFO] Predicted Answer: [http://dbpedia.org/resource/George_Washington, http://dbpedia.org/resource/Presidency_of_George_Washington, http://dbpedia.org/resource/Thomas_Jefferson] [INFO] P: 0.3333333333333333, R: 1.0, F: 0.5 -[INFO] Cumulative metrics, sample: 712, P: 0.507, R: 0.558, macro F1: 0.513, macro F1*: 0.531 -[INFO] Current time: 62417 ms, average: 2893.51 ms/ques, entity detection: 478.96 ms/ques (linking tool: 385.80 ms/ques), relation detection: 937.91 ms/ques (similarity: 710.40 ms/ques), EDG generation: 47.29 ms/ques +[INFO] Cumulative metrics, sample: 712, P: 0.507, R: 0.558, macro F1: 0.531 +[INFO] Current question consumed time: 55083 ms, average: 2608.84 ms/ques, entity detection: 423.77 ms/ques (linking tool: 344.94 ms/ques), relation detection: 866.14 ms/ques (similarity: 700.62 ms/ques), EDG generation: 47.30 ms/ques [INFO] Entity Linking P: 0.7478932584269663, R: 0.6987359550561798, micro F1: 0.7144194756554317, macro F1: 0.7224794098396423 [INFO] Relation Linking P: 0.601006554307116, R: 0.5470505617977528, micro F1: 0.5623127340823977, macro F1: 0.5727606554860761 [INFO] Type Linking P: 0.5435393258426966, R: 0.5435393258426966, micro F1: 0.5435393258426966, macro F1: 0.5435393258426966 @@ -9999,8 +9999,8 @@ Question 712: Where was James H. Fields buried ? [INFO] Golden Answer: [http://dbpedia.org/resource/Houston, http://dbpedia.org/resource/Houston_National_Cemetery] [INFO] Predicted Answer: [http://dbpedia.org/resource/Houston, http://dbpedia.org/resource/Houston_National_Cemetery] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 713, P: 0.508, R: 0.559, macro F1: 0.514, macro F1*: 0.532 -[INFO] Current time: 802 ms, average: 2890.57 ms/ques, entity detection: 478.49 ms/ques (linking tool: 385.37 ms/ques), relation detection: 936.92 ms/ques (similarity: 709.59 ms/ques), EDG generation: 47.23 ms/ques +[INFO] Cumulative metrics, sample: 713, P: 0.508, R: 0.559, macro F1: 0.532 +[INFO] Current question consumed time: 681 ms, average: 2606.13 ms/ques, entity detection: 423.34 ms/ques (linking tool: 344.55 ms/ques), relation detection: 865.22 ms/ques (similarity: 699.83 ms/ques), EDG generation: 47.24 ms/ques [INFO] Entity Linking P: 0.7482468443197756, R: 0.6991584852734923, micro F1: 0.7148200093501647, macro F1: 0.72287025560738 [INFO] Relation Linking P: 0.6001636278634875, R: 0.5462833099579243, micro F1: 0.5615240766713424, macro F1: 0.5719573446088166 [INFO] Type Linking P: 0.544179523141655, R: 0.544179523141655, micro F1: 0.544179523141655, macro F1: 0.544179523141655 @@ -10013,8 +10013,8 @@ Question 713: Is Henry Knox a commander of Continental Artillery Regiment? [INFO] Golden Answer: [true] [INFO] Predicted Answer: [true] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 714, P: 0.509, R: 0.560, macro F1: 0.515, macro F1*: 0.533 -[INFO] Current time: 1110 ms, average: 2888.08 ms/ques, entity detection: 478.30 ms/ques (linking tool: 385.12 ms/ques), relation detection: 935.89 ms/ques (similarity: 708.74 ms/ques), EDG generation: 47.18 ms/ques +[INFO] Cumulative metrics, sample: 714, P: 0.509, R: 0.560, macro F1: 0.533 +[INFO] Current question consumed time: 908 ms, average: 2603.75 ms/ques, entity detection: 423.13 ms/ques (linking tool: 344.30 ms/ques), relation detection: 864.27 ms/ques (similarity: 699.00 ms/ques), EDG generation: 47.18 ms/ques [INFO] Entity Linking P: 0.7485994397759104, R: 0.6995798319327731, micro F1: 0.715219421101775, macro F1: 0.7232600003250816 [INFO] Relation Linking P: 0.6007236227824463, R: 0.5469187675070029, micro F1: 0.5621381886087775, macro F1: 0.5725599301044532 [INFO] Type Linking P: 0.5448179271708683, R: 0.5448179271708683, micro F1: 0.5448179271708683, macro F1: 0.5448179271708683 @@ -10027,8 +10027,8 @@ Question 714: What is the commander of Battle of Brownstown ? [INFO] Golden Answer: [http://dbpedia.org/resource/Major, http://dbpedia.org/resource/Tecumseh] [INFO] Predicted Answer: [http://dbpedia.org/resource/Major, http://dbpedia.org/resource/Tecumseh, Major Thomas Van Horne@en] [INFO] P: 0.6666666666666666, R: 1.0, F: 0.8 -[INFO] Cumulative metrics, sample: 715, P: 0.509, R: 0.560, macro F1: 0.515, macro F1*: 0.533 -[INFO] Current time: 545 ms, average: 2884.80 ms/ques, entity detection: 477.79 ms/ques (linking tool: 384.70 ms/ques), relation detection: 934.76 ms/ques (similarity: 707.85 ms/ques), EDG generation: 47.13 ms/ques +[INFO] Cumulative metrics, sample: 715, P: 0.509, R: 0.560, macro F1: 0.533 +[INFO] Current question consumed time: 444 ms, average: 2600.74 ms/ques, entity detection: 422.66 ms/ques (linking tool: 343.91 ms/ques), relation detection: 863.22 ms/ques (similarity: 698.11 ms/ques), EDG generation: 47.13 ms/ques [INFO] Entity Linking P: 0.7489510489510489, R: 0.7, micro F1: 0.7156177156177167, macro F1: 0.7236486486486486 [INFO] Relation Linking P: 0.6012820512820513, R: 0.5475524475524476, micro F1: 0.5627505827505834, macro F1: 0.5731608150396829 [INFO] Type Linking P: 0.5454545454545454, R: 0.5454545454545454, micro F1: 0.5454545454545454, macro F1: 0.5454545454545454 @@ -10041,8 +10041,8 @@ Question 715: Is Baker's Dozen Donuts in the coffee industry? [INFO] Golden Answer: [true] [INFO] Predicted Answer: [false] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 716, P: 0.508, R: 0.559, macro F1: 0.514, macro F1*: 0.532 -[INFO] Current time: 1367 ms, average: 2882.68 ms/ques, entity detection: 477.69 ms/ques (linking tool: 384.50 ms/ques), relation detection: 933.62 ms/ques (similarity: 706.97 ms/ques), EDG generation: 47.08 ms/ques +[INFO] Cumulative metrics, sample: 716, P: 0.508, R: 0.559, macro F1: 0.532 +[INFO] Current question consumed time: 1107 ms, average: 2598.65 ms/ques, entity detection: 422.55 ms/ques (linking tool: 343.72 ms/ques), relation detection: 862.17 ms/ques (similarity: 697.25 ms/ques), EDG generation: 47.08 ms/ques [INFO] Entity Linking P: 0.7479050279329609, R: 0.6990223463687151, micro F1: 0.7146182495344517, macro F1: 0.7226379661784689 [INFO] Relation Linking P: 0.6004422718808193, R: 0.5467877094972067, micro F1: 0.5619646182495351, macro F1: 0.5723603111080632 [INFO] Type Linking P: 0.5460893854748603, R: 0.5460893854748603, micro F1: 0.5460893854748603, macro F1: 0.5460893854748603 @@ -10055,8 +10055,8 @@ Question 716: What is Jon Speelman famous for? [INFO] Golden Answer: [http://dbpedia.org/resource/Chess, http://dbpedia.org/resource/Mathematics, http://dbpedia.org/resource/Writing] [INFO] Predicted Answer: [http://dbpedia.org/resource/Worcester_College,_Oxford] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 717, P: 0.507, R: 0.559, macro F1: 0.514, macro F1*: 0.532 -[INFO] Current time: 635 ms, average: 2879.55 ms/ques, entity detection: 477.22 ms/ques (linking tool: 384.08 ms/ques), relation detection: 932.50 ms/ques (similarity: 706.11 ms/ques), EDG generation: 47.02 ms/ques +[INFO] Cumulative metrics, sample: 717, P: 0.507, R: 0.559, macro F1: 0.532 +[INFO] Current question consumed time: 545 ms, average: 2595.79 ms/ques, entity detection: 422.12 ms/ques (linking tool: 343.34 ms/ques), relation detection: 861.13 ms/ques (similarity: 696.39 ms/ques), EDG generation: 47.03 ms/ques [INFO] Entity Linking P: 0.7482566248256625, R: 0.699442119944212, micro F1: 0.7150162715016282, macro F1: 0.7230263918113097 [INFO] Relation Linking P: 0.5996048349604834, R: 0.5460251046025104, micro F1: 0.5611808461180853, macro F1: 0.5715620401023337 [INFO] Type Linking P: 0.5467224546722455, R: 0.5467224546722455, micro F1: 0.5467224546722455, macro F1: 0.5467224546722455 @@ -10069,8 +10069,8 @@ Question 717: Who is the fictional character whose family member is Padme Amidal [INFO] Golden Answer: [http://dbpedia.org/resource/Luke_Skywalker, http://dbpedia.org/resource/Princess_Leia] [INFO] Predicted Answer: [http://dbpedia.org/resource/Luke_Skywalker, http://dbpedia.org/resource/Princess_Leia] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 718, P: 0.508, R: 0.559, macro F1: 0.514, macro F1*: 0.532 -[INFO] Current time: 1275 ms, average: 2877.31 ms/ques, entity detection: 476.72 ms/ques (linking tool: 383.66 ms/ques), relation detection: 931.83 ms/ques (similarity: 705.79 ms/ques), EDG generation: 47.07 ms/ques +[INFO] Cumulative metrics, sample: 718, P: 0.508, R: 0.559, macro F1: 0.532 +[INFO] Current question consumed time: 1127 ms, average: 2593.74 ms/ques, entity detection: 421.68 ms/ques (linking tool: 342.97 ms/ques), relation detection: 860.50 ms/ques (similarity: 696.09 ms/ques), EDG generation: 47.00 ms/ques [INFO] Entity Linking P: 0.7486072423398329, R: 0.6998607242339833, micro F1: 0.7154131847725173, macro F1: 0.7234137293764732 [INFO] Relation Linking P: 0.6001624883936861, R: 0.5466573816155988, micro F1: 0.5617920148560823, macro F1: 0.5721617893602392 [INFO] Type Linking P: 0.5459610027855153, R: 0.5459610027855153, micro F1: 0.5459610027855153, macro F1: 0.5459610027855153 @@ -10083,8 +10083,8 @@ Question 718: What company is the one which wrote the google web toolkit and Pla [INFO] Golden Answer: [http://dbpedia.org/resource/Google] [INFO] Predicted Answer: [http://dbpedia.org/resource/Google] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 719, P: 0.509, R: 0.560, macro F1: 0.515, macro F1*: 0.533 -[INFO] Current time: 1448 ms, average: 2875.33 ms/ques, entity detection: 476.45 ms/ques (linking tool: 383.40 ms/ques), relation detection: 931.01 ms/ques (similarity: 705.24 ms/ques), EDG generation: 47.04 ms/ques +[INFO] Cumulative metrics, sample: 719, P: 0.509, R: 0.560, macro F1: 0.533 +[INFO] Current question consumed time: 2135 ms, average: 2593.10 ms/ques, entity detection: 421.45 ms/ques (linking tool: 342.74 ms/ques), relation detection: 859.72 ms/ques (similarity: 696.75 ms/ques), EDG generation: 46.97 ms/ques [INFO] Entity Linking P: 0.7489568845618915, R: 0.700278164116829, micro F1: 0.7158089939731118, macro F1: 0.723799983448968 [INFO] Relation Linking P: 0.6007185906351413, R: 0.547287899860918, micro F1: 0.5624014835419571, macro F1: 0.5727598556242579 [INFO] Type Linking P: 0.545201668984701, R: 0.545201668984701, micro F1: 0.545201668984701, macro F1: 0.545201668984701 @@ -10097,8 +10097,8 @@ Question 719: How many different songwriters have had their songs recorded in Lo [INFO] Golden Answer: [493] [INFO] Predicted Answer: [1803] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 720, P: 0.508, R: 0.559, macro F1: 0.514, macro F1*: 0.532 -[INFO] Current time: 16155 ms, average: 2893.77 ms/ques, entity detection: 480.41 ms/ques (linking tool: 387.35 ms/ques), relation detection: 937.09 ms/ques (similarity: 710.29 ms/ques), EDG generation: 47.01 ms/ques +[INFO] Cumulative metrics, sample: 720, P: 0.508, R: 0.559, macro F1: 0.532 +[INFO] Current question consumed time: 14231 ms, average: 2609.27 ms/ques, entity detection: 424.97 ms/ques (linking tool: 346.26 ms/ques), relation detection: 864.83 ms/ques (similarity: 701.78 ms/ques), EDG generation: 46.93 ms/ques [INFO] Entity Linking P: 0.7493055555555556, R: 0.7006944444444444, micro F1: 0.7162037037037048, macro F1: 0.7241851585781184 [INFO] Relation Linking P: 0.6012731481481481, R: 0.5472222222222223, micro F1: 0.5625462962962969, macro F1: 0.5729758025686676 [INFO] Type Linking P: 0.5458333333333333, R: 0.5458333333333333, micro F1: 0.5458333333333333, macro F1: 0.5458333333333333 @@ -10111,8 +10111,8 @@ Question 720: Which are the automobile platform whose one of the automobile pla [INFO] Golden Answer: [http://dbpedia.org/resource/Ford_Motor_Company, http://dbpedia.org/resource/Dodge, http://dbpedia.org/resource/Blue_Bird_Corporation] [INFO] Predicted Answer: [http://dbpedia.org/resource/Chiva_bus] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 721, P: 0.507, R: 0.558, macro F1: 0.513, macro F1*: 0.532 -[INFO] Current time: 1750 ms, average: 2892.18 ms/ques, entity detection: 480.12 ms/ques (linking tool: 387.03 ms/ques), relation detection: 936.60 ms/ques (similarity: 710.18 ms/ques), EDG generation: 46.99 ms/ques +[INFO] Cumulative metrics, sample: 721, P: 0.507, R: 0.558, macro F1: 0.532 +[INFO] Current question consumed time: 1576 ms, average: 2607.83 ms/ques, entity detection: 424.69 ms/ques (linking tool: 345.97 ms/ques), relation detection: 864.40 ms/ques (similarity: 701.69 ms/ques), EDG generation: 46.91 ms/ques [INFO] Entity Linking P: 0.7496532593619972, R: 0.7011095700416089, micro F1: 0.7165973185390672, macro F1: 0.7245692592877431 [INFO] Relation Linking P: 0.6018261673601479, R: 0.5478502080443828, micro F1: 0.5631530282015725, macro F1: 0.5735711336658492 [INFO] Type Linking P: 0.5450762829403606, R: 0.5450762829403606, micro F1: 0.5450762829403606, macro F1: 0.5450762829403606 @@ -10125,8 +10125,8 @@ Question 721: What is the total number of other combatant of the military confli [INFO] Golden Answer: [10] [INFO] Predicted Answer: [15, 10] [INFO] P: 0.5, R: 1.0, F: 0.6666666666666666 -[INFO] Cumulative metrics, sample: 722, P: 0.507, R: 0.559, macro F1: 0.514, macro F1*: 0.532 -[INFO] Current time: 1745 ms, average: 2890.60 ms/ques, entity detection: 479.60 ms/ques (linking tool: 386.57 ms/ques), relation detection: 936.27 ms/ques (similarity: 709.99 ms/ques), EDG generation: 47.02 ms/ques +[INFO] Cumulative metrics, sample: 722, P: 0.507, R: 0.559, macro F1: 0.532 +[INFO] Current question consumed time: 1554 ms, average: 2606.37 ms/ques, entity detection: 424.23 ms/ques (linking tool: 345.56 ms/ques), relation detection: 864.10 ms/ques (similarity: 701.51 ms/ques), EDG generation: 46.94 ms/ques [INFO] Entity Linking P: 0.75, R: 0.7015235457063712, micro F1: 0.7169898430286252, macro F1: 0.724952290076336 [INFO] Relation Linking P: 0.6023776546629732, R: 0.5484764542936288, micro F1: 0.5637580794090497, macro F1: 0.5741648009143413 [INFO] Type Linking P: 0.5443213296398892, R: 0.5443213296398892, micro F1: 0.5443213296398892, macro F1: 0.5443213296398892 @@ -10139,8 +10139,8 @@ Question 722: List some pupils of National Science Medal winning scientists? [INFO] Golden Answer: [http://dbpedia.org/resource/Robert_Marshak, http://dbpedia.org/resource/Ajoy_Ghatak, http://dbpedia.org/resource/Peter_A._Carruthers, http://dbpedia.org/resource/Jeffrey_Goldstone, http://dbpedia.org/resource/Roman_Jackiw, http://dbpedia.org/resource/David_J._Thouless, John Irwin@en, Benjamin Day@en, R. Rajaraman@en, John Negele@en, M. K. Sundaresan@en, P. S. Epstein@en, Roderick V. Reid@en, http://dbpedia.org/resource/Constance_Cepko, http://dbpedia.org/resource/Andrew_Fire, *@en, http://dbpedia.org/resource/Trey_Ideker, http://dbpedia.org/resource/Andrei_Broder, http://dbpedia.org/resource/Jeffrey_Vitter, http://dbpedia.org/resource/Leonidas_J._Guibas, http://dbpedia.org/resource/Michael_Fredman, http://dbpedia.org/resource/Scott_Kim, http://dbpedia.org/resource/Vaughan_Pratt, http://dbpedia.org/resource/Robert_Sedgewick_(computer_scientist), http://dbpedia.org/resource/Robert_Trivers, http://dbpedia.org/resource/Arthur_Wahl, http://dbpedia.org/resource/Joseph_W._Kennedy, http://dbpedia.org/resource/Elizabeth_Rauscher, Kimberly Williams@en, Ralph Arthur James@en, http://dbpedia.org/resource/Ruzena_Bajcsy, Barbara Liskov@en, Raj Reddy@en, Ramanathan V. Guha@en, Cordell Green@en, Francis Morris@en, Robert Moore@en, Aarati Parmar Martino@en, Donald Kaplan@en, Eyal Amir@en, Randall Davis@en, http://dbpedia.org/resource/Edgar_Bright_Wilson, http://dbpedia.org/resource/Martin_Karplus, http://dbpedia.org/resource/William_Lipscomb, http://dbpedia.org/resource/Matthew_Meselson, http://dbpedia.org/resource/Jerry_Donohue, http://dbpedia.org/resource/Jeremiah_P._Ostriker, http://dbpedia.org/resource/Donald_Edward_Osterbrock, http://dbpedia.org/resource/Jerome_Kristian, http://dbpedia.org/resource/Roland_Winston, http://dbpedia.org/resource/Yousef_Sobouti, http://dbpedia.org/resource/Linda_Preiss_Rothschild, http://dbpedia.org/resource/Dan_Freed, http://dbpedia.org/resource/John_Lott_(mathematician), http://dbpedia.org/resource/Richard_L._Bishop, http://dbpedia.org/resource/George_Logemann, http://dbpedia.org/resource/Alexandre_Chorin, http://dbpedia.org/resource/Ami_Harten, http://dbpedia.org/resource/Burton_Wendroff, http://dbpedia.org/resource/Charles_Epstein, http://dbpedia.org/resource/James_(Mac)_Hyman, http://dbpedia.org/resource/Jeffrey_Rauch, http://dbpedia.org/resource/Steve_Alpern, http://dbpedia.org/resource/Paul_von_Ragué_Schleyer, Aryeh Frimer@en, James Cullen Martin,@en, http://dbpedia.org/resource/Roy_J._Glauber, http://dbpedia.org/resource/Stanley_Deser, http://dbpedia.org/resource/Sam_Edwards_(physicist), http://dbpedia.org/resource/Bryce_DeWitt, http://dbpedia.org/resource/Daniel_Kleitman, http://dbpedia.org/resource/Sheldon_Lee_Glashow, http://dbpedia.org/resource/Gordon_Baym, http://dbpedia.org/resource/Lawrence_Paul_Horwitz, http://dbpedia.org/resource/Lowell_S._Brown, http://dbpedia.org/resource/Ben_Roy_Mottelson, Walter Kohn@en, Margaret G. Kivelson@en, http://dbpedia.org/resource/Mary_Ainsworth, http://dbpedia.org/resource/Abraham_Maslow, http://dbpedia.org/resource/John_Bowlby, http://dbpedia.org/resource/Edward_Mills_Purcell, http://dbpedia.org/resource/Philip_Warren_Anderson, http://dbpedia.org/resource/Robert_Serber, Thomas Kuhn@en, John Atanasoff@en, Martin Smith, Lane Johnson, Bruce Julian, \"Francis\" Wu, \"Leon\" Teng, Charles G. Sammis, Harmuth Spetzler, Thomas H. Jordan, James Whitcomb, Hsi-Ping Liu, Scott David King, Jeffrey Given, Shingo Watada, Janice Regan, Eric Chael, Hua-Wei Zhou, Lianxing Wen.@en, http://dbpedia.org/resource/Joseph_P._Allen, http://dbpedia.org/resource/Joel_Birnbaum, http://dbpedia.org/resource/Edward_B._Lewis, http://dbpedia.org/resource/Avner_Ash, http://dbpedia.org/resource/Jonathan_Wahl, http://dbpedia.org/resource/Cyril_Ponnamperuma, http://dbpedia.org/resource/Ronald_Breslow, http://dbpedia.org/resource/Christopher_Spencer_Foote, http://dbpedia.org/resource/Kendall_Houk, http://dbpedia.org/resource/William_R._Roush, http://dbpedia.org/resource/Stuart_Schreiber, Harry Wasserman@en, http://dbpedia.org/resource/Shing-Tung_Yau, http://dbpedia.org/resource/Harold_Levine, http://dbpedia.org/resource/Thomas_Banchoff, http://dbpedia.org/resource/Katsumi_Nomizu, http://dbpedia.org/resource/Alan_Weinstein, http://dbpedia.org/resource/Louis_Auslander, http://dbpedia.org/resource/Manfredo_do_Carmo, http://dbpedia.org/resource/Robert_Brown_Gardner, http://dbpedia.org/resource/Herbert_Robbins, http://dbpedia.org/resource/Paul_Olum, http://dbpedia.org/resource/James_Eells, http://dbpedia.org/resource/Wilfred_Kaplan, http://dbpedia.org/resource/Christopher_Burge, http://dbpedia.org/resource/Thomas_M._Liggett, http://dbpedia.org/resource/John_W._Pratt, http://dbpedia.org/resource/Peter_Sarnak, http://dbpedia.org/resource/Wylie_Vale, http://dbpedia.org/resource/Roger_Wolcott_Sperry, http://dbpedia.org/resource/Jianmin_Qu, http://dbpedia.org/resource/Lambert_B._Freund, http://dbpedia.org/resource/Herbert_S._Gutowsky, http://dbpedia.org/resource/Bruce_H._Mahan, http://dbpedia.org/resource/Robert_W._Boyd, http://dbpedia.org/resource/Ali_Javan, http://dbpedia.org/resource/James_P._Gordon, http://dbpedia.org/resource/Raymond_Chiao, http://dbpedia.org/resource/Irwin_Fridovich] [INFO] Predicted Answer: [http://dbpedia.org/resource/Elrick_Irastorza, http://dbpedia.org/resource/Édouard_Guillaud] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 723, P: 0.507, R: 0.558, macro F1: 0.513, macro F1*: 0.531 -[INFO] Current time: 984 ms, average: 2887.96 ms/ques, entity detection: 479.24 ms/ques (linking tool: 386.24 ms/ques), relation detection: 935.19 ms/ques (similarity: 709.15 ms/ques), EDG generation: 46.97 ms/ques +[INFO] Cumulative metrics, sample: 723, P: 0.507, R: 0.558, macro F1: 0.531 +[INFO] Current question consumed time: 872 ms, average: 2603.98 ms/ques, entity detection: 423.89 ms/ques (linking tool: 345.26 ms/ques), relation detection: 863.10 ms/ques (similarity: 700.67 ms/ques), EDG generation: 46.89 ms/ques [INFO] Entity Linking P: 0.7489626556016598, R: 0.7005532503457814, micro F1: 0.7159981558321817, macro F1: 0.7239495898134363 [INFO] Relation Linking P: 0.6015444905486399, R: 0.5477178423236515, micro F1: 0.5629783310281242, macro F1: 0.5733706587277378 [INFO] Type Linking P: 0.5435684647302904, R: 0.5435684647302904, micro F1: 0.5435684647302904, macro F1: 0.5435684647302904 @@ -10153,8 +10153,8 @@ Question 723: Relatives of Moses used to follow which religions? [INFO] Golden Answer: [http://dbpedia.org/resource/Christianity, http://dbpedia.org/resource/Islam, http://dbpedia.org/resource/Judaism] [INFO] Predicted Answer: [http://dbpedia.org/resource/Miriam, http://dbpedia.org/resource/Aaron] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 724, P: 0.506, R: 0.557, macro F1: 0.512, macro F1*: 0.530 -[INFO] Current time: 2319 ms, average: 2887.17 ms/ques, entity detection: 479.05 ms/ques (linking tool: 385.96 ms/ques), relation detection: 934.95 ms/ques (similarity: 709.28 ms/ques), EDG generation: 46.92 ms/ques +[INFO] Cumulative metrics, sample: 724, P: 0.506, R: 0.557, macro F1: 0.530 +[INFO] Current question consumed time: 2113 ms, average: 2603.30 ms/ques, entity detection: 423.72 ms/ques (linking tool: 344.99 ms/ques), relation detection: 862.92 ms/ques (similarity: 700.92 ms/ques), EDG generation: 46.85 ms/ques [INFO] Entity Linking P: 0.7493093922651933, R: 0.7009668508287292, micro F1: 0.7163904235727451, macro F1: 0.7243324125230203 [INFO] Relation Linking P: 0.6020948434622467, R: 0.5476519337016574, micro F1: 0.5631215469613267, macro F1: 0.5735843958741355 [INFO] Type Linking P: 0.5441988950276243, R: 0.5441988950276243, micro F1: 0.5441988950276243, macro F1: 0.5441988950276243 @@ -10167,8 +10167,8 @@ Question 724: Where was Ne daj se, Nina first aired ? [INFO] Golden Answer: [http://dbpedia.org/resource/Croatia, http://dbpedia.org/resource/Serbia, --01-03, --10-29] [INFO] Predicted Answer: [http://dbpedia.org/resource/Croatia, http://dbpedia.org/resource/Serbia, --01-03, --10-29] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 725, P: 0.507, R: 0.558, macro F1: 0.513, macro F1*: 0.531 -[INFO] Current time: 826 ms, average: 2884.33 ms/ques, entity detection: 478.60 ms/ques (linking tool: 385.60 ms/ques), relation detection: 933.82 ms/ques (similarity: 708.40 ms/ques), EDG generation: 46.87 ms/ques +[INFO] Cumulative metrics, sample: 725, P: 0.507, R: 0.558, macro F1: 0.531 +[INFO] Current question consumed time: 726 ms, average: 2600.71 ms/ques, entity detection: 423.32 ms/ques (linking tool: 344.67 ms/ques), relation detection: 861.88 ms/ques (similarity: 700.05 ms/ques), EDG generation: 46.80 ms/ques [INFO] Entity Linking P: 0.7496551724137931, R: 0.7013793103448276, micro F1: 0.7167816091954033, macro F1: 0.7247141733315853 [INFO] Relation Linking P: 0.6026436781609195, R: 0.5482758620689655, micro F1: 0.5637241379310352, macro F1: 0.5741756406326948 [INFO] Type Linking P: 0.5448275862068965, R: 0.5448275862068965, micro F1: 0.5448275862068965, macro F1: 0.5448275862068965 @@ -10181,8 +10181,8 @@ Question 725: Where did the 2015 MLS All-Star Game take place? [INFO] Golden Answer: [http://dbpedia.org/resource/Colorado, http://dbpedia.org/resource/Commerce_City,_Colorado] [INFO] Predicted Answer: [http://dbpedia.org/resource/Dick's_Sporting_Goods_Park, http://dbpedia.org/resource/Colorado, http://dbpedia.org/resource/Commerce_City,_Colorado] [INFO] P: 0.6666666666666666, R: 1.0, F: 0.8 -[INFO] Cumulative metrics, sample: 726, P: 0.507, R: 0.559, macro F1: 0.513, macro F1*: 0.531 -[INFO] Current time: 953 ms, average: 2881.67 ms/ques, entity detection: 478.16 ms/ques (linking tool: 385.20 ms/ques), relation detection: 932.97 ms/ques (similarity: 707.70 ms/ques), EDG generation: 46.82 ms/ques +[INFO] Cumulative metrics, sample: 726, P: 0.507, R: 0.559, macro F1: 0.531 +[INFO] Current question consumed time: 838 ms, average: 2598.28 ms/ques, entity detection: 422.92 ms/ques (linking tool: 344.30 ms/ques), relation detection: 861.10 ms/ques (similarity: 699.35 ms/ques), EDG generation: 46.74 ms/ques [INFO] Entity Linking P: 0.75, R: 0.7017906336088154, micro F1: 0.7171717171717182, macro F1: 0.7250948766603414 [INFO] Relation Linking P: 0.6018135904499541, R: 0.5475206611570248, micro F1: 0.5629476584022045, macro F1: 0.5733847650946332 [INFO] Type Linking P: 0.5454545454545454, R: 0.5454545454545454, micro F1: 0.5454545454545454, macro F1: 0.5454545454545454 @@ -10195,8 +10195,8 @@ Question 726: Is Nikolai Morozov the former coach of Stanislav Morozov? [INFO] Golden Answer: [true] [INFO] Predicted Answer: [true] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 727, P: 0.507, R: 0.559, macro F1: 0.514, macro F1*: 0.532 -[INFO] Current time: 1091 ms, average: 2879.21 ms/ques, entity detection: 478.03 ms/ques (linking tool: 384.97 ms/ques), relation detection: 932.04 ms/ques (similarity: 706.93 ms/ques), EDG generation: 46.78 ms/ques +[INFO] Cumulative metrics, sample: 727, P: 0.507, R: 0.559, macro F1: 0.532 +[INFO] Current question consumed time: 875 ms, average: 2595.91 ms/ques, entity detection: 422.74 ms/ques (linking tool: 344.07 ms/ques), relation detection: 860.23 ms/ques (similarity: 698.59 ms/ques), EDG generation: 46.70 ms/ques [INFO] Entity Linking P: 0.750343878954608, R: 0.702200825309491, micro F1: 0.7175607519486484, macro F1: 0.725474526905923 [INFO] Relation Linking P: 0.6023613021549747, R: 0.548143053645117, micro F1: 0.563548830811555, macro F1: 0.5739746432011735 [INFO] Type Linking P: 0.546079779917469, R: 0.546079779917469, micro F1: 0.546079779917469, macro F1: 0.546079779917469 @@ -10209,8 +10209,8 @@ Question 727: Did Alexandre Tichonov study at the Moscow university? [INFO] Golden Answer: [true] [INFO] Predicted Answer: [false] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 728, P: 0.507, R: 0.558, macro F1: 0.513, macro F1*: 0.531 -[INFO] Current time: 3657 ms, average: 2880.27 ms/ques, entity detection: 478.67 ms/ques (linking tool: 385.49 ms/ques), relation detection: 932.03 ms/ques (similarity: 706.64 ms/ques), EDG generation: 46.75 ms/ques +[INFO] Cumulative metrics, sample: 728, P: 0.507, R: 0.558, macro F1: 0.531 +[INFO] Current question consumed time: 2997 ms, average: 2596.46 ms/ques, entity detection: 423.24 ms/ques (linking tool: 344.48 ms/ques), relation detection: 860.10 ms/ques (similarity: 698.31 ms/ques), EDG generation: 46.67 ms/ques [INFO] Entity Linking P: 0.7493131868131868, R: 0.7012362637362637, micro F1: 0.7165750915750926, macro F1: 0.724477995962371 [INFO] Relation Linking P: 0.6015338827838828, R: 0.5473901098901099, micro F1: 0.5627747252747259, macro F1: 0.5731862164934797 [INFO] Type Linking P: 0.5467032967032966, R: 0.5467032967032966, micro F1: 0.5467032967032966, macro F1: 0.5467032967032966 @@ -10223,8 +10223,8 @@ Question 728: What is the name of the homeground of football team Panionios G.S. [INFO] Golden Answer: [http://dbpedia.org/resource/Athens, http://dbpedia.org/resource/Nea_Smyrni_Stadium] [INFO] Predicted Answer: [http://dbpedia.org/resource/Alexandros_Mouzakitis, http://dbpedia.org/resource/Damian_Gjini, http://dbpedia.org/resource/Pavlos_Mitropoulos, http://dbpedia.org/resource/Marinos_Ouzounidis, http://dbpedia.org/resource/Kosmas_Gezos, http://dbpedia.org/resource/Anastasios_Bakasetas, http://dbpedia.org/resource/Evangelos_Ikonomou, http://dbpedia.org/resource/Leonidas_Argyropoulos, http://dbpedia.org/resource/Nikolaos_Papadopoulos_(football), http://dbpedia.org/resource/Panagiotis_Korbos, http://dbpedia.org/resource/Stelios_Pozatzidis, http://dbpedia.org/resource/Valentinos_Vlachos] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 729, P: 0.506, R: 0.558, macro F1: 0.513, macro F1*: 0.531 -[INFO] Current time: 1477 ms, average: 2878.35 ms/ques, entity detection: 478.40 ms/ques (linking tool: 385.21 ms/ques), relation detection: 931.48 ms/ques (similarity: 706.12 ms/ques), EDG generation: 46.73 ms/ques +[INFO] Cumulative metrics, sample: 729, P: 0.506, R: 0.558, macro F1: 0.531 +[INFO] Current question consumed time: 1268 ms, average: 2594.64 ms/ques, entity detection: 422.97 ms/ques (linking tool: 344.23 ms/ques), relation detection: 859.60 ms/ques (similarity: 697.79 ms/ques), EDG generation: 46.65 ms/ques [INFO] Entity Linking P: 0.7482853223593965, R: 0.7002743484224966, micro F1: 0.7155921353452228, macro F1: 0.7234841989857422 [INFO] Relation Linking P: 0.6007087334247827, R: 0.546639231824417, micro F1: 0.5620027434842256, macro F1: 0.5723999528220207 [INFO] Type Linking P: 0.5473251028806584, R: 0.5473251028806584, micro F1: 0.5473251028806584, macro F1: 0.5473251028806584 @@ -10237,8 +10237,8 @@ Question 729: List presidents of the school whose one of the president is Charle [INFO] Golden Answer: [http://dbpedia.org/resource/Louis_Mountbatten,_1st_Earl_Mountbatten_of_Burma, http://dbpedia.org/resource/Nelson_Mandela, http://dbpedia.org/resource/Charles,_Prince_of_Wales, Queen Noor of Jordan@en, The Prince of Wales@en] [INFO] Predicted Answer: [http://dbpedia.org/resource/Koch_Industries] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 730, P: 0.505, R: 0.557, macro F1: 0.512, macro F1*: 0.530 -[INFO] Current time: 2004 ms, average: 2877.15 ms/ques, entity detection: 478.04 ms/ques (linking tool: 384.84 ms/ques), relation detection: 930.79 ms/ques (similarity: 705.72 ms/ques), EDG generation: 46.73 ms/ques +[INFO] Cumulative metrics, sample: 730, P: 0.505, R: 0.557, macro F1: 0.530 +[INFO] Current question consumed time: 1719 ms, average: 2593.44 ms/ques, entity detection: 422.63 ms/ques (linking tool: 343.88 ms/ques), relation detection: 858.95 ms/ques (similarity: 697.40 ms/ques), EDG generation: 46.65 ms/ques [INFO] Entity Linking P: 0.7472602739726028, R: 0.6993150684931507, micro F1: 0.7146118721461198, macro F1: 0.7224931247405563 [INFO] Relation Linking P: 0.5998858447488584, R: 0.5458904109589041, micro F1: 0.5612328767123295, macro F1: 0.5716158432976071 [INFO] Type Linking P: 0.5465753424657535, R: 0.5465753424657535, micro F1: 0.5465753424657535, macro F1: 0.5465753424657535 @@ -10251,8 +10251,8 @@ Question 730: Does Timothy Morton study Buddhism? [INFO] Golden Answer: [true] [INFO] Predicted Answer: [false] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 731, P: 0.505, R: 0.556, macro F1: 0.511, macro F1*: 0.529 -[INFO] Current time: 1243 ms, average: 2874.92 ms/ques, entity detection: 478.11 ms/ques (linking tool: 384.95 ms/ques), relation detection: 929.52 ms/ques (similarity: 704.76 ms/ques), EDG generation: 46.68 ms/ques +[INFO] Cumulative metrics, sample: 731, P: 0.505, R: 0.556, macro F1: 0.529 +[INFO] Current question consumed time: 1006 ms, average: 2591.27 ms/ques, entity detection: 422.70 ms/ques (linking tool: 343.99 ms/ques), relation detection: 857.78 ms/ques (similarity: 696.45 ms/ques), EDG generation: 46.60 ms/ques [INFO] Entity Linking P: 0.7462380300957593, R: 0.698358413132695, micro F1: 0.7136342909256737, macro F1: 0.7215047620528128 [INFO] Relation Linking P: 0.5990652074783401, R: 0.5451436388508892, micro F1: 0.5604651162790705, macro F1: 0.5708338790796897 [INFO] Type Linking P: 0.5471956224350205, R: 0.5471956224350205, micro F1: 0.5471956224350205, macro F1: 0.5471956224350205 @@ -10265,8 +10265,8 @@ Question 731: Brian Moriarty designed video games written by whom? [INFO] Golden Answer: [http://dbpedia.org/resource/Brian_Moriarty, http://dbpedia.org/resource/Orson_Scott_Card, http://dbpedia.org/resource/Steven_Spielberg] [INFO] Predicted Answer: [http://dbpedia.org/resource/Beyond_Zork, http://dbpedia.org/resource/Trinity_(video_game), http://dbpedia.org/resource/Wishbringer, http://dbpedia.org/resource/The_Dig, http://dbpedia.org/resource/Loom_(video_game)] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 732, P: 0.504, R: 0.555, macro F1: 0.510, macro F1*: 0.528 -[INFO] Current time: 955 ms, average: 2872.29 ms/ques, entity detection: 477.96 ms/ques (linking tool: 384.80 ms/ques), relation detection: 928.37 ms/ques (similarity: 703.88 ms/ques), EDG generation: 46.63 ms/ques +[INFO] Cumulative metrics, sample: 732, P: 0.504, R: 0.555, macro F1: 0.528 +[INFO] Current question consumed time: 758 ms, average: 2588.76 ms/ques, entity detection: 422.56 ms/ques (linking tool: 343.86 ms/ques), relation detection: 856.71 ms/ques (similarity: 695.58 ms/ques), EDG generation: 46.55 ms/ques [INFO] Entity Linking P: 0.7465846994535519, R: 0.6987704918032787, micro F1: 0.7140255009107479, macro F1: 0.7218867179026309 [INFO] Relation Linking P: 0.5996129326047358, R: 0.5450819672131147, micro F1: 0.5606102003642994, macro F1: 0.5710485770883093 [INFO] Type Linking P: 0.546448087431694, R: 0.546448087431694, micro F1: 0.546448087431694, macro F1: 0.546448087431694 @@ -10279,8 +10279,8 @@ Question 732: What are some movies whose distributors are led by Kevin Tsujihara [INFO] Golden Answer: [http://dbpedia.org/resource/Casablanca_(film), http://dbpedia.org/resource/Cleopatra_Jones, http://dbpedia.org/resource/Constantine_(film), http://dbpedia.org/resource/Mad_Max_Beyond_Thunderdome, http://dbpedia.org/resource/Major_League_II, http://dbpedia.org/resource/National_Lampoon's_Vacation_(film_series), http://dbpedia.org/resource/One-Trick_Pony_(film), http://dbpedia.org/resource/Tango_&_Cash, http://dbpedia.org/resource/Welcome_to_Collinwood, http://dbpedia.org/resource/A_Slight_Case_of_Murder, http://dbpedia.org/resource/A_Star_Is_Born_(1976_film), http://dbpedia.org/resource/Book_Revue, http://dbpedia.org/resource/Confidential_Agent, http://dbpedia.org/resource/Death_in_Venice_(film), http://dbpedia.org/resource/Exorcist_II:_The_Heretic, http://dbpedia.org/resource/Firefox_(film), http://dbpedia.org/resource/Free_Willy_(film_series), http://dbpedia.org/resource/Green_Lantern_(film), http://dbpedia.org/resource/Greystoke:_The_Legend_of_Tarzan,_Lord_of_the_Apes, http://dbpedia.org/resource/It's_a_Great_Feeling, http://dbpedia.org/resource/It_Happened_at_the_World's_Fair, http://dbpedia.org/resource/Jonah_Hex_(film), http://dbpedia.org/resource/Joyful_Noise_(film), http://dbpedia.org/resource/Looney_Tunes:_Back_in_Action, http://dbpedia.org/resource/Monty_Python's_Life_of_Brian, http://dbpedia.org/resource/Natural_Born_Killers, http://dbpedia.org/resource/Payback_(1999_film), http://dbpedia.org/resource/Strictly_Business_(1991_film), http://dbpedia.org/resource/Suicide_Squad_(film), http://dbpedia.org/resource/Superman_II, http://dbpedia.org/resource/The_Blue_Gardenia, http://dbpedia.org/resource/The_Flight_of_Dragons, http://dbpedia.org/resource/The_Hobbit:_The_Battle_of_the_Five_Armies, http://dbpedia.org/resource/The_Hobbit:_The_Desolation_of_Smaug, http://dbpedia.org/resource/The_Inspector_General_(film), http://dbpedia.org/resource/The_Madwoman_of_Chaillot_(film), http://dbpedia.org/resource/The_Man_from_U.N.C.L.E._(film), http://dbpedia.org/resource/Treasure_Island_(1972_animated_film), http://dbpedia.org/resource/Two_If_by_Sea, http://dbpedia.org/resource/We_Are_Your_Friends_(film), http://dbpedia.org/resource/Wonder_Woman_(2017_film), http://dbpedia.org/resource/300:_Rise_of_an_Empire, http://dbpedia.org/resource/A_Nightmare_on_Elm_Street_(2010_film), http://dbpedia.org/resource/American_Anthem, http://dbpedia.org/resource/Any_Wednesday, http://dbpedia.org/resource/Arizona_Dream, http://dbpedia.org/resource/Big_Wednesday, http://dbpedia.org/resource/Cats_Don't_Dance, http://dbpedia.org/resource/Chasers, http://dbpedia.org/resource/Entourage_(film), http://dbpedia.org/resource/First_to_Fight_(film), http://dbpedia.org/resource/Gigot_(film), http://dbpedia.org/resource/I_Am_Legend_(film), http://dbpedia.org/resource/Let's_Do_It_Again_(1975_film), http://dbpedia.org/resource/Marjorie_Morningstar_(film), http://dbpedia.org/resource/Mustafa_(film), http://dbpedia.org/resource/Night_Moves_(1975_film), http://dbpedia.org/resource/No_Reservations_(film), http://dbpedia.org/resource/Rhapsody_in_Blue_(film), http://dbpedia.org/resource/Scarecrow_(1973_film), http://dbpedia.org/resource/Straight_Time, http://dbpedia.org/resource/Super_Fly_(film), http://dbpedia.org/resource/Target_(1985_film), http://dbpedia.org/resource/The_Candidate_(1972_film), http://dbpedia.org/resource/The_Chapman_Report, http://dbpedia.org/resource/The_Gauntlet_(film), http://dbpedia.org/resource/The_Gumball_Rally, http://dbpedia.org/resource/The_Others_(2001_film), http://dbpedia.org/resource/The_Story_of_Mankind_(film), http://dbpedia.org/resource/Under_Siege, http://dbpedia.org/resource/What_Ever_Happened_to_Baby_Jane%3F_(1962_film), http://dbpedia.org/resource/Wyatt_Earp_(film), http://dbpedia.org/resource/Zandy's_Bride, http://dbpedia.org/resource/3000_Miles_to_Graceland, http://dbpedia.org/resource/Gossip_(2000_American_film), http://dbpedia.org/resource/Humoresque_(film), http://dbpedia.org/resource/Me_and_Orson_Welles, http://dbpedia.org/resource/New_Jack_City, http://dbpedia.org/resource/Ninja_Assassin, http://dbpedia.org/resource/Racing_Stripes, http://dbpedia.org/resource/Ricochet_(film), http://dbpedia.org/resource/Urgh!_A_Music_War, http://dbpedia.org/resource/A_Distant_Trumpet, http://dbpedia.org/resource/A_Stolen_Life_(1946_film), http://dbpedia.org/resource/A_Summer_Place_(film), http://dbpedia.org/resource/All_This,_and_Heaven_Too, http://dbpedia.org/resource/Band_of_Angels, http://dbpedia.org/resource/Battle_Cry_(film), http://dbpedia.org/resource/Chernobyl_Diaries, http://dbpedia.org/resource/Confessions_of_a_Nazi_Spy, http://dbpedia.org/resource/Creepshow, http://dbpedia.org/resource/Crime_School, http://dbpedia.org/resource/House_of_Wax_(2005_film), http://dbpedia.org/resource/Ice_Palace_(film), http://dbpedia.org/resource/Jezebel_(film), http://dbpedia.org/resource/Kid_Galahad_(1937_film), http://dbpedia.org/resource/Moby_Dick_(1956_film), http://dbpedia.org/resource/My_Wild_Irish_Rose, http://dbpedia.org/resource/Operation_Pacific, http://dbpedia.org/resource/Rocky_Mountain_(film), http://dbpedia.org/resource/Rome_Adventure, http://dbpedia.org/resource/Shining_Victory, http://dbpedia.org/resource/So_Big_(1953_film), http://dbpedia.org/resource/Spencer's_Mountain, http://dbpedia.org/resource/Susan_Slade, http://dbpedia.org/resource/That_Certain_Woman, http://dbpedia.org/resource/The_Beast_with_Five_Fingers, http://dbpedia.org/resource/The_Charge_at_Feather_River, http://dbpedia.org/resource/The_Conspirators, http://dbpedia.org/resource/The_Dawn_Patrol_(1938_film), http://dbpedia.org/resource/The_Glass_Menagerie_(1950_film), http://dbpedia.org/resource/The_Jazz_Singer_(1952_film), http://dbpedia.org/resource/The_Letter_(1940_film), http://dbpedia.org/resource/The_Man_Without_a_Face, http://dbpedia.org/resource/The_Miracle_of_Our_Lady_of_Fatima, http://dbpedia.org/resource/The_Old_Maid_(1939_film), http://dbpedia.org/resource/The_Sisters_(1938_film), http://dbpedia.org/resource/Two_on_a_Guillotine, http://dbpedia.org/resource/Winter_Meeting, http://dbpedia.org/resource/Action_in_the_North_Atlantic, http://dbpedia.org/resource/Any_Which_Way_You_Can, http://dbpedia.org/resource/Body_of_Lies_(film), http://dbpedia.org/resource/Edge_of_Darkness_(1943_film), http://dbpedia.org/resource/Every_Which_Way_but_Loose_(film), http://dbpedia.org/resource/Love_Is_on_the_Air, http://dbpedia.org/resource/Missing_in_Action_(film), http://dbpedia.org/resource/Proof_of_Life, http://dbpedia.org/resource/Red_Planet_(film), http://dbpedia.org/resource/Saturday's_Children, http://dbpedia.org/resource/Shorts:_The_Adventures_of_the_Wishing_Rock, http://dbpedia.org/resource/Stroker_Ace, http://dbpedia.org/resource/The_Box_(2009_film), http://dbpedia.org/resource/The_Dukes_of_Hazzard_(film), http://dbpedia.org/resource/The_Water_Diviner, http://dbpedia.org/resource/Winter's_Tale_(film), http://dbpedia.org/resource/Bald:_The_Making_of_THX_1138, http://dbpedia.org/resource/Creed_(film), http://dbpedia.org/resource/Freejack, http://dbpedia.org/resource/I'll_Be_There_(2003_film), http://dbpedia.org/resource/Nothing_but_Trouble_(1991_film), http://dbpedia.org/resource/Rocky_(film_series), http://dbpedia.org/resource/The_Hangover, http://dbpedia.org/resource/Baby_Face_(film), http://dbpedia.org/resource/Broadway_Gondolier, http://dbpedia.org/resource/Captain_Blood_(1935_film), http://dbpedia.org/resource/Deal_of_the_Century, http://dbpedia.org/resource/Godzilla-Kong_cinematic_universe, http://dbpedia.org/resource/Gypsy_(1962_film), http://dbpedia.org/resource/Hearts_in_Atlantis_(film), http://dbpedia.org/resource/Hollywood_Canteen_(film), http://dbpedia.org/resource/Kit_Kittredge:_An_American_Girl, http://dbpedia.org/resource/Lethal_Weapon_(film_series), http://dbpedia.org/resource/Lumber_Jack-Rabbit, http://dbpedia.org/resource/Maverick_(film), http://dbpedia.org/resource/Memoirs_of_an_Invisible_Man_(film), http://dbpedia.org/resource/Mom_and_Dad_Save_the_World, http://dbpedia.org/resource/No_Time_for_Sergeants_(1958_film), http://dbpedia.org/resource/Scooby-Doo_(film_series), http://dbpedia.org/resource/Simon_(1980_film), http://dbpedia.org/resource/Sinkin'_in_the_Bathtub, http://dbpedia.org/resource/Summer_Wars, http://dbpedia.org/resource/Teenage_Mutant_Ninja_Turtles_(film_series), http://dbpedia.org/resource/The_Bad_Seed_(1956_film), http://dbpedia.org/resource/The_Hangover_Part_II, http://dbpedia.org/resource/The_Man_Who_Played_God_(1932_film), http://dbpedia.org/resource/The_Prime_Minister_(film), http://dbpedia.org/resource/August_Rush, http://dbpedia.org/resource/Elvis_Has_Left_the_Building, http://dbpedia.org/resource/Movie_Movie, http://dbpedia.org/resource/Not_with_My_Wife,_You_Don't!, http://dbpedia.org/resource/Oh,_God!, http://dbpedia.org/resource/Plop_Goes_the_Weasel, http://dbpedia.org/resource/The_Ramen_Girl, http://dbpedia.org/resource/Unaccompanied_Minors, http://dbpedia.org/resource/Addams_Family_Reunion, http://dbpedia.org/resource/Bogus_(film), http://dbpedia.org/resource/Clara's_Heart, http://dbpedia.org/resource/Damn_Yankees_(film), http://dbpedia.org/resource/Madame_Du_Barry_(1934_film), http://dbpedia.org/resource/Made_in_America_(1993_film), http://dbpedia.org/resource/Major_League:_Back_to_the_Minors, http://dbpedia.org/resource/Mr._Church, http://dbpedia.org/resource/One_Eight_Seven, http://dbpedia.org/resource/Over_the_Top_(film), http://dbpedia.org/resource/Sarafina!_(film), http://dbpedia.org/resource/The_Hangover_Part_III, http://dbpedia.org/resource/Tin_Pan_Alley_Cats, http://dbpedia.org/resource/Viva_Knievel!, http://dbpedia.org/resource/Evidence_(1929_film), http://dbpedia.org/resource/In_the_Heart_of_the_Sea_(film), http://dbpedia.org/resource/Little_Nellie_Kelly, http://dbpedia.org/resource/Other_People's_Money, http://dbpedia.org/resource/The_Good_Lie, http://dbpedia.org/resource/Aviation_Vacation, http://dbpedia.org/resource/Bad_Education_(film), http://dbpedia.org/resource/Cannonball_Run_II, http://dbpedia.org/resource/Duffy_of_San_Quentin, http://dbpedia.org/resource/Illegal_(1955_film), http://dbpedia.org/resource/Smile,_Darn_Ya,_Smile!, http://dbpedia.org/resource/The_Train_Robbers, http://dbpedia.org/resource/U.S._Marshals_(film), http://dbpedia.org/resource/A_Bird_in_a_Bonnet, http://dbpedia.org/resource/A_Street_Cat_Named_Sylvester, http://dbpedia.org/resource/A_Tale_of_Two_Kitties, http://dbpedia.org/resource/Ain't_She_Tweet, http://dbpedia.org/resource/Bad_Ol'_Putty_Tat, http://dbpedia.org/resource/Birdy_and_the_Beast, http://dbpedia.org/resource/Catty_Cornered, http://dbpedia.org/resource/Dog_Pounded, http://dbpedia.org/resource/Fowl_Weather, http://dbpedia.org/resource/Freebie_and_the_Bean, http://dbpedia.org/resource/Front_Page_Woman, http://dbpedia.org/resource/Gift_Wrapped_(film), http://dbpedia.org/resource/Hawaiian_Aye_Aye, http://dbpedia.org/resource/Heir-Conditioned, http://dbpedia.org/resource/Hyde_and_Go_Tweet, http://dbpedia.org/resource/I'll_See_You_in_My_Dreams_(1951_film), http://dbpedia.org/resource/I_Taw_a_Putty_Tat, http://dbpedia.org/resource/Malibu's_Most_Wanted, http://dbpedia.org/resource/Muzzle_Tough, http://dbpedia.org/resource/Prince_Violent, http://dbpedia.org/resource/Putty_Tat_Trouble, http://dbpedia.org/resource/Red_Riding_Hoodwinked, http://dbpedia.org/resource/Rollover_(film), http://dbpedia.org/resource/Room_and_Bird, http://dbpedia.org/resource/Running_on_Empty_(1988_film), http://dbpedia.org/resource/Satan's_Waitin', http://dbpedia.org/resource/Snow_Business, http://dbpedia.org/resource/The_Jet_Cage, http://dbpedia.org/resource/The_Last_Hungry_Cat, http://dbpedia.org/resource/Tom_Tom_Tomcat, http://dbpedia.org/resource/Tree_Cornered_Tweety, http://dbpedia.org/resource/Trick_or_Tweet, http://dbpedia.org/resource/Tugboat_Granny, http://dbpedia.org/resource/Tweet_Tweet_Tweety, http://dbpedia.org/resource/Tweet_Zoo, http://dbpedia.org/resource/Tweet_and_Sour, http://dbpedia.org/resource/Tweety's_Circus, http://dbpedia.org/resource/Tweety's_S.O.S., http://dbpedia.org/resource/Tweety_and_the_Beanstalk, http://dbpedia.org/resource/Wacky_Blackout, http://dbpedia.org/resource/Wings_of_the_Navy, http://dbpedia.org/resource/Adventures_of_the_Road_Runner, http://dbpedia.org/resource/All_at_Sea_(1935_film), http://dbpedia.org/resource/Bosko's_Picture_Show, http://dbpedia.org/resource/Chaser_on_the_Rocks, http://dbpedia.org/resource/Clippety_Clobbered, http://dbpedia.org/resource/Compressed_Hare, http://dbpedia.org/resource/Fast_and_Furry-ous, http://dbpedia.org/resource/Fastest_with_the_Mostest, http://dbpedia.org/resource/Five_Days_One_Summer, http://dbpedia.org/resource/Gee_Whiz-z-z-z-z-z-z, http://dbpedia.org/resource/Going!_Going!_Gosh!, http://dbpedia.org/resource/Guided_Muscle, http://dbpedia.org/resource/Hare-Breadth_Hurry, http://dbpedia.org/resource/Hip_Hip-Hurry!, http://dbpedia.org/resource/Hook,_Line_and_Stinker, http://dbpedia.org/resource/Hopalong_Casualty, http://dbpedia.org/resource/Jamboree_(1957_film), http://dbpedia.org/resource/Lickety-Splat, http://dbpedia.org/resource/Operation:_Rabbit, http://dbpedia.org/resource/Pure_Country_2:_The_Gift, http://dbpedia.org/resource/Rabbit's_Feat, http://dbpedia.org/resource/Ready,_Set,_Zoom!, http://dbpedia.org/resource/Road_Runner_a_Go-Go, http://dbpedia.org/resource/Run,_Run,_Sweet_Road_Runner, http://dbpedia.org/resource/Rushing_Roulette, http://dbpedia.org/resource/Scrambled_Aches, http://dbpedia.org/resource/Soup_or_Sonic, http://dbpedia.org/resource/Stop!_Look!_And_Hasten!, http://dbpedia.org/resource/Sugar_and_Spies, http://dbpedia.org/resource/The_Mouse-Merized_Cat, http://dbpedia.org/resource/The_Ritz_(film), http://dbpedia.org/resource/The_Solid_Tin_Coyote, http://dbpedia.org/resource/The_Wild_Chase, http://dbpedia.org/resource/To_Beep_or_Not_to_Beep, http://dbpedia.org/resource/To_Hare_Is_Human, http://dbpedia.org/resource/War_and_Pieces, http://dbpedia.org/resource/Whoa,_Be-Gone!, http://dbpedia.org/resource/Wild_About_Hurry, http://dbpedia.org/resource/Zip_'N_Snort, http://dbpedia.org/resource/Zipping_Along, http://dbpedia.org/resource/Zoom_at_the_Top, http://dbpedia.org/resource/Death_to_Smoochy, http://dbpedia.org/resource/Divine_Madness_(film), http://dbpedia.org/resource/Jack_and_the_Beanstalk_(1952_film), http://dbpedia.org/resource/Jungle_Book:_Origins, http://dbpedia.org/resource/License_to_Wed, http://dbpedia.org/resource/Pie,_Pie_Blackbird, http://dbpedia.org/resource/Police_Academy_2:_Their_First_Assignment, http://dbpedia.org/resource/The_Pack_(1977_film), http://dbpedia.org/resource/The_Town_(2010_film), http://dbpedia.org/resource/Hearts_and_Armour, http://dbpedia.org/resource/Immigrants_(2008_film), http://dbpedia.org/resource/Rocket_Squad, http://dbpedia.org/resource/Surrender_(1987_film), http://dbpedia.org/resource/Made_in_Paris, http://dbpedia.org/resource/Observe_and_Report, http://dbpedia.org/resource/What_a_Girl_Wants_(film), http://dbpedia.org/resource/City_by_the_Sea, http://dbpedia.org/resource/Fool's_Gold_(2008_film), http://dbpedia.org/resource/Madame_Bovary_(2014_film), http://dbpedia.org/resource/The_Negotiator, http://dbpedia.org/resource/Chill_Factor_(film), http://dbpedia.org/resource/The_Stars_Fell_on_Henrietta, http://dbpedia.org/resource/Ain't_That_Ducky, http://dbpedia.org/resource/Bad_Moon, http://dbpedia.org/resource/Boiling_Point_(1993_film), http://dbpedia.org/resource/Bosko,_the_Talk-Ink_Kid, http://dbpedia.org/resource/Casper_Meets_Wendy, http://dbpedia.org/resource/Class_Act, http://dbpedia.org/resource/Daffy_Dilly, http://dbpedia.org/resource/Fallen_(1998_film), http://dbpedia.org/resource/His_Bitter_Half, http://dbpedia.org/resource/I_Died_a_Thousand_Times, http://dbpedia.org/resource/Little_Red_Riding_Rabbit, http://dbpedia.org/resource/Mickey_Blue_Eyes, http://dbpedia.org/resource/Million_Dollar_Baby_(1941_film), http://dbpedia.org/resource/Notes_to_You, http://dbpedia.org/resource/Peck_Up_Your_Troubles, http://dbpedia.org/resource/Pure_Country, http://dbpedia.org/resource/Red_Nightmare, http://dbpedia.org/resource/Scaredy_Cat, http://dbpedia.org/resource/Steel_(1997_film), http://dbpedia.org/resource/The_Big_Snooze, http://dbpedia.org/resource/The_D.I._(film), http://dbpedia.org/resource/The_NeverEnding_Story_II:_The_Next_Chapter, http://dbpedia.org/resource/Tin_Cup, http://dbpedia.org/resource/Trojan_War_(film), http://dbpedia.org/resource/Vision_Quest, http://dbpedia.org/resource/White_Sands_(film), http://dbpedia.org/resource/Who's_That_Girl_(1987_film), http://dbpedia.org/resource/You_Were_Never_Duckier, http://dbpedia.org/resource/Across_the_Pacific_(1926_film), http://dbpedia.org/resource/Catalina_Caper, http://dbpedia.org/resource/Fighter_Squadron, http://dbpedia.org/resource/Hardboiled_Rose, http://dbpedia.org/resource/High_Pressure_(film), http://dbpedia.org/resource/Jimi_Hendrix_(film), http://dbpedia.org/resource/Mr._Dodd_Takes_the_Air, http://dbpedia.org/resource/Page_Miss_Glory_(1935_film), http://dbpedia.org/resource/Something_Borrowed_(film), http://dbpedia.org/resource/Stand_and_Deliver, http://dbpedia.org/resource/The_Great_Divide_(1929_film), http://dbpedia.org/resource/The_Heart_of_New_York_(film), http://dbpedia.org/resource/The_Todd_Killings, http://dbpedia.org/resource/Get_Hard, http://dbpedia.org/resource/The_Captain's_Kid, http://dbpedia.org/resource/Space_Riders, http://dbpedia.org/resource/The_Factory_(film), http://dbpedia.org/resource/The_Man_Hunter, http://dbpedia.org/resource/Bugs_Bunny_and_the_Three_Bears, http://dbpedia.org/resource/One_Meat_Brawl, http://dbpedia.org/resource/Rabbit's_Kin, http://dbpedia.org/resource/Ichi_(2008_film), http://dbpedia.org/resource/The_Secret_of_Moonacre, http://dbpedia.org/resource/Toonpur_Ka_Super_Hero, http://dbpedia.org/resource/No_Parking_Hare, http://dbpedia.org/resource/The_Elephant_in_the_Living_Room, http://dbpedia.org/resource/Pigs_Is_Pigs_(1937_film), http://dbpedia.org/resource/She_Was_an_Acrobat's_Daughter, http://dbpedia.org/resource/You_Don't_Know_What_You're_Doin'!, http://dbpedia.org/resource/Claws_for_Alarm, http://dbpedia.org/resource/Oily_Hare, http://dbpedia.org/resource/The_Fair-Haired_Hare, http://dbpedia.org/resource/Springfield_Rifle_(film), http://dbpedia.org/resource/The_Widow_from_Monte_Carlo, http://dbpedia.org/resource/Born_Reckless_(1958_film), http://dbpedia.org/resource/Darby's_Rangers, http://dbpedia.org/resource/Me_Before_You_(film), http://dbpedia.org/resource/Production_of_Harry_Potter_and_the_Deathly_Hallows, http://dbpedia.org/resource/The_Keyhole, http://dbpedia.org/resource/Killing_Zelda_Sparks, http://dbpedia.org/resource/The_Ducksters, http://dbpedia.org/resource/Nickel_Mountain, http://dbpedia.org/resource/Deep_Sea_3D, http://dbpedia.org/resource/Windy_City_(film), http://dbpedia.org/resource/Bugs'_Bonnets, http://dbpedia.org/resource/Hare_Splitter, http://dbpedia.org/resource/Hold_the_Lion,_Please, http://dbpedia.org/resource/The_Diamond_Queen_(1953_film), http://dbpedia.org/resource/Henhouse_Henery, http://dbpedia.org/resource/Leghorn_Swoggled, http://dbpedia.org/resource/The_Egg-Cited_Rooster, http://dbpedia.org/resource/The_Leghorn_Blows_at_Midnight, http://dbpedia.org/resource/Chicago_Calling, http://dbpedia.org/resource/The_Sea_Gypsies_(1978_film), http://dbpedia.org/resource/Giovanni's_Island, http://dbpedia.org/resource/Efectos_secundarios, http://dbpedia.org/resource/General_Education_(film), http://dbpedia.org/resource/Lulu_and_Jimi, http://dbpedia.org/resource/Kokowääh, http://dbpedia.org/resource/Kokowääh_2, http://dbpedia.org/resource/A_Feather_in_His_Hare, http://dbpedia.org/resource/A_Star_Is_Bored, http://dbpedia.org/resource/Backwoods_Bunny, http://dbpedia.org/resource/Bonanza_Bunny, http://dbpedia.org/resource/Hare_Lift, http://dbpedia.org/resource/His_Hare-Raising_Tale, http://dbpedia.org/resource/Knights_Must_Fall, http://dbpedia.org/resource/Rabbit_Every_Monday, http://dbpedia.org/resource/Robot_Rabbit, http://dbpedia.org/resource/Wet_Hare, http://dbpedia.org/resource/Which_Is_Witch, http://dbpedia.org/resource/Gold_Rush_Daze, http://dbpedia.org/resource/Africa_United_(2010_film), http://dbpedia.org/resource/The_Babysitter_(2016_film), http://dbpedia.org/resource/Blue_Rodeo_(film), http://dbpedia.org/resource/The_Compulsory_Wife, http://dbpedia.org/resource/Now_That_Summer_is_Gone, http://dbpedia.org/resource/Central_Intelligence, http://dbpedia.org/resource/Don't_Look_Now_(1936_film), http://dbpedia.org/resource/The_Hidden_Hand_(1942_film), http://dbpedia.org/resource/Broadway_After_Dark, http://dbpedia.org/resource/Divorce_Among_Friends, http://dbpedia.org/resource/Milk_and_Money_(film), http://dbpedia.org/resource/All_a_Bir-r-r-d, http://dbpedia.org/resource/A_Bird_in_a_Guilty_Cage, http://dbpedia.org/resource/Neil's_Party, http://dbpedia.org/resource/War_Dogs_(2016_film), http://dbpedia.org/resource/A.I._Artificial_Intelligence, http://dbpedia.org/resource/ABBA:_The_Movie, http://dbpedia.org/resource/A_Star_Is_Born_(1954_film), http://dbpedia.org/resource/A_Streetcar_Named_Desire_(1951_film), http://dbpedia.org/resource/Adventures_of_Don_Juan, http://dbpedia.org/resource/Agatha_(film), http://dbpedia.org/resource/All_the_President's_Men_(film), http://dbpedia.org/resource/America_America, http://dbpedia.org/resource/Auntie_Mame_(film), http://dbpedia.org/resource/Badlands_(film), http://dbpedia.org/resource/Barry_Lyndon, http://dbpedia.org/resource/Batman_(1989_film), http://dbpedia.org/resource/Blade_Runner, http://dbpedia.org/resource/Captains_of_the_Clouds, http://dbpedia.org/resource/Corpse_Bride, http://dbpedia.org/resource/Dangerous_Liaisons, http://dbpedia.org/resource/Days_of_Wine_and_Roses_(film), http://dbpedia.org/resource/Dial_M_for_Murder, http://dbpedia.org/resource/Dreams_(1990_film), http://dbpedia.org/resource/Driving_Miss_Daisy, http://dbpedia.org/resource/Empire_of_the_Sun_(film), http://dbpedia.org/resource/Enchanted_Island_(film), http://dbpedia.org/resource/Evil_Under_the_Sun_(1982_film), http://dbpedia.org/resource/George_Washington_Slept_Here, http://dbpedia.org/resource/Giant_(1956_film), http://dbpedia.org/resource/Gods_and_Generals_(film), http://dbpedia.org/resource/Gravity_(film), http://dbpedia.org/resource/Harry_Potter_(film_series), http://dbpedia.org/resource/Harry_Potter_and_the_Deathly_Hallows_–_Part_1, http://dbpedia.org/resource/Harry_Potter_and_the_Deathly_Hallows_–_Part_2, http://dbpedia.org/resource/Harry_Potter_and_the_Goblet_of_Fire_(film), http://dbpedia.org/resource/Harry_Potter_and_the_Philosopher's_Stone_(film), http://dbpedia.org/resource/Harry_Potter_and_the_Prisoner_of_Azkaban_(film), http://dbpedia.org/resource/Helen_of_Troy_(film), http://dbpedia.org/resource/Her_(film), http://dbpedia.org/resource/Hereafter_(film), http://dbpedia.org/resource/I_Confess_(film), http://dbpedia.org/resource/Inception, http://dbpedia.org/resource/Inside_Daisy_Clover, http://dbpedia.org/resource/Insomnia_(2002_film), http://dbpedia.org/resource/Interstellar_(film), http://dbpedia.org/resource/JFK_(film), http://dbpedia.org/resource/Johnny_Belinda_(1948_film), http://dbpedia.org/resource/L.A._Confidential_(film), http://dbpedia.org/resource/Life_with_Father_(film), http://dbpedia.org/resource/Looney_Tunes, http://dbpedia.org/resource/Mad_Max:_Fury_Road, http://dbpedia.org/resource/Man_to_Man_(1930_film), http://dbpedia.org/resource/On_Deadly_Ground, http://dbpedia.org/resource/Osmosis_Jones, http://dbpedia.org/resource/Porky's_Hare_Hunt, http://dbpedia.org/resource/Purple_Rain_(film), http://dbpedia.org/resource/Romero_(film), http://dbpedia.org/resource/Rope_(film), http://dbpedia.org/resource/San_Antonio_(film), http://dbpedia.org/resource/Sayonara, http://dbpedia.org/resource/Sherlock_Holmes_(2009_film), http://dbpedia.org/resource/Space_Jam, http://dbpedia.org/resource/Stage_Fright_(1950_film), http://dbpedia.org/resource/Strangers_on_a_Train_(film), http://dbpedia.org/resource/Sunrise_at_Campobello, http://dbpedia.org/resource/The_Adventures_of_Mark_Twain_(1944_film), http://dbpedia.org/resource/The_Artist_(film), http://dbpedia.org/resource/The_Aviator_(2004_film), http://dbpedia.org/resource/The_Bad_and_the_Beautiful, http://dbpedia.org/resource/The_Color_Purple_(film), http://dbpedia.org/resource/The_Curious_Case_of_Benjamin_Button_(film), http://dbpedia.org/resource/The_Dark_Knight_(film), http://dbpedia.org/resource/The_Desert_Song_(1943_film), http://dbpedia.org/resource/The_Devils_(film), http://dbpedia.org/resource/The_Exorcist_(film), http://dbpedia.org/resource/The_Fountainhead_(film), http://dbpedia.org/resource/The_Great_Gatsby_(2013_film), http://dbpedia.org/resource/The_Hobbit:_An_Unexpected_Journey, http://dbpedia.org/resource/The_Iron_Giant, http://dbpedia.org/resource/The_Last_Samurai, http://dbpedia.org/resource/The_Mission_(1986_film), http://dbpedia.org/resource/The_Music_Man_(1962_film), http://dbpedia.org/resource/The_Phantom_of_the_Opera_(2004_film), http://dbpedia.org/resource/The_Polar_Express_(film), http://dbpedia.org/resource/The_Right_Stuff_(film), http://dbpedia.org/resource/The_Sea_Hawk_(1940_film), http://dbpedia.org/resource/The_Towering_Inferno, http://dbpedia.org/resource/This_Is_the_Army, http://dbpedia.org/resource/Troy_(film), http://dbpedia.org/resource/Under_Capricorn, http://dbpedia.org/resource/Unforgiven, http://dbpedia.org/resource/Who's_Afraid_of_Virginia_Woolf%3F_(film), http://dbpedia.org/resource/20,000_Years_in_Sing_Sing, http://dbpedia.org/resource/A_Perfect_Murder, http://dbpedia.org/resource/Americathon, http://dbpedia.org/resource/Baby_Doll, http://dbpedia.org/resource/Back_Pay, http://dbpedia.org/resource/Big_Business_Girl, http://dbpedia.org/resource/Blondie_Johnson, http://dbpedia.org/resource/Body_Heat, http://dbpedia.org/resource/Brainstorm_(1965_film), http://dbpedia.org/resource/Bride_of_the_Regiment, http://dbpedia.org/resource/Bright_Lights_(1930_film), http://dbpedia.org/resource/Bright_Star_(film), http://dbpedia.org/resource/British_Agent, http://dbpedia.org/resource/Broadway_Babies, http://dbpedia.org/resource/Captain_Thunder_(film), http://dbpedia.org/resource/Central_Airport_(film), http://dbpedia.org/resource/Chasing_Liberty, http://dbpedia.org/resource/Children_of_the_Ritz, http://dbpedia.org/resource/Convention_City, http://dbpedia.org/resource/Dames, http://dbpedia.org/resource/Dark_Hazard, http://dbpedia.org/resource/Dave_(film), http://dbpedia.org/resource/Dead_Calm_(film), http://dbpedia.org/resource/Do_Your_Duty, http://dbpedia.org/resource/Doctor_X_(film), http://dbpedia.org/resource/Employees'_Entrance, http://dbpedia.org/resource/Fashions_of_1934, http://dbpedia.org/resource/Female_(1933_film), http://dbpedia.org/resource/Final_Analysis, http://dbpedia.org/resource/Fireman,_Save_My_Child_(1932_film), http://dbpedia.org/resource/First_Family_(film), http://dbpedia.org/resource/Five_Star_Final, http://dbpedia.org/resource/Fog_Over_Frisco, http://dbpedia.org/resource/Footlights_and_Fools, http://dbpedia.org/resource/G_Men, http://dbpedia.org/resource/Gentlemen_Are_Born, http://dbpedia.org/resource/Go_into_Your_Dance, http://dbpedia.org/resource/Gold_Diggers_of_1935, http://dbpedia.org/resource/Goodbye,_My_Fancy, http://dbpedia.org/resource/Goya's_Ghosts, http://dbpedia.org/resource/Happiness_Ahead, http://dbpedia.org/resource/Hard_to_Get_(1929_film), http://dbpedia.org/resource/Havana_Widows, http://dbpedia.org/resource/Heat_(1995_film), http://dbpedia.org/resource/Hell_on_Frisco_Bay, http://dbpedia.org/resource/Her_Majesty,_Love, http://dbpedia.org/resource/Her_Private_Life, http://dbpedia.org/resource/Hercules_(1958_film), http://dbpedia.org/resource/Heroes_for_Sale_(film), http://dbpedia.org/resource/High_Sierra_(film), http://dbpedia.org/resource/Hot_Stuff_(1929_film), http://dbpedia.org/resource/I_Am_a_Fugitive_from_a_Chain_Gang, http://dbpedia.org/resource/I_Loved_a_Woman, http://dbpedia.org/resource/I_Sell_Anything, http://dbpedia.org/resource/In_Caliente, http://dbpedia.org/resource/It's_Tough_to_Be_Famous, http://dbpedia.org/resource/J._Edgar, http://dbpedia.org/resource/Key_Largo_(film), http://dbpedia.org/resource/Kismet_(1930_film), http://dbpedia.org/resource/Klute, http://dbpedia.org/resource/Life_Begins_(film), http://dbpedia.org/resource/Lilac_Time_(1928_film), http://dbpedia.org/resource/Little_Caesar_(film), http://dbpedia.org/resource/Local_Boy_Makes_Good, http://dbpedia.org/resource/Look_Back_in_Anger_(1959_film), http://dbpedia.org/resource/Love_Is_a_Racket, http://dbpedia.org/resource/Love_and_the_Devil, http://dbpedia.org/resource/Mandalay_(film), http://dbpedia.org/resource/Manpower_(1941_film), http://dbpedia.org/resource/Massacre_(film), http://dbpedia.org/resource/Maybe_It's_Love_(1935_film), http://dbpedia.org/resource/Midnight_Alibi, http://dbpedia.org/resource/Mothers_Cry, http://dbpedia.org/resource/Murder_in_the_Clouds, http://dbpedia.org/resource/My_Fellow_Americans, http://dbpedia.org/resource/My_Reputation, http://dbpedia.org/resource/Mystic_River_(film), http://dbpedia.org/resource/Nancy_Drew_(2007_film), http://dbpedia.org/resource/Night_and_Day_(1946_film), http://dbpedia.org/resource/Now,_Voyager, http://dbpedia.org/resource/Numbered_Men, http://dbpedia.org/resource/Nuts_(film), http://dbpedia.org/resource/Our_Miss_Brooks_(film), http://dbpedia.org/resource/Outcast_(1928_film), http://dbpedia.org/resource/Paris_(1929_film), http://dbpedia.org/resource/Party_Husband, http://dbpedia.org/resource/Penrod_and_Sam_(1931_film), http://dbpedia.org/resource/Porky_in_Wackyland, http://dbpedia.org/resource/Private_Detective_62, http://dbpedia.org/resource/Registered_Nurse_(film), http://dbpedia.org/resource/Return_of_the_Terror, http://dbpedia.org/resource/Revolution_(1985_film), http://dbpedia.org/resource/Safe_in_Hell, http://dbpedia.org/resource/Scarlet_Pages, http://dbpedia.org/resource/Scarlet_Seas, http://dbpedia.org/resource/Show_Girl_in_Hollywood, http://dbpedia.org/resource/Smiling_Irish_Eyes, http://dbpedia.org/resource/Son_of_the_Gods, http://dbpedia.org/resource/Song_of_the_Flame, http://dbpedia.org/resource/Spies_Like_Us, http://dbpedia.org/resource/Spring_Is_Here_(film), http://dbpedia.org/resource/Synthetic_Sin, http://dbpedia.org/resource/Tarzan_and_the_Valley_of_Gold, http://dbpedia.org/resource/The_Big_Red_One, http://dbpedia.org/resource/The_Case_of_the_Curious_Bride, http://dbpedia.org/resource/The_Case_of_the_Velvet_Claws, http://dbpedia.org/resource/The_Cranes_Are_Flying, http://dbpedia.org/resource/The_Crash_(film), http://dbpedia.org/resource/The_Daffy_Doc, http://dbpedia.org/resource/The_Dark_at_the_Top_of_the_Stairs, http://dbpedia.org/resource/The_Doughgirls, http://dbpedia.org/resource/The_Drowning_Pool_(film), http://dbpedia.org/resource/The_Enforcer_(1951_film), http://dbpedia.org/resource/The_FBI_Story, http://dbpedia.org/resource/The_Famous_Ferguson_Case, http://dbpedia.org/resource/The_Finger_Points, http://dbpedia.org/resource/The_Getaway_(1972_film), http://dbpedia.org/resource/The_Girl_of_the_Golden_West_(1930_film), http://dbpedia.org/resource/The_Goodbye_Girl, http://dbpedia.org/resource/The_Gorilla_(1930_film), http://dbpedia.org/resource/The_Hatchet_Man, http://dbpedia.org/resource/The_Hot_Heiress, http://dbpedia.org/resource/The_Irish_in_Us, http://dbpedia.org/resource/The_Killing_Fields_(film), http://dbpedia.org/resource/The_Lady_Takes_a_Sailor, http://dbpedia.org/resource/The_Lady_Who_Dared, http://dbpedia.org/resource/The_Little_Giant_(film), http://dbpedia.org/resource/The_Man_with_Two_Faces_(1934_film), http://dbpedia.org/resource/The_Match_King, http://dbpedia.org/resource/The_Mind_Reader, http://dbpedia.org/resource/The_Other_Tomorrow, http://dbpedia.org/resource/The_Outsiders_(film), http://dbpedia.org/resource/The_Painted_Angel, http://dbpedia.org/resource/The_Pledge_(film), http://dbpedia.org/resource/The_Ruling_Voice, http://dbpedia.org/resource/The_Strange_Love_of_Molly_Louvain, http://dbpedia.org/resource/The_Tenderfoot_(film), http://dbpedia.org/resource/The_Voice_of_the_Turtle_(film), http://dbpedia.org/resource/The_Woman_from_Monte_Carlo, http://dbpedia.org/resource/The_World_Changes, http://dbpedia.org/resource/They_Drive_by_Night, http://dbpedia.org/resource/They_Made_Me_a_Fugitive, http://dbpedia.org/resource/Tomorrow_Is_Another_Day_(1951_film), http://dbpedia.org/resource/Twenty_Million_Sweethearts, http://dbpedia.org/resource/Two_Seconds, http://dbpedia.org/resource/Under_the_Rainbow, http://dbpedia.org/resource/Watch_on_the_Rhine, http://dbpedia.org/resource/Wedding_Rings_(film), http://dbpedia.org/resource/Week-End_Marriage, http://dbpedia.org/resource/Whiplash_(1948_film), http://dbpedia.org/resource/White_Heat, http://dbpedia.org/resource/Why_Be_Good%3F, http://dbpedia.org/resource/Wild_Boys_of_the_Road, http://dbpedia.org/resource/Woman_Hungry_(film), http://dbpedia.org/resource/Women_in_the_Wind, http://dbpedia.org/resource/Wonder_Bar, http://dbpedia.org/resource/You_Said_a_Mouthful, http://dbpedia.org/resource/Get_Carter_(2000_film), http://dbpedia.org/resource/Jesus_(1979_film), http://dbpedia.org/resource/Lean_on_Me_(film), http://dbpedia.org/resource/Looker, http://dbpedia.org/resource/Mary_Magdalene_(1914_film), http://dbpedia.org/resource/National_Lampoon's_European_Vacation, http://dbpedia.org/resource/Pacific_Rim_(film), http://dbpedia.org/resource/Sphere_(1998_film), http://dbpedia.org/resource/The_Bucket_List, http://dbpedia.org/resource/The_Informant!, http://dbpedia.org/resource/The_King_and_I_(1999_film), http://dbpedia.org/resource/The_Terminal_Man_(film), http://dbpedia.org/resource/A_Gruesome_Twosome, http://dbpedia.org/resource/A_Sound_of_Thunder_(film), http://dbpedia.org/resource/A_Very_Harold_&_Kumar_3D_Christmas, http://dbpedia.org/resource/Alex_&_Emma, http://dbpedia.org/resource/Alibi_Ike, http://dbpedia.org/resource/Baby_Bottleneck, http://dbpedia.org/resource/Curly_Sue, http://dbpedia.org/resource/Deadly_Friend, http://dbpedia.org/resource/Flipped_(film), http://dbpedia.org/resource/Message_in_a_Bottle_(film), http://dbpedia.org/resource/Outland_(film), http://dbpedia.org/resource/Performance_(film), http://dbpedia.org/resource/Starship_Invasions, http://dbpedia.org/resource/The_Hill_(film), http://dbpedia.org/resource/The_Illustrated_Man_(film), http://dbpedia.org/resource/The_Sheltering_Sky_(film), http://dbpedia.org/resource/The_Story_of_Us_(film), http://dbpedia.org/resource/White_Oleander_(film), http://dbpedia.org/resource/Elmer's_Pet_Rabbit, http://dbpedia.org/resource/Falling_Hare, http://dbpedia.org/resource/False_Hare, http://dbpedia.org/resource/Hare_Trigger, http://dbpedia.org/resource/Lady,_Play_Your_Mandolin!, http://dbpedia.org/resource/Life_with_Feathers, http://dbpedia.org/resource/Rabbit_Transit_(film), http://dbpedia.org/resource/Señorella_and_the_Glass_Huarache, http://dbpedia.org/resource/Stage_Door_Cartoon, http://dbpedia.org/resource/The_Nun's_Story_(film), http://dbpedia.org/resource/I_Haven't_Got_a_Hat, http://dbpedia.org/resource/Rurouni_Kenshin_(film), http://dbpedia.org/resource/A_Modern_Hero, http://dbpedia.org/resource/Bastards_(2016_film), http://dbpedia.org/resource/Bordertown_(1935_film), http://dbpedia.org/resource/Bugs_Bunny's_3rd_Movie:_1001_Rabbit_Tales, http://dbpedia.org/resource/Daffy_Duck's_Quackbusters, http://dbpedia.org/resource/Eye_of_the_Devil, http://dbpedia.org/resource/Happy_Feet_Two, http://dbpedia.org/resource/Meeting_Venus, http://dbpedia.org/resource/Pan_(2015_film), http://dbpedia.org/resource/The_Cabin_in_the_Cotton, http://dbpedia.org/resource/The_Damned_(1969_film), http://dbpedia.org/resource/The_Wind_in_the_Willows_(1987_film), http://dbpedia.org/resource/Voltaire_(film), http://dbpedia.org/resource/We_Are_Not_Alone_(1939_film), http://dbpedia.org/resource/Yes_Man_(film), http://dbpedia.org/resource/A_Corny_Concerto, http://dbpedia.org/resource/A_Mutt_in_a_Rut, http://dbpedia.org/resource/Assault_and_Peppered, http://dbpedia.org/resource/Back_Roads_(film), http://dbpedia.org/resource/Beep,_Beep_(film), http://dbpedia.org/resource/Better_Late_Than_Never_(film), http://dbpedia.org/resource/Brother_Brat, http://dbpedia.org/resource/Corn_on_the_Cop, http://dbpedia.org/resource/Daffy's_Diner, http://dbpedia.org/resource/Daffy_Doodles, http://dbpedia.org/resource/Daffy_Duck_&_Egghead, http://dbpedia.org/resource/Daffy_Duck_Hunt, http://dbpedia.org/resource/Daffy_–_The_Commando, http://dbpedia.org/resource/Dear_Heart, http://dbpedia.org/resource/Deduce,_You_Say!, http://dbpedia.org/resource/Don't_Axe_Me, http://dbpedia.org/resource/Draftee_Daffy, http://dbpedia.org/resource/Drip-Along_Daffy, http://dbpedia.org/resource/Ducking_the_Devil, http://dbpedia.org/resource/Feather_Finger, http://dbpedia.org/resource/Gold_Diggers_of_'49, http://dbpedia.org/resource/Hot_Cross_Bunny, http://dbpedia.org/resource/Kitty_Kornered, http://dbpedia.org/resource/Plane_Daffy, http://dbpedia.org/resource/Porky's_Badtime_Story, http://dbpedia.org/resource/Robin_Hood_Daffy, http://dbpedia.org/resource/Santiago_(film), http://dbpedia.org/resource/Scrap_Happy_Daffy, http://dbpedia.org/resource/See_Ya_Later_Gladiator, http://dbpedia.org/resource/Show_Biz_Bugs, http://dbpedia.org/resource/The_Brave_One_(2007_film), http://dbpedia.org/resource/The_Dark_Tower_(1943_film), http://dbpedia.org/resource/The_Million_Hare, http://dbpedia.org/resource/The_Night_of_the_Living_Duck, http://dbpedia.org/resource/The_Rutles_2:_Can't_Buy_Me_Lunch, http://dbpedia.org/resource/The_Scarlet_Pumpernickel, http://dbpedia.org/resource/The_Thief_Who_Came_to_Dinner, http://dbpedia.org/resource/The_Wearing_of_the_Grin, http://dbpedia.org/resource/They_Drive_by_Night_(1938_film), http://dbpedia.org/resource/Thumb_Fun, http://dbpedia.org/resource/Tick_Tock_Tuckered, http://dbpedia.org/resource/Too_Dangerous_to_Live, http://dbpedia.org/resource/Well_Worn_Daffy, http://dbpedia.org/resource/Who_Is_Killing_the_Great_Chefs_of_Europe%3F, http://dbpedia.org/resource/Wholly_Smoke, http://dbpedia.org/resource/You_Ought_to_Be_in_Pictures, http://dbpedia.org/resource/Zoom_and_Bored, http://dbpedia.org/resource/Bombers_B-52, http://dbpedia.org/resource/Capricorn_One, http://dbpedia.org/resource/Everybody's_All-American_(film), http://dbpedia.org/resource/Everything's_Jake, http://dbpedia.org/resource/Fire_Down_Below_(1997_film), http://dbpedia.org/resource/Green_Light_(1937_film), http://dbpedia.org/resource/Hard_to_Kill, http://dbpedia.org/resource/Honeysuckle_Rose_(film), http://dbpedia.org/resource/Invisible_Stripes, http://dbpedia.org/resource/Jump_into_Hell, http://dbpedia.org/resource/Men_Don't_Leave, http://dbpedia.org/resource/Montana_(1950_film), http://dbpedia.org/resource/Operation_Thunderbolt_(film), http://dbpedia.org/resource/Silver_River_(film), http://dbpedia.org/resource/The_Animal_World_(film), http://dbpedia.org/resource/The_Baby_Maker, http://dbpedia.org/resource/The_Black_Scorpion_(film), http://dbpedia.org/resource/The_Glimmer_Man, http://dbpedia.org/resource/The_Perfect_Specimen, http://dbpedia.org/resource/The_Story_of_Will_Rogers, http://dbpedia.org/resource/Three_Secrets, http://dbpedia.org/resource/Tom_Horn_(film), http://dbpedia.org/resource/Too_Much,_Too_Soon, http://dbpedia.org/resource/Uncertain_Glory, http://dbpedia.org/resource/Beautiful_Creatures_(2013_film), http://dbpedia.org/resource/Final_Destination_5, http://dbpedia.org/resource/The_Gallows, http://dbpedia.org/resource/Just_Tell_Me_What_You_Want, http://dbpedia.org/resource/Nights_in_Rodanthe, http://dbpedia.org/resource/The_Nesting, http://dbpedia.org/resource/Wild_America_(film), http://dbpedia.org/resource/Danger_Signal, http://dbpedia.org/resource/Don't_Bet_on_Blondes, http://dbpedia.org/resource/Going_Highbrow, http://dbpedia.org/resource/I_Am_a_Thief, http://dbpedia.org/resource/See_Spot_Run, http://dbpedia.org/resource/Smarty_(film), http://dbpedia.org/resource/The_Conjuring, http://dbpedia.org/resource/The_Payoff_(1935_film), http://dbpedia.org/resource/The_Road_to_Singapore, http://dbpedia.org/resource/The_Wagons_Roll_at_Night, http://dbpedia.org/resource/Big_Boy_(film), http://dbpedia.org/resource/Bright_Lights_(1935_film), http://dbpedia.org/resource/Cowboy_from_Brooklyn, http://dbpedia.org/resource/Devil's_Island_(1939_film), http://dbpedia.org/resource/Going_Places_(1938_film), http://dbpedia.org/resource/It's_Alive_(1974_film), http://dbpedia.org/resource/Oh_Sailor_Behave, http://dbpedia.org/resource/Taking_Lives_(film), http://dbpedia.org/resource/The_Cool_Ones, http://dbpedia.org/resource/The_Legend_of_Tarzan_(film), http://dbpedia.org/resource/The_Singing_Cop_(film), http://dbpedia.org/resource/The_Singing_Marine, http://dbpedia.org/resource/Goodbye_Lover, http://dbpedia.org/resource/High_Road_to_China, http://dbpedia.org/resource/Mr._Wonderful_(film), http://dbpedia.org/resource/Must_Love_Dogs, http://dbpedia.org/resource/Oriundi_(film), http://dbpedia.org/resource/Painting_the_Clouds_with_Sunshine_(film), http://dbpedia.org/resource/Palmetto_(film), http://dbpedia.org/resource/Something_to_Believe_In_(film), http://dbpedia.org/resource/Sunchaser, http://dbpedia.org/resource/Tarzan_and_the_Lost_City_(film), http://dbpedia.org/resource/The_Noble_Family, http://dbpedia.org/resource/Up_Periscope, http://dbpedia.org/resource/A_Majority_of_One_(film), http://dbpedia.org/resource/Hyde_and_Hare, http://dbpedia.org/resource/Mexican_Boarders, http://dbpedia.org/resource/The_Windblown_Hare, http://dbpedia.org/resource/Three_Little_Bops, http://dbpedia.org/resource/Billboard_Frolics, http://dbpedia.org/resource/A_Day_at_the_Zoo, http://dbpedia.org/resource/Henpecked_Hoboes, http://dbpedia.org/resource/Toy_Town_Hall, http://dbpedia.org/resource/Breakthrough_(1950_film), http://dbpedia.org/resource/Two_Guys_from_Milwaukee, http://dbpedia.org/resource/Shadows_on_the_Stairs, http://dbpedia.org/resource/The_President's_Mistress, http://dbpedia.org/resource/The_Lazarus_Child, http://dbpedia.org/resource/The_Sky_Crawlers_(film), http://dbpedia.org/resource/Turtle_Beach_(film), http://dbpedia.org/resource/Whiskey_School, http://dbpedia.org/resource/Zweiohrküken, http://dbpedia.org/resource/Dying_God, http://dbpedia.org/resource/Gentleman_Jim_(film), http://dbpedia.org/resource/Chaos_Theory_(film), http://dbpedia.org/resource/Incognito_(1998_film), http://dbpedia.org/resource/Lost_River_(film), http://dbpedia.org/resource/Holiday_for_Shoestrings, http://dbpedia.org/resource/Blue_Collar_Comedy_Tour:_The_Movie, http://dbpedia.org/resource/L'Auberge_rouge_(film), http://dbpedia.org/resource/The_Ex-Wife_of_My_Life, http://dbpedia.org/resource/The_Blue_Squadron_(1934_film), http://dbpedia.org/resource/The_Viper_(film), http://dbpedia.org/resource/Wide_Open_(film), http://dbpedia.org/resource/Burning_Annie, http://dbpedia.org/resource/Quacker_Tracker, http://dbpedia.org/resource/The_Spy_Swatter, http://dbpedia.org/resource/Good_Morning,_Eve!, http://dbpedia.org/resource/Early_to_Bet, http://dbpedia.org/resource/Torrente_4:_Lethal_Crisis, http://dbpedia.org/resource/The_Mouse_That_Jack_Built, http://dbpedia.org/resource/Sir_or_Madam, http://dbpedia.org/resource/Demon_of_the_Sea, http://dbpedia.org/resource/Adventurous_Youth, http://dbpedia.org/resource/Tengo_Ganas_de_Ti, http://dbpedia.org/resource/Agneepath_(1990_film), http://dbpedia.org/resource/Batman_&_Robin_(film), http://dbpedia.org/resource/Christmas_in_Connecticut, http://dbpedia.org/resource/Clash_of_the_Titans_(2010_film), http://dbpedia.org/resource/Collateral_Damage_(film), http://dbpedia.org/resource/Demolition_Man_(film), http://dbpedia.org/resource/Disorderlies, http://dbpedia.org/resource/Eraser_(film), http://dbpedia.org/resource/Ladyhawke, http://dbpedia.org/resource/Porky's_Duck_Hunt, http://dbpedia.org/resource/Terminator_3:_Rise_of_the_Machines, http://dbpedia.org/resource/Terminator_Salvation, http://dbpedia.org/resource/The_Goonies, http://dbpedia.org/resource/The_Song_Remains_the_Same_(film), http://dbpedia.org/resource/Wrath_of_the_Titans, http://dbpedia.org/resource/A_Mighty_Wind, http://dbpedia.org/resource/A_Perfect_World, http://dbpedia.org/resource/Abbott_and_Costello_Meet_Captain_Kidd, http://dbpedia.org/resource/After_Hours_(film), http://dbpedia.org/resource/Almost_Heroes, http://dbpedia.org/resource/American_Idiot, http://dbpedia.org/resource/Be_with_Me, http://dbpedia.org/resource/Best_in_Show_(film), http://dbpedia.org/resource/Big_Bully_(film), http://dbpedia.org/resource/Bird_(film), http://dbpedia.org/resource/Charlie_and_the_Chocolate_Factory_(film), http://dbpedia.org/resource/Cobb_(film), http://dbpedia.org/resource/Cracking_Up_(film), http://dbpedia.org/resource/Dog_Day_Afternoon, http://dbpedia.org/resource/Duck_Amuck, http://dbpedia.org/resource/Elena_and_Her_Men, http://dbpedia.org/resource/Eyes_Wide_Shut, http://dbpedia.org/resource/Flags_of_Our_Fathers_(film), http://dbpedia.org/resource/For_Scent-imental_Reasons, http://dbpedia.org/resource/Gay_Purr-ee, http://dbpedia.org/resource/Girlfriends_(1978_film), http://dbpedia.org/resource/Goodfellas, http://dbpedia.org/resource/Gran_Torino, http://dbpedia.org/resource/Gremlins, http://dbpedia.org/resource/Gremlins_2:_The_New_Batch, http://dbpedia.org/resource/Happy_Feet, http://dbpedia.org/resource/Honkytonk_Man, http://dbpedia.org/resource/If_Looks_Could_Kill_(film), http://dbpedia.org/resource/Inherent_Vice_(film), http://dbpedia.org/resource/Innocent_Blood_(film), http://dbpedia.org/resource/Joe_Versus_the_Volcano, http://dbpedia.org/resource/Just_Cause_(film), http://dbpedia.org/resource/Lady_in_the_Water, http://dbpedia.org/resource/Legend_of_the_Guardians:_The_Owls_of_Ga'Hoole, http://dbpedia.org/resource/Little_Shop_of_Horrors_(film), http://dbpedia.org/resource/Love_Don't_Cost_a_Thing_(film), http://dbpedia.org/resource/M._Butterfly_(film), http://dbpedia.org/resource/Man_of_Steel_(film), http://dbpedia.org/resource/Merrie_Melodies, http://dbpedia.org/resource/My_Blue_Heaven_(1990_film), http://dbpedia.org/resource/New_Year's_Eve_(film), http://dbpedia.org/resource/Once_Upon_a_Time_in_America, http://dbpedia.org/resource/One_Froggy_Evening, http://dbpedia.org/resource/Penn_&_Teller_Get_Killed, http://dbpedia.org/resource/Pink_Cadillac_(film), http://dbpedia.org/resource/Protocol_(film), http://dbpedia.org/resource/Quest_for_Camelot, http://dbpedia.org/resource/Rebel_Without_a_Cause, http://dbpedia.org/resource/Space_Cowboys, http://dbpedia.org/resource/Stay_Tuned_(film), http://dbpedia.org/resource/Superman_II:_The_Richard_Donner_Cut, http://dbpedia.org/resource/Superman_III, http://dbpedia.org/resource/Superman_Returns, http://dbpedia.org/resource/The_Accidental_Tourist_(film), http://dbpedia.org/resource/The_Bridges_of_Madison_County_(film), http://dbpedia.org/resource/The_Crush_(1993_film), http://dbpedia.org/resource/The_Departed, http://dbpedia.org/resource/The_Fugitive_(1993_film), http://dbpedia.org/resource/The_Hobbit_(film_series), http://dbpedia.org/resource/The_Iceman_Ducketh, http://dbpedia.org/resource/The_Invention_of_Lying, http://dbpedia.org/resource/The_Island_(2005_film), http://dbpedia.org/resource/The_Perfect_Storm_(film), http://dbpedia.org/resource/The_Weakly_Reporter, http://dbpedia.org/resource/Tom_and_Jerry, http://dbpedia.org/resource/Transcendence_(2014_film), http://dbpedia.org/resource/True_Crime_(1999_film), http://dbpedia.org/resource/Twister_(1996_film), http://dbpedia.org/resource/Wild_Wild_West, http://dbpedia.org/resource/Zodiac_(film), http://dbpedia.org/resource/Baseball_Bugs, http://dbpedia.org/resource/City_of_Joy_(film), http://dbpedia.org/resource/Disclosure_(film), http://dbpedia.org/resource/Frantic_(film), http://dbpedia.org/resource/Lethal_Weapon_2, http://dbpedia.org/resource/Lethal_Weapon_3, http://dbpedia.org/resource/Live_in_Hyde_Park_(Eric_Clapton_album), http://dbpedia.org/resource/Love_Affair_(1994_film), http://dbpedia.org/resource/Mean_Streets, http://dbpedia.org/resource/Michael_Collins_(film), http://dbpedia.org/resource/Personal_Best_(film), http://dbpedia.org/resource/Round_Midnight_(film), http://dbpedia.org/resource/San_Andreas_(film), http://dbpedia.org/resource/So_Fine_(film), http://dbpedia.org/resource/The_Book_of_Eli, http://dbpedia.org/resource/The_Five_Man_Army, http://dbpedia.org/resource/The_Time_Machine_(2002_film), http://dbpedia.org/resource/Without_Limits, http://dbpedia.org/resource/Wrestling_Ernest_Hemingway, http://dbpedia.org/resource/Never_Apologize, http://dbpedia.org/resource/O_Lucky_Man!, http://dbpedia.org/resource/Seven_Men_from_Now, http://dbpedia.org/resource/Class_of_'44, http://dbpedia.org/resource/Cloak_and_Dagger_(1946_film), http://dbpedia.org/resource/Crazy,_Stupid,_Love, http://dbpedia.org/resource/Dr._Ehrlich's_Magic_Bullet, http://dbpedia.org/resource/Othello_(1965_film), http://dbpedia.org/resource/Summer_of_'42, http://dbpedia.org/resource/That_Hagen_Girl, http://dbpedia.org/resource/Operation_Daybreak, http://dbpedia.org/resource/Stupor_Duck, http://dbpedia.org/resource/Super-Rabbit, http://dbpedia.org/resource/The_Sisterhood_of_the_Traveling_Pants_(film), http://dbpedia.org/resource/Back_Alley_Oproar, http://dbpedia.org/resource/Heartbreak_Ridge, http://dbpedia.org/resource/Hitler_Lives, http://dbpedia.org/resource/Miracle_in_the_Rain, http://dbpedia.org/resource/Star_in_the_Night, http://dbpedia.org/resource/The_Verdict_(1946_film), http://dbpedia.org/resource/Up_the_Down_Staircase_(film)]... [INFO] Predicted Answer: [http://dbpedia.org/resource/Warner_Bros., http://dbpedia.org/resource/The_CW] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 733, P: 0.503, R: 0.555, macro F1: 0.510, macro F1*: 0.528 -[INFO] Current time: 1031 ms, average: 2869.78 ms/ques, entity detection: 477.50 ms/ques (linking tool: 384.38 ms/ques), relation detection: 927.44 ms/ques (similarity: 703.26 ms/ques), EDG generation: 46.59 ms/ques +[INFO] Cumulative metrics, sample: 733, P: 0.503, R: 0.555, macro F1: 0.528 +[INFO] Current question consumed time: 835 ms, average: 2586.37 ms/ques, entity detection: 422.14 ms/ques (linking tool: 343.49 ms/ques), relation detection: 855.83 ms/ques (similarity: 694.94 ms/ques), EDG generation: 46.51 ms/ques [INFO] Entity Linking P: 0.7469304229195088, R: 0.699181446111869, micro F1: 0.7144156434743075, macro F1: 0.7222676259363174 [INFO] Relation Linking P: 0.6001591632560254, R: 0.5450204638472033, micro F1: 0.5607548885857215, macro F1: 0.5712623902808271 [INFO] Type Linking P: 0.5457025920873124, R: 0.5457025920873124, micro F1: 0.5457025920873124, macro F1: 0.5457025920873124 @@ -10293,8 +10293,8 @@ Question 733: Name the common associate band of Blayse and Teddy Riley ? [INFO] Golden Answer: [http://dbpedia.org/resource/Blackstreet] [INFO] Predicted Answer: [http://dbpedia.org/resource/Blackstreet] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 734, P: 0.504, R: 0.555, macro F1: 0.510, macro F1*: 0.528 -[INFO] Current time: 1477 ms, average: 2867.88 ms/ques, entity detection: 477.55 ms/ques (linking tool: 384.33 ms/ques), relation detection: 926.69 ms/ques (similarity: 702.64 ms/ques), EDG generation: 46.58 ms/ques +[INFO] Cumulative metrics, sample: 734, P: 0.504, R: 0.555, macro F1: 0.528 +[INFO] Current question consumed time: 1250 ms, average: 2584.55 ms/ques, entity detection: 422.14 ms/ques (linking tool: 343.40 ms/ques), relation detection: 855.11 ms/ques (similarity: 694.31 ms/ques), EDG generation: 46.51 ms/ques [INFO] Entity Linking P: 0.747275204359673, R: 0.6995912806539509, micro F1: 0.7148047229791109, macro F1: 0.7226474904683466 [INFO] Relation Linking P: 0.6007039055404177, R: 0.5456403269754768, micro F1: 0.5613533151680298, macro F1: 0.5718496523773888 [INFO] Type Linking P: 0.5463215258855586, R: 0.5463215258855586, micro F1: 0.5463215258855586, macro F1: 0.5463215258855586 @@ -10307,8 +10307,8 @@ Question 734: What did Steven T Mason's father die from? [INFO] Golden Answer: [http://dbpedia.org/resource/Cholera] [INFO] Predicted Answer: [New York City, New York@en] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 735, P: 0.503, R: 0.554, macro F1: 0.510, macro F1*: 0.528 -[INFO] Current time: 1234 ms, average: 2865.66 ms/ques, entity detection: 477.38 ms/ques (linking tool: 384.22 ms/ques), relation detection: 925.83 ms/ques (similarity: 701.92 ms/ques), EDG generation: 46.54 ms/ques +[INFO] Cumulative metrics, sample: 735, P: 0.503, R: 0.554, macro F1: 0.528 +[INFO] Current question consumed time: 978 ms, average: 2582.36 ms/ques, entity detection: 421.93 ms/ques (linking tool: 343.25 ms/ques), relation detection: 854.31 ms/ques (similarity: 693.59 ms/ques), EDG generation: 46.46 ms/ques [INFO] Entity Linking P: 0.7476190476190476, R: 0.7, micro F1: 0.7151927437641734, macro F1: 0.7230263157894736 [INFO] Relation Linking P: 0.5998866213151927, R: 0.5448979591836735, micro F1: 0.5605895691609984, macro F1: 0.5710716256394605 [INFO] Type Linking P: 0.5469387755102041, R: 0.5469387755102041, micro F1: 0.5469387755102041, macro F1: 0.5469387755102041 @@ -10321,8 +10321,8 @@ Question 735: Whose military units are VMF-218 and VMF-155? [INFO] Golden Answer: [http://dbpedia.org/resource/John_Glenn] [INFO] Predicted Answer: [http://dbpedia.org/resource/World_War_II] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 736, P: 0.503, R: 0.554, macro F1: 0.509, macro F1*: 0.527 -[INFO] Current time: 1499 ms, average: 2863.80 ms/ques, entity detection: 477.25 ms/ques (linking tool: 383.94 ms/ques), relation detection: 924.98 ms/ques (similarity: 701.21 ms/ques), EDG generation: 46.52 ms/ques +[INFO] Cumulative metrics, sample: 736, P: 0.503, R: 0.554, macro F1: 0.527 +[INFO] Current question consumed time: 1195 ms, average: 2580.48 ms/ques, entity detection: 421.74 ms/ques (linking tool: 342.96 ms/ques), relation detection: 853.51 ms/ques (similarity: 692.89 ms/ques), EDG generation: 46.44 ms/ques [INFO] Entity Linking P: 0.7479619565217391, R: 0.7004076086956522, micro F1: 0.7155797101449286, macro F1: 0.7234041061668978 [INFO] Relation Linking P: 0.5990715579710144, R: 0.5441576086956522, micro F1: 0.5598278985507253, macro F1: 0.5702957131046242 [INFO] Type Linking P: 0.5475543478260869, R: 0.5475543478260869, micro F1: 0.5475543478260869, macro F1: 0.5475543478260869 @@ -10335,8 +10335,8 @@ Question 736: What automobile engine's successor is BMW M40 [INFO] Golden Answer: [http://dbpedia.org/resource/BMW_M10] [INFO] Predicted Answer: [http://dbpedia.org/resource/BMW_M10, http://dbpedia.org/resource/BMW_M43] [INFO] P: 0.5, R: 1.0, F: 0.6666666666666666 -[INFO] Cumulative metrics, sample: 737, P: 0.503, R: 0.554, macro F1: 0.509, macro F1*: 0.527 -[INFO] Current time: 1001 ms, average: 2861.28 ms/ques, entity detection: 477.03 ms/ques (linking tool: 383.76 ms/ques), relation detection: 923.98 ms/ques (similarity: 700.50 ms/ques), EDG generation: 46.47 ms/ques +[INFO] Cumulative metrics, sample: 737, P: 0.503, R: 0.554, macro F1: 0.527 +[INFO] Current question consumed time: 862 ms, average: 2578.15 ms/ques, entity detection: 421.53 ms/ques (linking tool: 342.79 ms/ques), relation detection: 852.59 ms/ques (similarity: 692.20 ms/ques), EDG generation: 46.39 ms/ques [INFO] Entity Linking P: 0.748303934871099, R: 0.7008141112618724, micro F1: 0.7159656264133887, macro F1: 0.7237808658444244 [INFO] Relation Linking P: 0.5996155585707824, R: 0.5447761194029851, micro F1: 0.5604251469923119, macro F1: 0.5708818814729829 [INFO] Type Linking P: 0.5468113975576662, R: 0.5468113975576662, micro F1: 0.5468113975576662, macro F1: 0.5468113975576662 @@ -10349,8 +10349,8 @@ Question 737: Name the company which produces both Boeing F/A-18 E/F Hornet nad [INFO] Golden Answer: [http://dbpedia.org/resource/Boeing] [INFO] Predicted Answer: [http://dbpedia.org/resource/McDonnell_Douglas, http://dbpedia.org/resource/Boeing_Defense,_Space_&_Security] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 738, P: 0.502, R: 0.554, macro F1: 0.509, macro F1*: 0.526 -[INFO] Current time: 4493 ms, average: 2863.49 ms/ques, entity detection: 479.05 ms/ques (linking tool: 385.88 ms/ques), relation detection: 924.03 ms/ques (similarity: 700.94 ms/ques), EDG generation: 46.43 ms/ques +[INFO] Cumulative metrics, sample: 738, P: 0.502, R: 0.554, macro F1: 0.526 +[INFO] Current question consumed time: 4001 ms, average: 2580.07 ms/ques, entity detection: 423.38 ms/ques (linking tool: 344.72 ms/ques), relation detection: 852.60 ms/ques (similarity: 692.64 ms/ques), EDG generation: 46.35 ms/ques [INFO] Entity Linking P: 0.747289972899729, R: 0.6998644986449865, micro F1: 0.7149954832881672, macro F1: 0.7228001329638765 [INFO] Relation Linking P: 0.5988030713640469, R: 0.5440379403794038, micro F1: 0.5596657633243006, macro F1: 0.5701083287880601 [INFO] Type Linking P: 0.5460704607046071, R: 0.5460704607046071, micro F1: 0.5460704607046071, macro F1: 0.5460704607046071 @@ -10363,8 +10363,8 @@ Question 738: What is the district of Methodist Central Hall Westminster ? [INFO] Golden Answer: [http://dbpedia.org/resource/London] [INFO] Predicted Answer: [http://dbpedia.org/resource/London, London@en] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 739, P: 0.503, R: 0.554, macro F1: 0.509, macro F1*: 0.527 -[INFO] Current time: 1165 ms, average: 2861.19 ms/ques, entity detection: 478.83 ms/ques (linking tool: 385.69 ms/ques), relation detection: 923.00 ms/ques (similarity: 700.12 ms/ques), EDG generation: 46.38 ms/ques +[INFO] Cumulative metrics, sample: 739, P: 0.503, R: 0.554, macro F1: 0.527 +[INFO] Current question consumed time: 938 ms, average: 2577.85 ms/ques, entity detection: 423.13 ms/ques (linking tool: 344.51 ms/ques), relation detection: 851.64 ms/ques (similarity: 691.83 ms/ques), EDG generation: 46.30 ms/ques [INFO] Entity Linking P: 0.7476319350473613, R: 0.7002706359945873, micro F1: 0.7153811456923781, macro F1: 0.7231766848355317 [INFO] Relation Linking P: 0.5993459630130807, R: 0.5446549391069012, micro F1: 0.5602616147947684, macro F1: 0.5706931496014155 [INFO] Type Linking P: 0.5466847090663058, R: 0.5466847090663058, micro F1: 0.5466847090663058, macro F1: 0.5466847090663058 @@ -10377,8 +10377,8 @@ Question 739: Who develops the computation platform of Microsoft Expression Enco [INFO] Golden Answer: [http://dbpedia.org/resource/Apple_Inc., http://dbpedia.org/resource/Microsoft] [INFO] Predicted Answer: [http://dbpedia.org/resource/Apple_Inc., http://dbpedia.org/resource/Microsoft, Apple Inc.@en, AviSynth developers@en] [INFO] P: 0.6666666666666666, R: 1.0, F: 0.8 -[INFO] Cumulative metrics, sample: 740, P: 0.503, R: 0.555, macro F1: 0.510, macro F1*: 0.528 -[INFO] Current time: 1653 ms, average: 2859.56 ms/ques, entity detection: 478.59 ms/ques (linking tool: 385.49 ms/ques), relation detection: 922.46 ms/ques (similarity: 699.53 ms/ques), EDG generation: 46.42 ms/ques +[INFO] Cumulative metrics, sample: 740, P: 0.503, R: 0.555, macro F1: 0.528 +[INFO] Current question consumed time: 1449 ms, average: 2576.33 ms/ques, entity detection: 422.91 ms/ques (linking tool: 344.32 ms/ques), relation detection: 851.11 ms/ques (similarity: 691.24 ms/ques), EDG generation: 46.41 ms/ques [INFO] Entity Linking P: 0.7479729729729729, R: 0.7006756756756757, micro F1: 0.7157657657657668, macro F1: 0.7235522135941913 [INFO] Relation Linking P: 0.5998873873873873, R: 0.5452702702702703, micro F1: 0.5608558558558566, macro F1: 0.5712763752049818 [INFO] Type Linking P: 0.5472972972972973, R: 0.5472972972972973, micro F1: 0.5472972972972973, macro F1: 0.5472972972972973 @@ -10391,8 +10391,8 @@ Question 740: What is the official language of Ladonia, which is the target area [INFO] Golden Answer: [http://dbpedia.org/resource/Latin] [INFO] Predicted Answer: [http://dbpedia.org/resource/Probus,_Cornwall] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 741, P: 0.502, R: 0.554, macro F1: 0.509, macro F1*: 0.527 -[INFO] Current time: 2758 ms, average: 2859.42 ms/ques, entity detection: 478.27 ms/ques (linking tool: 385.13 ms/ques), relation detection: 922.59 ms/ques (similarity: 699.33 ms/ques), EDG generation: 46.42 ms/ques +[INFO] Cumulative metrics, sample: 741, P: 0.502, R: 0.554, macro F1: 0.527 +[INFO] Current question consumed time: 2309 ms, average: 2575.97 ms/ques, entity detection: 422.60 ms/ques (linking tool: 344.00 ms/ques), relation detection: 851.18 ms/ques (similarity: 691.06 ms/ques), EDG generation: 46.41 ms/ques [INFO] Entity Linking P: 0.7469635627530364, R: 0.6997300944669366, micro F1: 0.7147998200629789, macro F1: 0.7225757598646444 [INFO] Relation Linking P: 0.5990778227620333, R: 0.5445344129554656, micro F1: 0.560098965362124, macro F1: 0.5705054219321005 [INFO] Type Linking P: 0.5479082321187584, R: 0.5479082321187584, micro F1: 0.5479082321187584, macro F1: 0.5479082321187584 @@ -10405,8 +10405,8 @@ Question 741: Where did the conflict take place where Louis Briere de I'Isle was [INFO] Golden Answer: [http://dbpedia.org/resource/Annam_(French_protectorate), http://dbpedia.org/resource/Tonkin] [INFO] Predicted Answer: [] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 742, P: 0.501, R: 0.553, macro F1: 0.508, macro F1*: 0.526 -[INFO] Current time: 2579 ms, average: 2859.04 ms/ques, entity detection: 478.21 ms/ques (linking tool: 385.07 ms/ques), relation detection: 922.47 ms/ques (similarity: 699.30 ms/ques), EDG generation: 46.40 ms/ques +[INFO] Cumulative metrics, sample: 742, P: 0.501, R: 0.553, macro F1: 0.526 +[INFO] Current question consumed time: 2123 ms, average: 2575.36 ms/ques, entity detection: 422.49 ms/ques (linking tool: 343.90 ms/ques), relation detection: 851.05 ms/ques (similarity: 691.03 ms/ques), EDG generation: 46.39 ms/ques [INFO] Entity Linking P: 0.7459568733153639, R: 0.6987870619946092, micro F1: 0.7138364779874224, macro F1: 0.7216019380858512 [INFO] Relation Linking P: 0.5982704402515723, R: 0.543800539083558, micro F1: 0.5593441150044931, macro F1: 0.5697365467003861 [INFO] Type Linking P: 0.5471698113207547, R: 0.5471698113207547, micro F1: 0.5471698113207547, macro F1: 0.5471698113207547 @@ -10419,8 +10419,8 @@ Question 742: In which team was the person drafted who has studied at the Worces [INFO] Golden Answer: [http://dbpedia.org/resource/New_York_Knicks] [INFO] Predicted Answer: [http://dbpedia.org/resource/Out4Marriage] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 743, P: 0.501, R: 0.553, macro F1: 0.508, macro F1*: 0.525 -[INFO] Current time: 7186 ms, average: 2864.87 ms/ques, entity detection: 478.16 ms/ques (linking tool: 385.07 ms/ques), relation detection: 925.96 ms/ques (similarity: 701.68 ms/ques), EDG generation: 46.38 ms/ques +[INFO] Cumulative metrics, sample: 743, P: 0.501, R: 0.553, macro F1: 0.525 +[INFO] Current question consumed time: 6116 ms, average: 2580.12 ms/ques, entity detection: 422.41 ms/ques (linking tool: 343.86 ms/ques), relation detection: 854.12 ms/ques (similarity: 693.41 ms/ques), EDG generation: 46.37 ms/ques [INFO] Entity Linking P: 0.7449528936742934, R: 0.6978465679676985, micro F1: 0.7128757290264703, macro F1: 0.7206307376308231 [INFO] Relation Linking P: 0.598138178555406, R: 0.5437415881561238, micro F1: 0.559264244055631, macro F1: 0.5696442175889616 [INFO] Type Linking P: 0.5464333781965006, R: 0.5464333781965006, micro F1: 0.5464333781965006, macro F1: 0.5464333781965006 @@ -10433,8 +10433,8 @@ Question 743: How many people hold important positions alongwith Donna Shalala? [INFO] Golden Answer: [8] [INFO] Predicted Answer: [8] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 744, P: 0.501, R: 0.553, macro F1: 0.508, macro F1*: 0.526 -[INFO] Current time: 3113 ms, average: 2865.20 ms/ques, entity detection: 477.67 ms/ques (linking tool: 384.69 ms/ques), relation detection: 925.71 ms/ques (similarity: 701.76 ms/ques), EDG generation: 46.34 ms/ques +[INFO] Cumulative metrics, sample: 744, P: 0.501, R: 0.553, macro F1: 0.526 +[INFO] Current question consumed time: 2649 ms, average: 2580.21 ms/ques, entity detection: 421.96 ms/ques (linking tool: 343.51 ms/ques), relation detection: 853.93 ms/ques (similarity: 693.49 ms/ques), EDG generation: 46.33 ms/ques [INFO] Entity Linking P: 0.7452956989247311, R: 0.698252688172043, micro F1: 0.7132616487455208, macro F1: 0.7210076640435714 [INFO] Relation Linking P: 0.5986783154121863, R: 0.5443548387096774, micro F1: 0.5598566308243735, macro F1: 0.5702256958164087 [INFO] Type Linking P: 0.5470430107526881, R: 0.5470430107526881, micro F1: 0.5470430107526881, macro F1: 0.5470430107526881 @@ -10447,8 +10447,8 @@ Question 744: What other things are provided by the producer of Office 365? [INFO] Golden Answer: [http://dbpedia.org/resource/Microsoft_Office, http://dbpedia.org/resource/Office_365, http://dbpedia.org/resource/Microsoft_Exchange_Server, http://dbpedia.org/resource/Skype, http://dbpedia.org/resource/Microsoft_Azure, http://dbpedia.org/resource/Web_hosting_service, http://dbpedia.org/resource/OneDrive, http://dbpedia.org/resource/Data_center_services, *@en] [INFO] Predicted Answer: [] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 745, P: 0.501, R: 0.552, macro F1: 0.508, macro F1*: 0.525 -[INFO] Current time: 1823 ms, average: 2863.80 ms/ques, entity detection: 477.26 ms/ques (linking tool: 384.31 ms/ques), relation detection: 925.41 ms/ques (similarity: 701.52 ms/ques), EDG generation: 46.39 ms/ques +[INFO] Cumulative metrics, sample: 745, P: 0.501, R: 0.552, macro F1: 0.525 +[INFO] Current question consumed time: 1568 ms, average: 2578.86 ms/ques, entity detection: 421.59 ms/ques (linking tool: 343.17 ms/ques), relation detection: 853.62 ms/ques (similarity: 693.24 ms/ques), EDG generation: 46.39 ms/ques [INFO] Entity Linking P: 0.7442953020134229, R: 0.6973154362416107, micro F1: 0.7123042505592851, macro F1: 0.7200398685213653 [INFO] Relation Linking P: 0.5978747203579418, R: 0.5436241610738255, micro F1: 0.559105145413871, macro F1: 0.5694602921978632 [INFO] Type Linking P: 0.5476510067114094, R: 0.5476510067114094, micro F1: 0.5476510067114094, macro F1: 0.5476510067114094 @@ -10461,8 +10461,8 @@ Question 745: Who owns the company that made the Edsel Villager? [INFO] Golden Answer: [http://dbpedia.org/resource/Ford_Motor_Company] [INFO] Predicted Answer: [http://dbpedia.org/resource/Edsel_Bermuda, http://dbpedia.org/resource/Edsel_Citation, http://dbpedia.org/resource/Edsel_Corsair, http://dbpedia.org/resource/Edsel_Pacer, http://dbpedia.org/resource/Edsel_Ranger, http://dbpedia.org/resource/Edsel_Roundup, http://dbpedia.org/resource/Edsel_Villager, http://dbpedia.org/resource/Ford_Motor_Company, Edsel Division of Ford Motor Company@en] [INFO] P: 0.1111111111111111, R: 1.0, F: 0.19999999999999998 -[INFO] Cumulative metrics, sample: 746, P: 0.500, R: 0.553, macro F1: 0.507, macro F1*: 0.525 -[INFO] Current time: 1348 ms, average: 2861.77 ms/ques, entity detection: 476.89 ms/ques (linking tool: 383.94 ms/ques), relation detection: 924.73 ms/ques (similarity: 701.02 ms/ques), EDG generation: 46.35 ms/ques +[INFO] Cumulative metrics, sample: 746, P: 0.500, R: 0.553, macro F1: 0.525 +[INFO] Current question consumed time: 1160 ms, average: 2576.95 ms/ques, entity detection: 421.23 ms/ques (linking tool: 342.82 ms/ques), relation detection: 852.98 ms/ques (similarity: 692.75 ms/ques), EDG generation: 46.34 ms/ques [INFO] Entity Linking P: 0.7446380697050938, R: 0.6977211796246648, micro F1: 0.7126899016979457, macro F1: 0.7204165711552069 [INFO] Relation Linking P: 0.5970732797140303, R: 0.5428954423592494, micro F1: 0.5583556747095628, macro F1: 0.5686969405997429 [INFO] Type Linking P: 0.546916890080429, R: 0.546916890080429, micro F1: 0.546916890080429, macro F1: 0.546916890080429 @@ -10475,8 +10475,8 @@ Question 746: Which collaborator of Nice and Smooth wrote Same Song ? [INFO] Golden Answer: [http://dbpedia.org/resource/Tupac_Shakur] [INFO] Predicted Answer: [http://dbpedia.org/resource/Money-B, http://dbpedia.org/resource/Shock_G, http://dbpedia.org/resource/Tupac_Shakur] [INFO] P: 0.3333333333333333, R: 1.0, F: 0.5 -[INFO] Cumulative metrics, sample: 747, P: 0.500, R: 0.554, macro F1: 0.507, macro F1*: 0.525 -[INFO] Current time: 1428 ms, average: 2859.85 ms/ques, entity detection: 476.79 ms/ques (linking tool: 383.82 ms/ques), relation detection: 923.96 ms/ques (similarity: 700.37 ms/ques), EDG generation: 46.31 ms/ques +[INFO] Cumulative metrics, sample: 747, P: 0.500, R: 0.554, macro F1: 0.525 +[INFO] Current question consumed time: 1192 ms, average: 2575.10 ms/ques, entity detection: 421.11 ms/ques (linking tool: 342.68 ms/ques), relation detection: 852.26 ms/ques (similarity: 692.10 ms/ques), EDG generation: 46.30 ms/ques [INFO] Entity Linking P: 0.7449799196787149, R: 0.6974564926372155, micro F1: 0.7126282909415449, macro F1: 0.7204353376382561 [INFO] Relation Linking P: 0.5976126729138777, R: 0.5428380187416332, micro F1: 0.5585006693440436, macro F1: 0.5689099611462249 [INFO] Type Linking P: 0.5475234270414994, R: 0.5475234270414994, micro F1: 0.5475234270414994, macro F1: 0.5475234270414994 @@ -10489,8 +10489,8 @@ Question 747: Who fought in the battle at Tupelo Mississippi? [INFO] Golden Answer: [http://dbpedia.org/resource/Chickasaw, http://dbpedia.org/resource/Louisiana_(New_France), Confederate States@en, United States@en] [INFO] Predicted Answer: [] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 748, P: 0.499, R: 0.553, macro F1: 0.506, macro F1*: 0.525 -[INFO] Current time: 4805 ms, average: 2862.45 ms/ques, entity detection: 476.45 ms/ques (linking tool: 383.53 ms/ques), relation detection: 925.81 ms/ques (similarity: 700.12 ms/ques), EDG generation: 46.35 ms/ques +[INFO] Cumulative metrics, sample: 748, P: 0.499, R: 0.553, macro F1: 0.525 +[INFO] Current question consumed time: 4085 ms, average: 2577.12 ms/ques, entity detection: 420.78 ms/ques (linking tool: 342.41 ms/ques), relation detection: 853.74 ms/ques (similarity: 691.85 ms/ques), EDG generation: 46.35 ms/ques [INFO] Entity Linking P: 0.7439839572192514, R: 0.696524064171123, micro F1: 0.7116755793226391, macro F1: 0.7194721887911462 [INFO] Relation Linking P: 0.5968137254901961, R: 0.5421122994652406, micro F1: 0.5577540106951879, macro F1: 0.5681493863318584 [INFO] Type Linking P: 0.5467914438502673, R: 0.5467914438502673, micro F1: 0.5467914438502673, macro F1: 0.5467914438502673 @@ -10503,8 +10503,8 @@ Question 748: List the movies whose cast member were born in Scotland. [INFO] Golden Answer: [http://dbpedia.org/resource/Wide_Open_Town, http://dbpedia.org/resource/The_Real_McCoys, http://dbpedia.org/resource/Pirates_on_Horseback, http://dbpedia.org/resource/Plainsman_and_the_Lady, http://dbpedia.org/resource/Straight_from_the_Shoulder_(film), http://dbpedia.org/resource/McFadden's_Flats_(1935_film), http://dbpedia.org/resource/Lost_Canyon, http://dbpedia.org/resource/Bar_20, http://dbpedia.org/resource/Border_Patrol_(film), http://dbpedia.org/resource/Colt_Comrades, http://dbpedia.org/resource/Forty_Thieves_(film), http://dbpedia.org/resource/Borrowed_Trouble, http://dbpedia.org/resource/Dangerous_Venture, http://dbpedia.org/resource/False_Colors, http://dbpedia.org/resource/Fool's_Gold_(1947_film), http://dbpedia.org/resource/Hoppy_Serves_a_Writ, http://dbpedia.org/resource/Mystery_Man_(film), http://dbpedia.org/resource/Outlaws_of_the_Desert, http://dbpedia.org/resource/Riders_of_the_Deadline, http://dbpedia.org/resource/Secret_of_the_Wasteland, http://dbpedia.org/resource/Silent_Conflict, http://dbpedia.org/resource/Sinister_Journey, http://dbpedia.org/resource/Strange_Gamble, http://dbpedia.org/resource/Texas_Masquerade, http://dbpedia.org/resource/The_Dead_Don't_Dream, http://dbpedia.org/resource/The_Devil's_Playground_(1946_film), http://dbpedia.org/resource/The_Leather_Burners, http://dbpedia.org/resource/Three_Men_from_Texas, http://dbpedia.org/resource/Million_Dollar_Legs_(1932_film), http://dbpedia.org/resource/Hoppy's_Holiday, http://dbpedia.org/resource/Twilight_on_the_Trail, http://dbpedia.org/resource/In_Old_Colorado, http://dbpedia.org/resource/The_Marauders_(1947_film), http://dbpedia.org/resource/Cherokee_Strip_(film), http://dbpedia.org/resource/Stick_to_Your_Guns_(film), http://dbpedia.org/resource/Border_Vigilantes, http://dbpedia.org/resource/Riders_of_the_Timberline, http://dbpedia.org/resource/The_Road_to_Denver, http://dbpedia.org/resource/Timeline_(film), http://dbpedia.org/resource/Attila_(miniseries), http://dbpedia.org/resource/The_Hunt_for_Tony_Blair, http://dbpedia.org/resource/Machine_Gun_Preacher, http://dbpedia.org/resource/Shooters_(2002_film), http://dbpedia.org/resource/Wrath_of_Gods, http://dbpedia.org/resource/Happy_Hollidays, http://dbpedia.org/resource/How_to_Train_Your_Dragon_(film), http://dbpedia.org/resource/The_Phantom_of_the_Opera_(2004_film), http://dbpedia.org/resource/Sir_Billi, http://dbpedia.org/resource/The_Game_of_Their_Lives_(2005_film), http://dbpedia.org/resource/Butterfly_on_a_Wheel, http://dbpedia.org/resource/Nim's_Island, http://dbpedia.org/resource/The_Bounty_Hunter_(2010_film), http://dbpedia.org/resource/Dracula_2000, http://dbpedia.org/resource/The_Ugly_Truth, http://dbpedia.org/resource/Chewin'_the_Fat, http://dbpedia.org/resource/Fast_Food_(1999_film), http://dbpedia.org/resource/Gift_of_the_Night_Fury, http://dbpedia.org/resource/One_More_Kiss_(film), http://dbpedia.org/resource/Beowulf_&_Grendel, http://dbpedia.org/resource/Movie_43, http://dbpedia.org/resource/Chasing_Mavericks, http://dbpedia.org/resource/Geostorm, http://dbpedia.org/resource/Gods_of_Egypt_(film), http://dbpedia.org/resource/Legend_of_the_Boneknapper_Dragon, http://dbpedia.org/resource/Lara_Croft_Tomb_Raider:_The_Cradle_of_Life, http://dbpedia.org/resource/300_(film), http://dbpedia.org/resource/Reign_of_Fire_(film), http://dbpedia.org/resource/Playing_for_Keeps_(2012_film), http://dbpedia.org/resource/RocknRolla, http://dbpedia.org/resource/The_Headhunter's_Calling, http://dbpedia.org/resource/Jewel_of_the_Sahara, http://dbpedia.org/resource/Dear_Frankie, http://dbpedia.org/resource/Mrs_Brown, http://dbpedia.org/resource/Gamer_(film), http://dbpedia.org/resource/By_the_Sea_(1982_film), http://dbpedia.org/resource/Father_Came_Too!, http://dbpedia.org/resource/The_Picnic, http://dbpedia.org/resource/Now_Look_Here, http://dbpedia.org/resource/Crooks_Anonymous, http://dbpedia.org/resource/Fierce_Creatures, http://dbpedia.org/resource/Very_Important_Person_(film), http://dbpedia.org/resource/No_Sex_Please,_We're_British_(film), http://dbpedia.org/resource/Joey_Boy_(film), http://dbpedia.org/resource/The_Fast_Lady, http://dbpedia.org/resource/A_Quiet_Passion, http://dbpedia.org/resource/Three_Crooked_Men, http://dbpedia.org/resource/Blind_Spot_(1958_film), http://dbpedia.org/resource/Two_Wives_at_One_Wedding, http://dbpedia.org/resource/Millions_Like_Us, http://dbpedia.org/resource/Whisky_Galore!_(film), http://dbpedia.org/resource/Eureka_Stockade_(1949_film), http://dbpedia.org/resource/Upstairs,_Downstairs_(1971_TV_series), http://dbpedia.org/resource/Golden_Rendezvous, http://dbpedia.org/resource/Yesterday's_Enemy, http://dbpedia.org/resource/The_Whistle_Blower, http://dbpedia.org/resource/Danger_Route, http://dbpedia.org/resource/Floodtide, http://dbpedia.org/resource/Windfall_(1955_film), http://dbpedia.org/resource/Shaka_Zulu_(TV_series), http://dbpedia.org/resource/The_Shooting_Party, http://dbpedia.org/resource/Against_the_Wind_(film), http://dbpedia.org/resource/Stop_Press_Girl, http://dbpedia.org/resource/Nine_Men_(film), http://dbpedia.org/resource/A_Town_Like_Alice_(miniseries), http://dbpedia.org/resource/Sailor_Beware!_(1956_film), http://dbpedia.org/resource/Bitter_Springs_(film), http://dbpedia.org/resource/Pink_String_and_Sealing_Wax, http://dbpedia.org/resource/The_Foreman_Went_to_France, http://dbpedia.org/resource/The_Masks_of_Death, http://dbpedia.org/resource/Colour_Blind_(2009_film), http://dbpedia.org/resource/Seven_Wonders_of_the_Industrial_World, http://dbpedia.org/resource/Wuthering_Heights_(1998_film), http://dbpedia.org/resource/Time_Lock, http://dbpedia.org/resource/Gorgo_(film), http://dbpedia.org/resource/The_Little_Kidnappers_(1953_film), http://dbpedia.org/resource/The_Best_House_in_London, http://dbpedia.org/resource/The_Corn_Is_Green_(1979_film), http://dbpedia.org/resource/The_Army_Game, http://dbpedia.org/resource/I've_Gotta_Horse, http://dbpedia.org/resource/Love_and_Death_on_Long_Island, http://dbpedia.org/resource/That's_Your_Funeral, http://dbpedia.org/resource/August_(1996_film), http://dbpedia.org/resource/The_Man_Who_Liked_Funerals, http://dbpedia.org/resource/Up_the_Front, http://dbpedia.org/resource/Captain_Nemo_and_the_Underwater_City, http://dbpedia.org/resource/Dangerous_Moonlight, http://dbpedia.org/resource/The_Great_Game_(1953_film), http://dbpedia.org/resource/The_Edge_of_the_World, http://dbpedia.org/resource/Dad's_Army, http://dbpedia.org/resource/The_Ghost_of_St._Michael's, http://dbpedia.org/resource/Siege_of_the_Saxons, http://dbpedia.org/resource/Dad's_Army_(1971_film), http://dbpedia.org/resource/The_Brothers_(1947_film), http://dbpedia.org/resource/Fanny_by_Gaslight_(film), http://dbpedia.org/resource/Space:_1999, http://dbpedia.org/resource/Chocky_(TV_series), http://dbpedia.org/resource/Cashback_(film), http://dbpedia.org/resource/The_Winter_Guest, http://dbpedia.org/resource/Hippie_Hippie_Shake, http://dbpedia.org/resource/Consenting_Adults_(2007_film), http://dbpedia.org/resource/Shang-a-Lang_(TV_series), http://dbpedia.org/resource/A_Taste_of_Money, http://dbpedia.org/resource/Anna_the_Adventuress, http://dbpedia.org/resource/Fires_of_Fate_(1932_film), http://dbpedia.org/resource/Waking_the_Dead_(TV_series), http://dbpedia.org/resource/1001_Inventions_and_the_World_of_Ibn_Al-Haytham, http://dbpedia.org/resource/The_Square_(2013_film), http://dbpedia.org/resource/Unforgotten, http://dbpedia.org/resource/Bad_Day_(film), http://dbpedia.org/resource/Tigers_(film), http://dbpedia.org/resource/The_Kite_Runner_(film), http://dbpedia.org/resource/The_Lash_(1930_film), http://dbpedia.org/resource/Remodeling_Her_Husband, http://dbpedia.org/resource/His_Children's_Children, http://dbpedia.org/resource/Neverwhere, http://dbpedia.org/resource/Titus_(film), http://dbpedia.org/resource/Whatever_Happened_to_Harold_Smith%3F, http://dbpedia.org/resource/Virtual_Sexuality, http://dbpedia.org/resource/Small_Faces_(film), http://dbpedia.org/resource/Den_of_Lions, http://dbpedia.org/resource/Divorcing_Jack_(film), http://dbpedia.org/resource/The_Boys_Are_Back_(film), http://dbpedia.org/resource/Flutter_(2011_film), http://dbpedia.org/resource/Left_Luggage_(film), http://dbpedia.org/resource/Devil's_Gate_(film), http://dbpedia.org/resource/House_of_9, http://dbpedia.org/resource/I_Am_Not_a_Serial_Killer_(film), http://dbpedia.org/resource/Back_Stage_(1923_film), http://dbpedia.org/resource/Giants_vs._Yanks, http://dbpedia.org/resource/16_Years_of_Alcohol, http://dbpedia.org/resource/Cuckoo_(2009_film), http://dbpedia.org/resource/The_Sinking_of_the_Laconia, http://dbpedia.org/resource/The_Match_(film), http://dbpedia.org/resource/One_Terrible_Day, http://dbpedia.org/resource/Forgive_and_Forget_(2000_film), http://dbpedia.org/resource/Lilting_(film), http://dbpedia.org/resource/Nina's_Heavenly_Delights, http://dbpedia.org/resource/A_Quiet_Street, http://dbpedia.org/resource/July_Days_(film), http://dbpedia.org/resource/Saturday_Morning_(1922_film), http://dbpedia.org/resource/Stage_Fright_(1923_film), http://dbpedia.org/resource/The_Flying_Scotsman_(2006_film), http://dbpedia.org/resource/Labyrinth_(miniseries), http://dbpedia.org/resource/Red_Road_(film), http://dbpedia.org/resource/The_League_of_Extraordinary_Gentlemen_(film), http://dbpedia.org/resource/Underworld:_Evolution, http://dbpedia.org/resource/Don't_Talk_(film), http://dbpedia.org/resource/Heroes_and_Villains_(TV_series), http://dbpedia.org/resource/The_Lazarus_Project, http://dbpedia.org/resource/The_Groundsman, http://dbpedia.org/resource/Shuttle_(film), http://dbpedia.org/resource/The_Ghost_and_the_Darkness, http://dbpedia.org/resource/Flight_of_the_Phoenix_(2004_film), http://dbpedia.org/resource/The_Presence_(film), http://dbpedia.org/resource/Doors_Open_(film), http://dbpedia.org/resource/Funny_Man_(film), http://dbpedia.org/resource/Kidnapped_(1995_film), http://dbpedia.org/resource/Cat_Run, http://dbpedia.org/resource/In_the_Dark_Half, http://dbpedia.org/resource/Ned_Kelly_(1970_film), http://dbpedia.org/resource/2000_Weeks, http://dbpedia.org/resource/Dead_Set, http://dbpedia.org/resource/Morvern_Callar_(film), http://dbpedia.org/resource/The_Steamie, http://dbpedia.org/resource/Venus_Peter, http://dbpedia.org/resource/Cross_(2011_film), http://dbpedia.org/resource/Nightworld_(2016_film), http://dbpedia.org/resource/Notes_(2013_film), http://dbpedia.org/resource/The_Black_Adder_(pilot_episode), http://dbpedia.org/resource/The_Girl_in_a_Swing_(film), http://dbpedia.org/resource/Australian_Made:_The_Movie, http://dbpedia.org/resource/The_Last_Stand_(1984_film), http://dbpedia.org/resource/Rockpalast_(film), http://dbpedia.org/resource/Fakin'_da_Funk, http://dbpedia.org/resource/Ride_(1998_film), http://dbpedia.org/resource/I_Got_the_Hook_Up, http://dbpedia.org/resource/Rome_(TV_series), http://dbpedia.org/resource/Trainspotting_(film), http://dbpedia.org/resource/Toy_Story_That_Time_Forgot, http://dbpedia.org/resource/AfterLife_(film), http://dbpedia.org/resource/The_Last_Legion, http://dbpedia.org/resource/Brave_(2012_film), http://dbpedia.org/resource/Justice_League:_The_Flashpoint_Paradox, http://dbpedia.org/resource/Gunpowder,_Treason_&_Plot, http://dbpedia.org/resource/Bunraku_(film), http://dbpedia.org/resource/Home_Sweet_Hell, http://dbpedia.org/resource/The_Rocket_Post, http://dbpedia.org/resource/Comes_a_Bright_Day, http://dbpedia.org/resource/North_Square, http://dbpedia.org/resource/Bedrooms_and_Hallways, http://dbpedia.org/resource/Dad_Savage, http://dbpedia.org/resource/Dog_Soldiers_(film), http://dbpedia.org/resource/Dragonheart, http://dbpedia.org/resource/You_Only_Live_Twice_(film), http://dbpedia.org/resource/Wrong_Is_Right, http://dbpedia.org/resource/Five_Days_One_Summer, http://dbpedia.org/resource/Sword_of_the_Valiant, http://dbpedia.org/resource/The_Man_Who_Would_Be_King_(film), http://dbpedia.org/resource/The_Untouchables_(film), http://dbpedia.org/resource/Diamonds_Are_Forever_(film), http://dbpedia.org/resource/Time_Bandits, http://dbpedia.org/resource/Rising_Sun_(film), http://dbpedia.org/resource/The_First_Great_Train_Robbery, http://dbpedia.org/resource/Action_of_the_Tiger, http://dbpedia.org/resource/Another_Time,_Another_Place_(1958_film), http://dbpedia.org/resource/First_Knight, http://dbpedia.org/resource/No_Road_Back, http://dbpedia.org/resource/Outland_(film), http://dbpedia.org/resource/The_Hill_(film), http://dbpedia.org/resource/The_Molly_Maguires_(film), http://dbpedia.org/resource/A_Good_Man_in_Africa, http://dbpedia.org/resource/A_Time_to_Kill_(1955_film), http://dbpedia.org/resource/Cuba_(film), http://dbpedia.org/resource/Just_Cause_(film), http://dbpedia.org/resource/From_Russia_with_Love_(film), http://dbpedia.org/resource/Medicine_Man_(film), http://dbpedia.org/resource/Macbeth_(1961_film), http://dbpedia.org/resource/A_Bridge_Too_Far_(film), http://dbpedia.org/resource/Indiana_Jones_and_the_Last_Crusade, http://dbpedia.org/resource/The_Frightened_City, http://dbpedia.org/resource/Little_Red_Monkey, http://dbpedia.org/resource/Tarzan's_Greatest_Adventure, http://dbpedia.org/resource/Ransom_(1974_film), http://dbpedia.org/resource/Finding_Forrester, http://dbpedia.org/resource/Never_Say_Never_Again, http://dbpedia.org/resource/Shalako_(film), http://dbpedia.org/resource/Thunderball_(film), http://dbpedia.org/resource/Robin_and_Marian, http://dbpedia.org/resource/The_Name_of_the_Rose_(film), http://dbpedia.org/resource/The_Rock_(film), http://dbpedia.org/resource/The_Avengers_(1998_film), http://dbpedia.org/resource/Sleeping_Car_to_Trieste, http://dbpedia.org/resource/The_Next_Man, http://dbpedia.org/resource/The_Presidio_(film), http://dbpedia.org/resource/Torment_(1950_British_film), http://dbpedia.org/resource/The_Twenty_Questions_Murder_Mystery, http://dbpedia.org/resource/Dr._No_(film), http://dbpedia.org/resource/Darby_O'Gill_and_the_Little_People, http://dbpedia.org/resource/Highlander_(film), http://dbpedia.org/resource/Highlander_II:_The_Quickening, http://dbpedia.org/resource/Entrapment_(film), http://dbpedia.org/resource/The_Hunt_for_Red_October_(film), http://dbpedia.org/resource/The_Russia_House_(film), http://dbpedia.org/resource/Playing_by_Heart, http://dbpedia.org/resource/Zardoz, http://dbpedia.org/resource/Family_Business_(film), http://dbpedia.org/resource/A_Fine_Madness, http://dbpedia.org/resource/The_Anderson_Tapes, http://dbpedia.org/resource/Woman_of_Straw, http://dbpedia.org/resource/The_Red_Tent_(film), http://dbpedia.org/resource/Home_to_Danger, http://dbpedia.org/resource/The_Black_Rider_(film), http://dbpedia.org/resource/The_Offence, http://dbpedia.org/resource/Poet's_Pub, http://dbpedia.org/resource/Black_13, http://dbpedia.org/resource/Meteor_(film), http://dbpedia.org/resource/Her_Favourite_Husband, http://dbpedia.org/resource/On_the_Fiddle, http://dbpedia.org/resource/Man_with_a_Gun, http://dbpedia.org/resource/Whispering_Smith_Hits_London, http://dbpedia.org/resource/We_Are_the_People_We've_Been_Waiting_For, http://dbpedia.org/resource/Our_New_Errand_Boy, http://dbpedia.org/resource/Stella_Does_Tricks, http://dbpedia.org/resource/Strictly_Sinatra, http://dbpedia.org/resource/Anna_Karenina_(2012_film), http://dbpedia.org/resource/Gosford_Park, http://dbpedia.org/resource/Two_Family_House, http://dbpedia.org/resource/Intermission_(film), http://dbpedia.org/resource/Swallows_and_Amazons_(film), http://dbpedia.org/resource/Entropy_(film), http://dbpedia.org/resource/My_Life_So_Far, http://dbpedia.org/resource/In_the_Electric_Mist, http://dbpedia.org/resource/Some_Voices_(film), http://dbpedia.org/resource/State_of_Play_(TV_serial), http://dbpedia.org/resource/The_Merry_Gentleman, http://dbpedia.org/resource/The_Girl_in_the_Café, http://dbpedia.org/resource/Special_Correspondents, http://dbpedia.org/resource/House!, http://dbpedia.org/resource/The_Decoy_Bride, http://dbpedia.org/resource/Splendor_(1999_film), http://dbpedia.org/resource/All_the_Invisible_Children, http://dbpedia.org/resource/Skellig_(film), http://dbpedia.org/resource/Carry_On_Columbus, http://dbpedia.org/resource/Gregory's_Girl, http://dbpedia.org/resource/Comfort_and_Joy_(1984_film), http://dbpedia.org/resource/Hammer_the_Toff, http://dbpedia.org/resource/The_Dancing_Years_(film), http://dbpedia.org/resource/At_the_Stroke_of_Nine, http://dbpedia.org/resource/The_House_in_Marsh_Road, http://dbpedia.org/resource/Affairs_of_Geraldine, http://dbpedia.org/resource/The_Three_Wise_Guys, http://dbpedia.org/resource/Six_Cylinder_Love, http://dbpedia.org/resource/Blonde_Inspiration, http://dbpedia.org/resource/The_Return_of_Peter_Grimm, http://dbpedia.org/resource/Sky_Murder, http://dbpedia.org/resource/The_Ghost_Comes_Home, http://dbpedia.org/resource/The_Wild_Man_of_Borneo_(film), http://dbpedia.org/resource/The_Girl_Habit, http://dbpedia.org/resource/The_Fabulous_Joe, http://dbpedia.org/resource/Brandy_for_the_Parson, http://dbpedia.org/resource/Lust_for_Life_(film), http://dbpedia.org/resource/The_Pickwick_Papers_(1952_film), http://dbpedia.org/resource/Cast_a_Giant_Shadow, http://dbpedia.org/resource/The_Great_Escape_(film), http://dbpedia.org/resource/The_Net_(1953_film), http://dbpedia.org/resource/The_Bridge_on_the_River_Kwai, http://dbpedia.org/resource/The_Vikings_(1958_film), http://dbpedia.org/resource/The_Citadel_(1960_American_miniseries), http://dbpedia.org/resource/Trottie_True, http://dbpedia.org/resource/Gift_Horse_(film), http://dbpedia.org/resource/Cage_of_Gold, http://dbpedia.org/resource/Broken_Journey, http://dbpedia.org/resource/Quatermass_and_the_Pit_(film), http://dbpedia.org/resource/The_Birthday_Party_(film), http://dbpedia.org/resource/Not_Another_Happy_Ending, http://dbpedia.org/resource/Blood_Monkey, http://dbpedia.org/resource/Desperate_Romantics, http://dbpedia.org/resource/Law_of_the_Pampas, http://dbpedia.org/resource/Bar_20_Justice, http://dbpedia.org/resource/Three_on_the_Trail_(film), http://dbpedia.org/resource/Pumpkinhead:_Blood_Feud, http://dbpedia.org/resource/The_Frontiersmen, http://dbpedia.org/resource/Wolves_of_the_North, http://dbpedia.org/resource/Nevada_(1935_film), http://dbpedia.org/resource/The_Fast_Express, http://dbpedia.org/resource/The_Steel_Trail, http://dbpedia.org/resource/Smashing_Barriers, http://dbpedia.org/resource/The_Fighting_Trail, http://dbpedia.org/resource/The_Cuban_Love_Song, http://dbpedia.org/resource/Mantrap_(1926_film), http://dbpedia.org/resource/Speedway_(1929_film), http://dbpedia.org/resource/Night_Life_of_New_York, http://dbpedia.org/resource/The_King_of_Kings_(1927_film), http://dbpedia.org/resource/Tol'able_David, http://dbpedia.org/resource/The_Blind_Goddess, http://dbpedia.org/resource/Broken_Chains_(film), http://dbpedia.org/resource/I_Cover_the_Waterfront, http://dbpedia.org/resource/Desert_Nights, http://dbpedia.org/resource/The_Side_Show_of_Life, http://dbpedia.org/resource/The_Heritage_of_the_Desert_(film), http://dbpedia.org/resource/The_Lady_of_the_Harem, http://dbpedia.org/resource/The_Rainmaker_(1926_film), http://dbpedia.org/resource/Fighting_Caravans, http://dbpedia.org/resource/The_Unholy_Night, http://dbpedia.org/resource/Ruggles_of_Red_Gap_(1923_film), http://dbpedia.org/resource/Shipmates_(film), http://dbpedia.org/resource/Peter_Pan_(1924_film), http://dbpedia.org/resource/Across_to_Singapore, http://dbpedia.org/resource/Untamed_(1929_film), http://dbpedia.org/resource/Twelve_Miles_Out, http://dbpedia.org/resource/The_Fighting_Coward, http://dbpedia.org/resource/Strictly_Unconventional, http://dbpedia.org/resource/Sweet_Kitty_Bellairs, http://dbpedia.org/resource/Sporting_Blood, http://dbpedia.org/resource/North_of_36, http://dbpedia.org/resource/West_of_the_Water_Tower, http://dbpedia.org/resource/The_Best_Years_(film), http://dbpedia.org/resource/Down_Where_The_Buffalo_Go, http://dbpedia.org/resource/Tangled_Lives, http://dbpedia.org/resource/Under_the_Red_Robe_(1923_film), http://dbpedia.org/resource/A_Wife's_Sacrifice, http://dbpedia.org/resource/Magic_Island_(film), http://dbpedia.org/resource/The_Concrete_Jungle_(film), http://dbpedia.org/resource/Summer_City, http://dbpedia.org/resource/The_Beautiful_Spy, http://dbpedia.org/resource/Romeo.Juliet, http://dbpedia.org/resource/Balamory, http://dbpedia.org/resource/Irvine_Welsh's_Ecstasy, http://dbpedia.org/resource/The_Shepherd:_Border_Patrol, http://dbpedia.org/resource/True_North_(film), http://dbpedia.org/resource/Inside_I'm_Dancing, http://dbpedia.org/resource/The_Comedian_(2012_film), http://dbpedia.org/resource/A_Little_Bit_of_Fluff_(1919_film), http://dbpedia.org/resource/Four_Sons, http://dbpedia.org/resource/Once_Upon_a_Time_(1918_film), http://dbpedia.org/resource/I_Will_(film), http://dbpedia.org/resource/Grave_Tales, http://dbpedia.org/resource/Forest_of_the_Damned, http://dbpedia.org/resource/Forest_of_the_Damned_2, http://dbpedia.org/resource/Sunset_Song_(film), http://dbpedia.org/resource/The_Angels'_Share, http://dbpedia.org/resource/The_Wee_Man, http://dbpedia.org/resource/Your_Witness_(film), http://dbpedia.org/resource/There_Was_a_Crooked_Man_(film), http://dbpedia.org/resource/The_Idol_of_Paris, http://dbpedia.org/resource/Auld_Lang_Syne_(1937_film), http://dbpedia.org/resource/Space_Milkshake, http://dbpedia.org/resource/Dorothy_and_the_Witches_of_Oz, http://dbpedia.org/resource/Master_and_Commander:_The_Far_Side_of_the_World, http://dbpedia.org/resource/Stone_of_Destiny_(film), http://dbpedia.org/resource/The_Lord_of_the_Rings_(film_series), http://dbpedia.org/resource/Seed_of_Chucky, http://dbpedia.org/resource/Moby_Dick_(2011_miniseries), http://dbpedia.org/resource/Chucky_(franchise), http://dbpedia.org/resource/Stories_of_Lost_Souls, http://dbpedia.org/resource/The_Forger_(2012_film), http://dbpedia.org/resource/Porridge_(film), http://dbpedia.org/resource/Defence_of_the_Realm, http://dbpedia.org/resource/Britannia_Hospital, http://dbpedia.org/resource/Local_Hero, http://dbpedia.org/resource/If_You_Go_Down_in_the_Woods_Today, http://dbpedia.org/resource/300:_Rise_of_an_Empire, http://dbpedia.org/resource/Bathory_(film), http://dbpedia.org/resource/I_Am_Dina, http://dbpedia.org/resource/Doctor_Zhivago_(miniseries), http://dbpedia.org/resource/The_Tudors, http://dbpedia.org/resource/The_Christmas_Candle, http://dbpedia.org/resource/Half_Light_(film), http://dbpedia.org/resource/Nero_(2004_film), http://dbpedia.org/resource/Tess_of_the_D'Urbervilles_(TV_serial), http://dbpedia.org/resource/Paris_(1929_film), http://dbpedia.org/resource/Goodnight,_Vienna, http://dbpedia.org/resource/The_Third_Round, http://dbpedia.org/resource/Monte_Carlo_(1930_film), http://dbpedia.org/resource/A_Man_of_Mayfair, http://dbpedia.org/resource/Her_Heritage, http://dbpedia.org/resource/Yes,_Mr_Brown, http://dbpedia.org/resource/Josephine_and_Men, http://dbpedia.org/resource/As_Long_as_They're_Happy, http://dbpedia.org/resource/Confetti_(1927_film), http://dbpedia.org/resource/The_Audacious_Mr._Squire, http://dbpedia.org/resource/The_Middle_Watch_(1940_film), http://dbpedia.org/resource/Toni_(1928_film), http://dbpedia.org/resource/The_French,_They_Are_a_Funny_Race, http://dbpedia.org/resource/Bulldog_Sees_It_Through, http://dbpedia.org/resource/Sliding_Doors, http://dbpedia.org/resource/The_Mummy_Returns, http://dbpedia.org/resource/The_Mummy_(1999_film), http://dbpedia.org/resource/Pandaemonium_(film), http://dbpedia.org/resource/Madagascar_Skin, http://dbpedia.org/resource/Before_You_Go_(film), http://dbpedia.org/resource/Spartacus:_Gods_of_the_Arena, http://dbpedia.org/resource/Resurrection_Man_(film), http://dbpedia.org/resource/Four_Weddings_and_a_Funeral, http://dbpedia.org/resource/The_Mummy:_Tomb_of_the_Dragon_Emperor, http://dbpedia.org/resource/Spartacus_(TV_series), http://dbpedia.org/resource/Another_Mother's_Son, http://dbpedia.org/resource/Bone_in_the_Throat, http://dbpedia.org/resource/Harbour_Beat, http://dbpedia.org/resource/I_Accuse, http://dbpedia.org/resource/Kidnap_and_Ransom, http://dbpedia.org/resource/Ping_Pong_Summer, http://dbpedia.org/resource/Sinking_of_the_Lusitania:_Terror_at_Sea, http://dbpedia.org/resource/The_Innocent_Sleep, http://dbpedia.org/resource/Zip_'n_Zoo, http://dbpedia.org/resource/The_Love_Bug_(1997_film), http://dbpedia.org/resource/Circus_(2000_film), http://dbpedia.org/resource/The_Intruder_(1999_film), http://dbpedia.org/resource/I'm_with_Lucy, http://dbpedia.org/resource/Ghost_Son, http://dbpedia.org/resource/The_Big_Man, http://dbpedia.org/resource/Salvage_(2009_film), http://dbpedia.org/resource/Bodies_(TV_series), http://dbpedia.org/resource/Gypsy_Woman_(film), http://dbpedia.org/resource/Niceland_(Population._1.000.002), http://dbpedia.org/resource/Wild_Country_(2005_film), http://dbpedia.org/resource/Mrs_Caldicot's_Cabbage_War, http://dbpedia.org/resource/Big_Fat_Gypsy_Gangster, http://dbpedia.org/resource/The_Thick_of_It, http://dbpedia.org/resource/The_Devil's_Whore, http://dbpedia.org/resource/In_the_Loop_(film), http://dbpedia.org/resource/Modigliani_(film), http://dbpedia.org/resource/The_Lair_of_the_White_Worm_(film), http://dbpedia.org/resource/The_Greatest_Store_in_the_World, http://dbpedia.org/resource/Solid_Geometry_(film), http://dbpedia.org/resource/Unconditional_Love_(TV_drama), http://dbpedia.org/resource/The_Suspicions_of_Mr_Whicher, http://dbpedia.org/resource/The_Hour_(BBC_TV_series), http://dbpedia.org/resource/G.I._Joe_(film_series), http://dbpedia.org/resource/G.I._Joe:_The_Rise_of_Cobra, http://dbpedia.org/resource/G.I._Joe:_Retaliation, http://dbpedia.org/resource/Jinn_(film), http://dbpedia.org/resource/Ballistic:_Ecks_vs._Sever, http://dbpedia.org/resource/The_Last_Witch_Hunter, http://dbpedia.org/resource/Downton_Abbey, http://dbpedia.org/resource/Sticky_Notes_(film), http://dbpedia.org/resource/Honeymoon_(2014_film), http://dbpedia.org/resource/Jeevan_Dhaara, http://dbpedia.org/resource/Shakka_(film), http://dbpedia.org/resource/Pyaar_Ke_Do_Pal, http://dbpedia.org/resource/Anurodh, http://dbpedia.org/resource/Rehguzar, http://dbpedia.org/resource/Calendar_Girls, http://dbpedia.org/resource/Old_Harry's_Game, http://dbpedia.org/resource/A_Picture_of_Katherine_Mansfield, http://dbpedia.org/resource/Sky_West_and_Crooked, http://dbpedia.org/resource/Wyrd_Sisters_(TV_series), http://dbpedia.org/resource/The_Slipper_and_the_Rose, http://dbpedia.org/resource/Hope_Springs_(TV_series), http://dbpedia.org/resource/Company_Man_(film), http://dbpedia.org/resource/Josie_and_the_Pussycats_(film), http://dbpedia.org/resource/Bam_Bam_and_Celeste, http://dbpedia.org/resource/Hurricane_Bianca, http://dbpedia.org/resource/Rick_&_Steve:_The_Happiest_Gay_Couple_in_All_the_World, http://dbpedia.org/resource/Son_of_the_Mask, http://dbpedia.org/resource/The_Smurfs:_The_Legend_of_Smurfy_Hollow, http://dbpedia.org/resource/Circle_of_Friends_(1995_film), http://dbpedia.org/resource/Jackboots_on_Whitehall, http://dbpedia.org/resource/Maladies_(film), http://dbpedia.org/resource/Emma_(1996_theatrical_film), http://dbpedia.org/resource/Annie_(1999_film), http://dbpedia.org/resource/Buddy_(1997_film), http://dbpedia.org/resource/Get_Carter_(2000_film), http://dbpedia.org/resource/The_Flintstones_in_Viva_Rock_Vegas, http://dbpedia.org/resource/The_Flintstones_(film_series), http://dbpedia.org/resource/Strange_Magic_(film), http://dbpedia.org/resource/The_Smurfs_(film), http://dbpedia.org/resource/The_High_Life_(TV_series), http://dbpedia.org/resource/Plunkett_&_Macleane, http://dbpedia.org/resource/Spy_Kids, http://dbpedia.org/resource/Burlesque_(2010_American_film), http://dbpedia.org/resource/Sweet_Land, http://dbpedia.org/resource/X2_(film), http://dbpedia.org/resource/ShowBusiness:_The_Road_to_Broadway, http://dbpedia.org/resource/Nicholas_Nickleby_(2002_film), http://dbpedia.org/resource/The_Outback_(2012_film), http://dbpedia.org/resource/Reefer_Madness_(2005_film), http://dbpedia.org/resource/The_Touch_(1971_film), http://dbpedia.org/resource/Boogie_Woogie_(film), http://dbpedia.org/resource/Tin_Man_(miniseries), http://dbpedia.org/resource/Black_Beauty_(1994_film), http://dbpedia.org/resource/The_Tempest_(2010_film), http://dbpedia.org/resource/Gray_Matters, http://dbpedia.org/resource/Ripley_Under_Ground_(film), http://dbpedia.org/resource/Any_Day_Now_(2012_film), http://dbpedia.org/resource/The_Anniversary_Party, http://dbpedia.org/resource/Urbania_(film), http://dbpedia.org/resource/Second_Best_(film), http://dbpedia.org/resource/Suffering_Man's_Charity, http://dbpedia.org/resource/Eighteen_(film), http://dbpedia.org/resource/Investigating_Sex, http://dbpedia.org/resource/Bernard_and_the_Genie, http://dbpedia.org/resource/Sir_Henry_at_Rawlinson_End_(film), http://dbpedia.org/resource/Full_Grown_Men, http://dbpedia.org/resource/Prague_(1992_film), http://dbpedia.org/resource/Containment_(film), http://dbpedia.org/resource/Terminator:_The_Sarah_Connor_Chronicles, http://dbpedia.org/resource/Margaret_(2009_film), http://dbpedia.org/resource/The_Reflecting_Skin, http://dbpedia.org/resource/Richard_II_(2012_film), http://dbpedia.org/resource/Under_the_Tuscan_Sun_(film), http://dbpedia.org/resource/Burlesque_Fairytales, http://dbpedia.org/resource/Dead_Head_(TV_series), http://dbpedia.org/resource/Loose_Connections, http://dbpedia.org/resource/Grown-Ups, http://dbpedia.org/resource/Frankenstein_(2007_film), http://dbpedia.org/resource/Shooting_the_Past, http://dbpedia.org/resource/Longford_(film), http://dbpedia.org/resource/Le_Week-End, http://dbpedia.org/resource/Gifted_(film), http://dbpedia.org/resource/Manifesto_(1988_film), http://dbpedia.org/resource/Teen_Beach_Movie, http://dbpedia.org/resource/Wizards_of_Waverly_Place:_The_Movie, http://dbpedia.org/resource/Dementamania, http://dbpedia.org/resource/The_Midnight_Man_(2016_film), http://dbpedia.org/resource/Don't_Look_Under_the_Bed, http://dbpedia.org/resource/Disraeli_(1929_film), http://dbpedia.org/resource/The_Prisoner_of_Zenda_(1913_film), http://dbpedia.org/resource/The_Black_Watch, http://dbpedia.org/resource/Horse_Play, http://dbpedia.org/resource/Scotland_Yard_(1930_film), http://dbpedia.org/resource/The_Mysterious_Rider_(1927_film), http://dbpedia.org/resource/Wolfshead:_The_Legend_of_Robin_Hood, http://dbpedia.org/resource/Dead_Ringers_(comedy), http://dbpedia.org/resource/The_Inside_of_the_Cup_(film), http://dbpedia.org/resource/Headcases, http://dbpedia.org/resource/2DTV, http://dbpedia.org/resource/Tiger_Love_(1924_film), http://dbpedia.org/resource/The_Light_That_Failed_(1923_film), http://dbpedia.org/resource/Men_of_Yesterday, http://dbpedia.org/resource/Spellbound_(1941_film), http://dbpedia.org/resource/The_Old_Curiosity_Shop_(1934_film), http://dbpedia.org/resource/Keepers_of_Youth, http://dbpedia.org/resource/Rush_Hour_(1941_film), http://dbpedia.org/resource/Suspense_(1930_film), http://dbpedia.org/resource/Inquest_(1939_film), http://dbpedia.org/resource/Lord_Edgware_Dies_(film), http://dbpedia.org/resource/Song_of_the_Plough, http://dbpedia.org/resource/The_Curse_of_the_Werewolf, http://dbpedia.org/resource/The_Masked_Marvel, http://dbpedia.org/resource/The_Haunted_Strangler, http://dbpedia.org/resource/Death_Rides_a_Horse, http://dbpedia.org/resource/O.K._Connery, http://dbpedia.org/resource/Battle_of_Neretva_(film), http://dbpedia.org/resource/Dangers_of_the_Canadian_Mounted, http://dbpedia.org/resource/G-Men_Never_Forget, http://dbpedia.org/resource/Deadlock_(1970_film), http://dbpedia.org/resource/The_Last_Shot_You_Hear, http://dbpedia.org/resource/The_Laughter_of_Fools, http://dbpedia.org/resource/Follow_the_Lady_(film), http://dbpedia.org/resource/The_Sword_and_the_Rose, http://dbpedia.org/resource/Murder_by_Rope, http://dbpedia.org/resource/Menace_(1934_British_film), http://dbpedia.org/resource/Jurm_(2005_film), http://dbpedia.org/resource/Rules:_Pyaar_Ka_Superhit_Formula, http://dbpedia.org/resource/Paiyaa, http://dbpedia.org/resource/Katputtli, http://dbpedia.org/resource/Say_Salaam_India, http://dbpedia.org/resource/Arn_–_The_Kingdom_at_Road's_End, http://dbpedia.org/resource/Arn_–_The_Knight_Templar, http://dbpedia.org/resource/Shadow_(2009_film), http://dbpedia.org/resource/Satyameva_Jayate_(film), http://dbpedia.org/resource/16_December_(film), http://dbpedia.org/resource/Alex_Pandian, http://dbpedia.org/resource/Bhagmati_(film), http://dbpedia.org/resource/Bheja_Fry_(film), http://dbpedia.org/resource/Bhram, http://dbpedia.org/resource/Gandha_(film), http://dbpedia.org/resource/Nakshatra_(film), http://dbpedia.org/resource/Pachaikili_Muthucharam, http://dbpedia.org/resource/Valley_of_Flowers_(film), http://dbpedia.org/resource/Blue_Dream, http://dbpedia.org/resource/Exam_(film), http://dbpedia.org/resource/Bats:_Human_Harvest, http://dbpedia.org/resource/A_Matter_of_Murder, http://dbpedia.org/resource/Nuclear_Secrets, http://dbpedia.org/resource/Let_Us_Prey_(film), http://dbpedia.org/resource/The_Woman_(2011_film), http://dbpedia.org/resource/White_Settlers, http://dbpedia.org/resource/Hide_in_Plain_Sight, http://dbpedia.org/resource/Love_Up_the_Pole, http://dbpedia.org/resource/Master_and_Man_(1934_film), http://dbpedia.org/resource/No_Lady, http://dbpedia.org/resource/Trust_the_Navy, http://dbpedia.org/resource/The_Bad_Companions, http://dbpedia.org/resource/Never_Trouble_Trouble, http://dbpedia.org/resource/Packed_to_the_Rafters, http://dbpedia.org/resource/The_Alice_(film), http://dbpedia.org/resource/We're_Here_to_Help, http://dbpedia.org/resource/Chasing_the_Deer, http://dbpedia.org/resource/The_Black_Balloon_(film), http://dbpedia.org/resource/Beautiful_(2009_film), http://dbpedia.org/resource/Somersault_(film), http://dbpedia.org/resource/Huntingtower_(film), http://dbpedia.org/resource/Auld_Lang_Syne_(1929_film), http://dbpedia.org/resource/The_End_of_the_Road_(1936_film), http://dbpedia.org/resource/Sons_and_Lovers_(1960_film), http://dbpedia.org/resource/Look_Back_in_Anger_(1959_film), http://dbpedia.org/resource/Custer_of_the_West, http://dbpedia.org/resource/Where_Eagles_Dare, http://dbpedia.org/resource/Storm_Over_the_Nile, http://dbpedia.org/resource/The_Mind_Benders_(film), http://dbpedia.org/resource/A_Reflection_of_Fear, http://dbpedia.org/resource/The_Luck_of_Ginger_Coffey_(film), http://dbpedia.org/resource/Car_Trouble_(film), http://dbpedia.org/resource/Codename:_Kyril, http://dbpedia.org/resource/Chariots_of_Fire, http://dbpedia.org/resource/Ascendancy_(film), http://dbpedia.org/resource/Opera_(film), http://dbpedia.org/resource/The_Sun_Also_Rises_(1984_film), http://dbpedia.org/resource/The_Uninvited_(2008_film), http://dbpedia.org/resource/Raw_Silk_(film), http://dbpedia.org/resource/The_Craic, http://dbpedia.org/resource/The_Man_from_U.N.C.L.E., http://dbpedia.org/resource/Batman:_Gotham_Knight, http://dbpedia.org/resource/How_to_Steal_the_World, http://dbpedia.org/resource/The_Helicopter_Spies, http://dbpedia.org/resource/The_Karate_Killers, http://dbpedia.org/resource/The_Spy_in_the_Green_Hat, http://dbpedia.org/resource/The_Spy_with_My_Face, http://dbpedia.org/resource/The_Watcher_in_the_Woods, http://dbpedia.org/resource/Sol_Madrid, http://dbpedia.org/resource/Healer_(film), http://dbpedia.org/resource/The_Kingfisher_Caper, http://dbpedia.org/resource/Batman_vs._Robin, http://dbpedia.org/resource/The_Long_and_the_Short_and_the_Tall_(film), http://dbpedia.org/resource/Frankenstein:_The_True_Story, http://dbpedia.org/resource/King_Solomon's_Treasure, http://dbpedia.org/resource/Return_of_the_Man_from_U.N.C.L.E., http://dbpedia.org/resource/Dirty_Weekend_(1993_film), http://dbpedia.org/resource/Mosquito_Squadron, http://dbpedia.org/resource/The_Ravine, http://dbpedia.org/resource/Wonder_Woman_(2009_film), http://dbpedia.org/resource/Violent_Playground, http://dbpedia.org/resource/Shark_Attack_3, http://dbpedia.org/resource/Putting_It_Together:_Direct_from_Broadway, http://dbpedia.org/resource/Spawn_(film), http://dbpedia.org/resource/The_Wilby_Conspiracy, http://dbpedia.org/resource/Return_to_Oz, http://dbpedia.org/resource/I'm_Dancing_as_Fast_as_I_Can, http://dbpedia.org/resource/The_Human_Factor_(1979_film), http://dbpedia.org/resource/Black_Widow_(1987_film), http://dbpedia.org/resource/Hamlet_(1969_film), http://dbpedia.org/resource/The_Hour_of_the_Pig, http://dbpedia.org/resource/The_Exorcist_III, http://dbpedia.org/resource/Laughter_in_the_Dark_(film), http://dbpedia.org/resource/The_Jerusalem_File, http://dbpedia.org/resource/Sakharov_(film), http://dbpedia.org/resource/Excalibur_(film), http://dbpedia.org/resource/The_Wind_in_the_Willows_(1996_film), http://dbpedia.org/resource/The_Seven-Per-Cent_Solution_(film), http://dbpedia.org/resource/Inadmissible_Evidence_(film), http://dbpedia.org/resource/Venom_(1981_film), http://dbpedia.org/resource/The_Likely_Lads_(film), http://dbpedia.org/resource/Still_Open_All_Hours, http://dbpedia.org/resource/Boon_(TV_series), http://dbpedia.org/resource/Tower_of_Terror_(1941_film), http://dbpedia.org/resource/The_Science_of_Doctor_Who, http://dbpedia.org/resource/Kill_by_Inches, http://dbpedia.org/resource/The_Visit_(2015_American_film), http://dbpedia.org/resource/Rambo_(2008_film), http://dbpedia.org/resource/Dead_Space:_Aftermath, http://dbpedia.org/resource/Dante's_Inferno:_An_Animated_Epic, http://dbpedia.org/resource/Plastic_(film), http://dbpedia.org/resource/The_Wicker_Tree, http://dbpedia.org/resource/Merlin:_The_Quest_Begins, http://dbpedia.org/resource/Sisterhood_(film), http://dbpedia.org/resource/Green_Street_2:_Stand_Your_Ground, http://dbpedia.org/resource/The_Final_Cut_(1983_film), http://dbpedia.org/resource/Deadwood_(TV_series), http://dbpedia.org/resource/In_Search_of_La_Che, http://dbpedia.org/resource/Abattoir_(film), http://dbpedia.org/resource/Alleluia!_The_Devil's_Carnival, http://dbpedia.org/resource/The_Greyness_of_Autumn, http://dbpedia.org/resource/Midsomer_Murders, http://dbpedia.org/resource/House_Broken_(1936_film), http://dbpedia.org/resource/Ae_Fond_Kiss..., http://dbpedia.org/resource/Katie_Morag_(TV_series), http://dbpedia.org/resource/Jetsam_(film), http://dbpedia.org/resource/Verdict_of_the_Sea, http://dbpedia.org/resource/The_Perfect_Lady, http://dbpedia.org/resource/All_That_Glitters_(film), http://dbpedia.org/resource/The_Village_Squire, http://dbpedia.org/resource/Luck_of_the_Turf, http://dbpedia.org/resource/Full_Speed_Ahead_(film), http://dbpedia.org/resource/The_Strangler_(1932_film), http://dbpedia.org/resource/True_Stories_(film), http://dbpedia.org/resource/Heavy_Petting_(1989_film), http://dbpedia.org/resource/Love_in_Paris, http://dbpedia.org/resource/This_Year's_Love, http://dbpedia.org/resource/Things_to_Do_Before_You're_30, http://dbpedia.org/resource/Taken_3, http://dbpedia.org/resource/Dark_Water_(2005_film), http://dbpedia.org/resource/Gregory's_Two_Girls, http://dbpedia.org/resource/The_Ten_Commandments_(miniseries), http://dbpedia.org/resource/The_Truth_About_Love_(film), http://dbpedia.org/resource/Ripley's_Game_(film), http://dbpedia.org/resource/The_Place_of_the_Dead, http://dbpedia.org/resource/Death_Race_3:_Inferno, http://dbpedia.org/resource/Last_Passenger, http://dbpedia.org/resource/Ever_After, http://dbpedia.org/resource/Ride,_Rise,_Roar, http://dbpedia.org/resource/United_(2011_film), http://dbpedia.org/resource/Enigma_(2001_film), http://dbpedia.org/resource/Mission:_Impossible_II, http://dbpedia.org/resource/To_Kill_a_King, http://dbpedia.org/resource/False_Witness, http://dbpedia.org/resource/Perfect_Creature, http://dbpedia.org/resource/The_Vatican_Tapes, http://dbpedia.org/resource/Hitman_(2007_film), http://dbpedia.org/resource/Twin_Town, http://dbpedia.org/resource/Arabian_Nights_(miniseries), http://dbpedia.org/resource/Father_&_Son_(TV_serial), http://dbpedia.org/resource/There_Be_Dragons, http://dbpedia.org/resource/A_Thousand_Kisses_Deep_(film), http://dbpedia.org/resource/Faeries_(film), http://dbpedia.org/resource/New_Town_Killers, http://dbpedia.org/resource/Love's_Kitchen, http://dbpedia.org/resource/The_Day_of_the_Triffids_(2009_TV_miniseries), http://dbpedia.org/resource/Tiger_House, http://dbpedia.org/resource/Blood_from_the_Mummy's_Tomb, http://dbpedia.org/resource/The_Night_Visitor, http://dbpedia.org/resource/Daleks_–_Invasion_Earth:_2150_A.D., http://dbpedia.org/resource/The_Outsiders_(Australian_TV_series), http://dbpedia.org/resource/The_Devil-Ship_Pirates, http://dbpedia.org/resource/The_World_Cup:_A_Captain's_Tale, http://dbpedia.org/resource/The_Quatermass_Memoirs, http://dbpedia.org/resource/Dragonworld:_The_Legend_Continues, http://dbpedia.org/resource/The_Lady_Craved_Excitement, http://dbpedia.org/resource/Attack_on_the_Iron_Coast, http://dbpedia.org/resource/Oedipus_Rex_(1957_film), http://dbpedia.org/resource/The_Iron_Lady_(film), http://dbpedia.org/resource/The_Bad_Education_Movie, http://dbpedia.org/resource/Song_for_a_Raggy_Boy, http://dbpedia.org/resource/Death_of_a_Salesman_(1996_film), http://dbpedia.org/resource/Man_to_Man_(2005_film), http://dbpedia.org/resource/Paranoid_(2000_thriller_film), http://dbpedia.org/resource/Mountains_of_the_Moon_(film), http://dbpedia.org/resource/Lara_Croft:_Tomb_Raider, http://dbpedia.org/resource/Gabriel_&_Me, http://dbpedia.org/resource/Frankie's_House_(miniseries), http://dbpedia.org/resource/Starting_Over_(2007_film), http://dbpedia.org/resource/Resident_Evil:_Extinction, http://dbpedia.org/resource/Tara_Road_(film), http://dbpedia.org/resource/Darkness_(2002_film), http://dbpedia.org/resource/Fools_of_Fortune, http://dbpedia.org/resource/Silent_Scream_(1990_film), http://dbpedia.org/resource/Kidnapped_(2005_miniseries), http://dbpedia.org/resource/The_Diary_of_Anne_Frank_(2009_miniseries), http://dbpedia.org/resource/Mrs_Ratcliffe's_Revolution, http://dbpedia.org/resource/Resident_Evil:_The_Final_Chapter, http://dbpedia.org/resource/Beautiful_Creatures_(2000_film), http://dbpedia.org/resource/The_Red_Shoes_(1948_film), http://dbpedia.org/resource/The_Story_of_Three_Loves, http://dbpedia.org/resource/The_Tales_of_Hoffmann_(film), http://dbpedia.org/resource/Peeping_Tom_(film), http://dbpedia.org/resource/The_Man_Who_Loved_Redheads, http://dbpedia.org/resource/The_Last_Impresario, http://dbpedia.org/resource/Bastille_Day_(film), http://dbpedia.org/resource/How_to_Get_Ahead_in_Advertising, http://dbpedia.org/resource/Jeffrey_Archer:_The_Truth, http://dbpedia.org/resource/Last_of_the_Summer_Wine, http://dbpedia.org/resource/Cinderella_(2015_Disney_film), http://dbpedia.org/resource/Danny_and_the_Human_Zoo, http://dbpedia.org/resource/A_Promise_(2013_film), http://dbpedia.org/resource/A_Harlot's_Progress_(film), http://dbpedia.org/resource/The_Man_Who_Knew_Too_Little, http://dbpedia.org/resource/Lady_Chatterley's_Lover_(2015_film), http://dbpedia.org/resource/Frank_Herbert's_Children_of_Dune, http://dbpedia.org/resource/The_Ghost_Writer_(film), http://dbpedia.org/resource/Frankenstein_(miniseries), http://dbpedia.org/resource/G:MT_–_Greenwich_Mean_Time, http://dbpedia.org/resource/The_Gene_Generation, http://dbpedia.org/resource/Long_Time_Dead, http://dbpedia.org/resource/The_Principles_of_Lust, http://dbpedia.org/resource/Unrelated, http://dbpedia.org/resource/Peppa_Pig, http://dbpedia.org/resource/Pride_and_Prejudice_(1980_TV_series), http://dbpedia.org/resource/Moonlight_Serenade_(2009_film), http://dbpedia.org/resource/The_Cleopatras, http://dbpedia.org/resource/The_Missionary, http://dbpedia.org/resource/O_Lucky_Man!, http://dbpedia.org/resource/Romance_with_a_Double_Bass, http://dbpedia.org/resource/A_Very_Peculiar_Practice, http://dbpedia.org/resource/Appearances_(film), http://dbpedia.org/resource/The_Princess_of_New_York, http://dbpedia.org/resource/The_Spanish_Jade, http://dbpedia.org/resource/She_Knows_Y'Know, http://dbpedia.org/resource/The_Right_to_Love_(1920_film), http://dbpedia.org/resource/Gloria's_Romance, http://dbpedia.org/resource/Missing_Millions, http://dbpedia.org/resource/Disraeli_(1916_film), http://dbpedia.org/resource/Idols_of_Clay_(1920_film), http://dbpedia.org/resource/On_with_the_Dance_(1920_film), http://dbpedia.org/resource/Outcast_(1922_film), http://dbpedia.org/resource/Lady_Rose's_Daughter, http://dbpedia.org/resource/The_Fordington_Twins, http://dbpedia.org/resource/The_Man_Who_Stayed_at_Home, http://dbpedia.org/resource/The_Mystery_Road, http://dbpedia.org/resource/The_Fatal_Card, http://dbpedia.org/resource/Over_the_Odds, http://dbpedia.org/resource/The_Teeth_of_the_Tiger_(film), http://dbpedia.org/resource/The_Dawn_of_a_Tomorrow_(1915_film), http://dbpedia.org/resource/The_Green_Goddess_(1923_film), http://dbpedia.org/resource/Counterfeit_(film), http://dbpedia.org/resource/The_Lost_Chord_(1925_film), http://dbpedia.org/resource/Dangerous_Lies, http://dbpedia.org/resource/Fog_Bound, http://dbpedia.org/resource/The_Siren_Call, http://dbpedia.org/resource/Kill_Your_Friends_(film), http://dbpedia.org/resource/Tanner_Hall_(film), http://dbpedia.org/resource/Cockneys_vs_Zombies, http://dbpedia.org/resource/Tormented_(2009_British_film), http://dbpedia.org/resource/The_Flame_(1926_film), http://dbpedia.org/resource/Love's_Option, http://dbpedia.org/resource/Quinneys_(1927_film), http://dbpedia.org/resource/The_Four_Feathers_(1921_film), http://dbpedia.org/resource/The_Great_Prince_Shan, http://dbpedia.org/resource/A_Woman_of_No_Importance_(1921_film), http://dbpedia.org/resource/The_Flying_Squad_(1929_film), http://dbpedia.org/resource/The_Little_Minister_(1915_film), http://dbpedia.org/resource/Alien_Invasion_(film), http://dbpedia.org/resource/The_Trouble_with_Men_and_Women, http://dbpedia.org/resource/Poppies_of_Flanders, http://dbpedia.org/resource/Hope_and_Glory_(film), http://dbpedia.org/resource/Ordinary_Decent_Criminal, http://dbpedia.org/resource/The_Boy_in_the_Striped_Pyjamas_(film), http://dbpedia.org/resource/The_Lady_in_the_Car_with_Glasses_and_a_Gun, http://dbpedia.org/resource/Sunshine_on_Leith_(film), http://dbpedia.org/resource/A_Shot_in_the_West, http://dbpedia.org/resource/Soccer_Dog:_European_Cup, http://dbpedia.org/resource/Sonia_(film), http://dbpedia.org/resource/Blond_Cheat, http://dbpedia.org/resource/His_Wife's_Husband, http://dbpedia.org/resource/A_Gamble_with_Hearts, http://dbpedia.org/resource/Master_and_Man_(1929_film), http://dbpedia.org/resource/Money_(1921_film), http://dbpedia.org/resource/The_Amazing_Grace, http://dbpedia.org/resource/All_the_World's_a_Stooge, http://dbpedia.org/resource/The_Reverse_of_the_Medal_(film), http://dbpedia.org/resource/Drums_of_Fu_Manchu, http://dbpedia.org/resource/The_Wonderful_Story_(1922_film), http://dbpedia.org/resource/Distant_Trumpet, http://dbpedia.org/resource/High_Terrace, http://dbpedia.org/resource/Secrets_of_a_Windmill_Girl, http://dbpedia.org/resource/The_Quiet_Woman, http://dbpedia.org/resource/My_Dog_Tulip, http://dbpedia.org/resource/The_Loves_of_Joanna_Godden, http://dbpedia.org/resource/Marry_Me!_(1949_film), http://dbpedia.org/resource/The_Life_and_Adventures_of_Nicholas_Nickleby_(1947_film), http://dbpedia.org/resource/The_Hand_(1960_film), http://dbpedia.org/resource/Dombey_and_Son_(film), http://dbpedia.org/resource/Everybody's_Business, http://dbpedia.org/resource/A_Gamble_in_Lives, http://dbpedia.org/resource/The_Shulamite_(film), http://dbpedia.org/resource/Postmortem_(1998_film), http://dbpedia.org/resource/The_Innocents_(1961_film), http://dbpedia.org/resource/Rushmore_(film), http://dbpedia.org/resource/Contamination_(film), http://dbpedia.org/resource/Zombi_2, http://dbpedia.org/resource/Zombie_Holocaust, http://dbpedia.org/resource/Secrets_&_Lies_(film), http://dbpedia.org/resource/The_Peterville_Diamond, http://dbpedia.org/resource/Fine_Feathers_(1937_film), http://dbpedia.org/resource/The_Kitchen_Toto, http://dbpedia.org/resource/A_Girl_Must_Live, http://dbpedia.org/resource/The_Horse's_Mouth_(film), http://dbpedia.org/resource/No_Monkey_Business, http://dbpedia.org/resource/Their_Night_Out, http://dbpedia.org/resource/The_Final_Stage, http://dbpedia.org/resource/The_Mysterious_Geographic_Explorations_of_Jasper_Morello, http://dbpedia.org/resource/A_Thin_Life, http://dbpedia.org/resource/Come_Rain_or_Shine, http://dbpedia.org/resource/Loose_Women, http://dbpedia.org/resource/You_Instead, http://dbpedia.org/resource/See_Grace_Fly, http://dbpedia.org/resource/The_Marine_4:_Moving_Target, http://dbpedia.org/resource/Mom,_Dad_and_Her, http://dbpedia.org/resource/Dancing_in_the_Dark_(1986_film), http://dbpedia.org/resource/Beethoven_Lives_Upstairs, http://dbpedia.org/resource/Stargate_Atlantis, http://dbpedia.org/resource/Magic_Beyond_Words, http://dbpedia.org/resource/A_Dog's_Breakfast, http://dbpedia.org/resource/Christmas_Magic_(film), http://dbpedia.org/resource/Hercules_(miniseries), http://dbpedia.org/resource/The_Little_Kidnappers_(1990_film), http://dbpedia.org/resource/Rabid, http://dbpedia.org/resource/Keep_It_in_the_Family_(1973_film), http://dbpedia.org/resource/Survivors_(2008_TV_series), http://dbpedia.org/resource/The_Sleeping_Room, http://dbpedia.org/resource/You_Pay_Your_Money, http://dbpedia.org/resource/Four_Days_(film), http://dbpedia.org/resource/Devil_Girl_from_Mars, http://dbpedia.org/resource/Lilli_Marlene_(film), http://dbpedia.org/resource/A_Shine_of_Rainbows, http://dbpedia.org/resource/No_Orchids_for_Miss_Blandish_(film), http://dbpedia.org/resource/Two_Guys_Abroad, http://dbpedia.org/resource/Under_Jakob's_Ladder, http://dbpedia.org/resource/Command_Performance_(1937_film), http://dbpedia.org/resource/Follow_Your_Star, http://dbpedia.org/resource/The_Third_String_(1932_film), http://dbpedia.org/resource/Up_in_the_World, http://dbpedia.org/resource/Outpost:_Rise_of_the_Spetsnaz, http://dbpedia.org/resource/Millie_Inbetween:_Access_All_Areas, http://dbpedia.org/resource/Running_in_Traffic, http://dbpedia.org/resource/Garfield:_A_Tail_of_Two_Kitties, http://dbpedia.org/resource/Fuck_(film), http://dbpedia.org/resource/The_Boondock_Saints, http://dbpedia.org/resource/Weekend_In_Wallop, http://dbpedia.org/resource/Wild_Oats_(film), http://dbpedia.org/resource/The_Debt_Collector, http://dbpedia.org/resource/Billy_Connolly_Live_1994, http://dbpedia.org/resource/Absolution_(1978_film), http://dbpedia.org/resource/Open_Season_(2006_film), http://dbpedia.org/resource/Still_Crazy, http://dbpedia.org/resource/Open_Season_2, http://dbpedia.org/resource/Paws_(film), http://dbpedia.org/resource/Just_Another_Saturday, http://dbpedia.org/resource/Pixels_(2015_film), http://dbpedia.org/resource/Manhunter_(film)]... [INFO] Predicted Answer: [http://dbpedia.org/resource/Andy_Clyde, http://dbpedia.org/resource/Benny_Young, http://dbpedia.org/resource/Gianni_Capaldi, http://dbpedia.org/resource/Sara_Crowe, http://dbpedia.org/resource/Martin_Boddey, http://dbpedia.org/resource/Moultrie_Kelsall, http://dbpedia.org/resource/William_Duncan_(actor), http://dbpedia.org/resource/Bill_Simpson_(actor), http://dbpedia.org/resource/Adam_Sinclair_(actor), http://dbpedia.org/resource/Margaret_Mann, http://dbpedia.org/resource/Jackie_Oudney, http://dbpedia.org/resource/Brown_Derby_(actor), http://dbpedia.org/resource/John_Hannah_(actor), http://dbpedia.org/resource/Colin_Campbell_(director), http://dbpedia.org/resource/Jack_Lambert_(British_actor), http://dbpedia.org/resource/Iain_Glen, http://dbpedia.org/resource/David_Hayman, http://dbpedia.org/resource/Richard_Rankin, http://dbpedia.org/resource/James_Martin_(Scottish_actor), http://dbpedia.org/resource/Juliet_Cadzow, http://dbpedia.org/resource/Tommy_Flanagan_(actor), http://dbpedia.org/resource/Mark_Prendergast, http://dbpedia.org/resource/Irene_Napier, http://dbpedia.org/resource/Kenneth_Cranham, http://dbpedia.org/resource/Michael_E._Rodgers, http://dbpedia.org/resource/Paul_Higgins_(actor), http://dbpedia.org/resource/Nicola_Black, http://dbpedia.org/resource/George_T._Miller, http://dbpedia.org/resource/John_Cairney, http://dbpedia.org/resource/James_Watson_(actor), http://dbpedia.org/resource/Melissa_Stribling, http://dbpedia.org/resource/Ivan_Simpson] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 749, P: 0.499, R: 0.552, macro F1: 0.506, macro F1*: 0.524 -[INFO] Current time: 8935 ms, average: 2870.56 ms/ques, entity detection: 478.32 ms/ques (linking tool: 385.38 ms/ques), relation detection: 928.47 ms/ques (similarity: 702.34 ms/ques), EDG generation: 46.31 ms/ques +[INFO] Cumulative metrics, sample: 749, P: 0.499, R: 0.552, macro F1: 0.524 +[INFO] Current question consumed time: 8165 ms, average: 2584.58 ms/ques, entity detection: 422.62 ms/ques (linking tool: 344.23 ms/ques), relation detection: 856.18 ms/ques (similarity: 694.10 ms/ques), EDG generation: 46.33 ms/ques [INFO] Entity Linking P: 0.7443257676902537, R: 0.6969292389853138, micro F1: 0.7120605251446382, macro F1: 0.7198481718097498 [INFO] Relation Linking P: 0.5973520249221184, R: 0.5420560747663551, micro F1: 0.557899421450824, macro F1: 0.5683622820858433 [INFO] Type Linking P: 0.5460614152202937, R: 0.5460614152202937, micro F1: 0.5460614152202937, macro F1: 0.5460614152202937 @@ -10517,8 +10517,8 @@ Question 749: List the other nicknames of city which has one of the nickname as [INFO] Golden Answer: [http://dbpedia.org/resource/Nicolaus_Copernicus, http://dbpedia.org/resource/Toruń_gingerbread, http://dbpedia.org/resource/City_of_Angels] [INFO] Predicted Answer: [Bob Cavallo@en] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 750, P: 0.498, R: 0.551, macro F1: 0.505, macro F1*: 0.523 -[INFO] Current time: 2768 ms, average: 2870.42 ms/ques, entity detection: 479.37 ms/ques (linking tool: 386.44 ms/ques), relation detection: 927.90 ms/ques (similarity: 702.03 ms/ques), EDG generation: 46.30 ms/ques +[INFO] Cumulative metrics, sample: 750, P: 0.498, R: 0.551, macro F1: 0.523 +[INFO] Current question consumed time: 2429 ms, average: 2584.37 ms/ques, entity detection: 423.54 ms/ques (linking tool: 345.17 ms/ques), relation detection: 855.66 ms/ques (similarity: 693.81 ms/ques), EDG generation: 46.32 ms/ques [INFO] Entity Linking P: 0.7433333333333333, R: 0.696, micro F1: 0.711111111111112, macro F1: 0.7188883742473366 [INFO] Relation Linking P: 0.5965555555555555, R: 0.5413333333333333, micro F1: 0.5571555555555563, macro F1: 0.5676044657097288 [INFO] Type Linking P: 0.5466666666666666, R: 0.5466666666666666, micro F1: 0.5466666666666666, macro F1: 0.5466666666666666 @@ -10531,8 +10531,8 @@ Question 750: Count the writers whose work has been played in NY? [INFO] Golden Answer: [13] [INFO] Predicted Answer: [] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 751, P: 0.497, R: 0.551, macro F1: 0.504, macro F1*: 0.523 -[INFO] Current time: 542 ms, average: 2867.32 ms/ques, entity detection: 478.73 ms/ques (linking tool: 385.92 ms/ques), relation detection: 926.99 ms/ques (similarity: 701.30 ms/ques), EDG generation: 46.26 ms/ques +[INFO] Cumulative metrics, sample: 751, P: 0.497, R: 0.551, macro F1: 0.523 +[INFO] Current question consumed time: 465 ms, average: 2581.55 ms/ques, entity detection: 422.98 ms/ques (linking tool: 344.71 ms/ques), relation detection: 854.81 ms/ques (similarity: 693.09 ms/ques), EDG generation: 46.27 ms/ques [INFO] Entity Linking P: 0.7423435419440746, R: 0.6950732356857523, micro F1: 0.7101642254771425, macro F1: 0.7179311327370207 [INFO] Relation Linking P: 0.5957612072791832, R: 0.5406125166444741, micro F1: 0.5564136706613412, macro F1: 0.5668486674864137 [INFO] Type Linking P: 0.5472703062583223, R: 0.5472703062583223, micro F1: 0.5472703062583223, macro F1: 0.5472703062583223 @@ -10545,8 +10545,8 @@ Question 751: Where did the graduates of Memphis, Tenessee continue their educat [INFO] Golden Answer: [http://dbpedia.org/resource/Vanderbilt_University] [INFO] Predicted Answer: [http://dbpedia.org/resource/Broken_Social_Scene, http://dbpedia.org/resource/Torquil_Campbell, http://dbpedia.org/resource/Evan_Cranley, http://dbpedia.org/resource/Amy_Millan, http://dbpedia.org/resource/Da_Gryptions, http://dbpedia.org/resource/Memphis_(band), http://dbpedia.org/resource/Stars_(Canadian_band), http://dbpedia.org/resource/Dead_Child_Star_(band)] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 752, P: 0.497, R: 0.550, macro F1: 0.504, macro F1*: 0.522 -[INFO] Current time: 2208 ms, average: 2866.44 ms/ques, entity detection: 478.68 ms/ques (linking tool: 385.85 ms/ques), relation detection: 926.53 ms/ques (similarity: 700.83 ms/ques), EDG generation: 46.32 ms/ques +[INFO] Cumulative metrics, sample: 752, P: 0.497, R: 0.550, macro F1: 0.522 +[INFO] Current question consumed time: 1934 ms, average: 2580.69 ms/ques, entity detection: 422.92 ms/ques (linking tool: 344.63 ms/ques), relation detection: 854.38 ms/ques (similarity: 692.66 ms/ques), EDG generation: 46.35 ms/ques [INFO] Entity Linking P: 0.7413563829787234, R: 0.6941489361702128, micro F1: 0.7092198581560293, macro F1: 0.7169764370817853 [INFO] Relation Linking P: 0.5949689716312057, R: 0.5398936170212766, micro F1: 0.555673758865249, macro F1: 0.5660948793647561 [INFO] Type Linking P: 0.5478723404255319, R: 0.5478723404255319, micro F1: 0.5478723404255319, macro F1: 0.5478723404255319 @@ -10559,8 +10559,8 @@ Question 752: What subsequent work of Moondog Matinee has the composer of Motorp [INFO] Golden Answer: [http://dbpedia.org/resource/Bob_Dylan] [INFO] Predicted Answer: [http://dbpedia.org/resource/Bob_Dylan] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 753, P: 0.497, R: 0.550, macro F1: 0.504, macro F1*: 0.523 -[INFO] Current time: 1492 ms, average: 2864.62 ms/ques, entity detection: 478.54 ms/ques (linking tool: 385.63 ms/ques), relation detection: 925.82 ms/ques (similarity: 700.22 ms/ques), EDG generation: 46.39 ms/ques +[INFO] Cumulative metrics, sample: 753, P: 0.497, R: 0.550, macro F1: 0.523 +[INFO] Current question consumed time: 1251 ms, average: 2578.92 ms/ques, entity detection: 422.75 ms/ques (linking tool: 344.41 ms/ques), relation detection: 853.71 ms/ques (similarity: 692.06 ms/ques), EDG generation: 46.43 ms/ques [INFO] Entity Linking P: 0.7416998671978752, R: 0.6945551128818062, micro F1: 0.7096060203629935, macro F1: 0.7173537319361789 [INFO] Relation Linking P: 0.5955068614431164, R: 0.5405046480743692, micro F1: 0.556263833554671, macro F1: 0.5666742350293581 [INFO] Type Linking P: 0.548472775564409, R: 0.548472775564409, micro F1: 0.548472775564409, macro F1: 0.548472775564409 @@ -10573,8 +10573,8 @@ Question 753: What was the language of the single which came before To Know Him [INFO] Golden Answer: [http://dbpedia.org/resource/Slovakia] [INFO] Predicted Answer: [] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 754, P: 0.497, R: 0.550, macro F1: 0.504, macro F1*: 0.522 -[INFO] Current time: 1245 ms, average: 2862.47 ms/ques, entity detection: 478.06 ms/ques (linking tool: 385.23 ms/ques), relation detection: 925.20 ms/ques (similarity: 699.93 ms/ques), EDG generation: 46.40 ms/ques +[INFO] Cumulative metrics, sample: 754, P: 0.497, R: 0.550, macro F1: 0.522 +[INFO] Current question consumed time: 1103 ms, average: 2576.96 ms/ques, entity detection: 422.31 ms/ques (linking tool: 344.05 ms/ques), relation detection: 853.14 ms/ques (similarity: 691.80 ms/ques), EDG generation: 46.44 ms/ques [INFO] Entity Linking P: 0.7407161803713528, R: 0.6936339522546419, micro F1: 0.7086648983200717, macro F1: 0.716402334413717 [INFO] Relation Linking P: 0.5947170645446507, R: 0.5397877984084881, micro F1: 0.5555260831122908, macro F1: 0.5659226776884703 [INFO] Type Linking P: 0.5490716180371353, R: 0.5490716180371353, micro F1: 0.5490716180371353, macro F1: 0.5490716180371353 @@ -10587,8 +10587,8 @@ Question 754: What are the television shows whose company is owned by Viacom? [INFO] Golden Answer: [http://dbpedia.org/resource/Family_Ties, http://dbpedia.org/resource/The_Tortellis, http://dbpedia.org/resource/Baby_Bob, http://dbpedia.org/resource/Dynasty_(TV_series), http://dbpedia.org/resource/Look_What's_Happened_to_Rosemary's_Baby, http://dbpedia.org/resource/Love_Story_(1973_TV_series), http://dbpedia.org/resource/Mission:_Impossible, http://dbpedia.org/resource/Sabrina,_the_Teenage_Witch_(TV_series), http://dbpedia.org/resource/The_Brady_Bunch, http://dbpedia.org/resource/The_Lucy_Show, http://dbpedia.org/resource/Behind_the_Music, http://dbpedia.org/resource/The_Insider_(TV_series), http://dbpedia.org/resource/The_Parkers, http://dbpedia.org/resource/Webster_(TV_series), http://dbpedia.org/resource/Grease:_Live, http://dbpedia.org/resource/Vega$, http://dbpedia.org/resource/Don't_Go_to_Sleep, http://dbpedia.org/resource/The_Sentinel_(TV_series), http://dbpedia.org/resource/Titans_(U.S._TV_series), http://dbpedia.org/resource/Longstreet_(TV_series), http://dbpedia.org/resource/Sex,_Love_&_Secrets, http://dbpedia.org/resource/First_Monday, http://dbpedia.org/resource/Gung_Ho_(TV_series), http://dbpedia.org/resource/Key_West_(TV_series), http://dbpedia.org/resource/Return_to_Mayberry, http://dbpedia.org/resource/Heaven_Help_Us_(TV_series), http://dbpedia.org/resource/The_Silent_Force_(TV_series), http://dbpedia.org/resource/I_Love_the_'80s_(U.S._TV_series), http://dbpedia.org/resource/Kate_McShane, http://dbpedia.org/resource/Matt_Houston, http://dbpedia.org/resource/The_Bad_News_Bears_(TV_series), http://dbpedia.org/resource/Goodtime_Girls, http://dbpedia.org/resource/LateLine, http://dbpedia.org/resource/Almost_Perfect, http://dbpedia.org/resource/The_Immortal_(TV_series), http://dbpedia.org/resource/Hudson_Street_(TV_series), http://dbpedia.org/resource/Flying_Blind_(TV_series), http://dbpedia.org/resource/Kingpin_(TV_series), http://dbpedia.org/resource/Washington:_Behind_Closed_Doors, http://dbpedia.org/resource/Itsy_Bitsy_Spider_(TV_series), http://dbpedia.org/resource/Getting_Married_(film), http://dbpedia.org/resource/Big_Wave_Dave's, http://dbpedia.org/resource/Winnetka_Road, http://dbpedia.org/resource/Ghost_Whisperer, http://dbpedia.org/resource/Star_Trek:_The_Original_Series, http://dbpedia.org/resource/Star_Trek:_Voyager, http://dbpedia.org/resource/The_Love_Boat, http://dbpedia.org/resource/The_Young_Indiana_Jones_Chronicles, http://dbpedia.org/resource/MacGyver, http://dbpedia.org/resource/When_Things_Were_Rotten, http://dbpedia.org/resource/Cuts, http://dbpedia.org/resource/Hope_for_Haiti_Now:_A_Global_Benefit_for_Earthquake_Relief, http://dbpedia.org/resource/One_on_One_(TV_series), http://dbpedia.org/resource/Cybill, http://dbpedia.org/resource/Finding_Carter, http://dbpedia.org/resource/Linc's, http://dbpedia.org/resource/Me_and_the_Chimp, http://dbpedia.org/resource/Kristin_(TV_series), http://dbpedia.org/resource/Hard_Knocks_(1987_TV_series), http://dbpedia.org/resource/The_Girl,_the_Gold_Watch_&_Everything_(film), http://dbpedia.org/resource/The_House_That_Would_Not_Die, http://dbpedia.org/resource/America_(U.S._TV_series), http://dbpedia.org/resource/Anything_for_Money, http://dbpedia.org/resource/Medium_(TV_series), http://dbpedia.org/resource/The_Adventures_of_Jimmy_Neutron:_Boy_Genius, http://dbpedia.org/resource/The_San_Pedro_Beach_Bums, http://dbpedia.org/resource/American_Dreamer_(TV_series), http://dbpedia.org/resource/Brooklyn_Bridge_(TV_series), http://dbpedia.org/resource/Laverne_&_Shirley, http://dbpedia.org/resource/Mork_&_Mindy, http://dbpedia.org/resource/Tales_from_the_Darkside, http://dbpedia.org/resource/Taxi_(TV_series), http://dbpedia.org/resource/That's_Life_(2000_TV_series), http://dbpedia.org/resource/Soul_Food_(TV_series), http://dbpedia.org/resource/Beverly_Hills,_90210, http://dbpedia.org/resource/Minority_Report_(TV_series), http://dbpedia.org/resource/Brothers_and_Sisters_(1979_TV_series), http://dbpedia.org/resource/MacGruder_and_Loud, http://dbpedia.org/resource/Out_of_the_Blue_(1979_TV_series), http://dbpedia.org/resource/The_Cougar_(TV_series), http://dbpedia.org/resource/The_Love_Boat:_The_Next_Wave, http://dbpedia.org/resource/Hope_Island_(TV_series), http://dbpedia.org/resource/The_Busy_World_of_Richard_Scarry, http://dbpedia.org/resource/Wanted_(TV_series), http://dbpedia.org/resource/Wipeout_(1988_U.S._game_show), http://dbpedia.org/resource/Ace_Crawford,_Private_Eye, http://dbpedia.org/resource/Aloha_Paradise, http://dbpedia.org/resource/Andy_Richter_Controls_the_Universe, http://dbpedia.org/resource/Busting_Loose_(TV_series), http://dbpedia.org/resource/Courting_Alex, http://dbpedia.org/resource/Joanie_Loves_Chachi, http://dbpedia.org/resource/The_Mommies_(TV_series), http://dbpedia.org/resource/The_Pursuit_of_Happiness_(TV_series), http://dbpedia.org/resource/Summerland_(TV_series), http://dbpedia.org/resource/Keen_Eddie, http://dbpedia.org/resource/The_Harlem_Globetrotters_Popcorn_Machine, http://dbpedia.org/resource/Dog_and_Cat, http://dbpedia.org/resource/Madman_of_the_People, http://dbpedia.org/resource/School_of_Rock_(TV_series), http://dbpedia.org/resource/Shooter_(TV_series), http://dbpedia.org/resource/The_Home_Court, http://dbpedia.org/resource/Seven_Days_(TV_series), http://dbpedia.org/resource/Barbary_Coast_(TV_series), http://dbpedia.org/resource/Diagnosis:_Murder, http://dbpedia.org/resource/Mannix, http://dbpedia.org/resource/Numbers_(TV_series), http://dbpedia.org/resource/Sons_and_Daughters_(1991_TV_series), http://dbpedia.org/resource/The_Magician_(U.S._TV_series), http://dbpedia.org/resource/Deadly_Games, http://dbpedia.org/resource/Ed_(TV_series), http://dbpedia.org/resource/Melrose_Place, http://dbpedia.org/resource/Nurse_(1981_TV_series), http://dbpedia.org/resource/Duckman, http://dbpedia.org/resource/Police_Squad!, http://dbpedia.org/resource/Shōgun_(miniseries), http://dbpedia.org/resource/The_Pentagon_Papers_(film), http://dbpedia.org/resource/Grandpa_Goes_to_Washington, http://dbpedia.org/resource/Love_&_Money, http://dbpedia.org/resource/Haunted_(TV_series), http://dbpedia.org/resource/HeartBeat_(U.S._TV_series), http://dbpedia.org/resource/Wolf_Lake, http://dbpedia.org/resource/Nightmare_on_the_13th_Floor, http://dbpedia.org/resource/Throb, http://dbpedia.org/resource/Who's_Watching_the_Kids%3F, http://dbpedia.org/resource/Becker_(TV_series), http://dbpedia.org/resource/Cheers, http://dbpedia.org/resource/Love,_American_Style, http://dbpedia.org/resource/The_Untouchables_(1993_TV_series), http://dbpedia.org/resource/Twin_Peaks, http://dbpedia.org/resource/7th_Heaven_(TV_series), http://dbpedia.org/resource/Brothers_(TV_series), http://dbpedia.org/resource/Burke's_Law, http://dbpedia.org/resource/Deadwood_(TV_series), http://dbpedia.org/resource/Future_Cop_(TV_series), http://dbpedia.org/resource/The_Young_Lawyers, http://dbpedia.org/resource/Strike_Force_(TV_series), http://dbpedia.org/resource/Alcoa_Presents:_One_Step_Beyond, http://dbpedia.org/resource/Super_Force, http://dbpedia.org/resource/Unexplained_Mysteries, http://dbpedia.org/resource/Glitter_(TV_series), http://dbpedia.org/resource/Julie_(TV_series), http://dbpedia.org/resource/Bosom_Buddies, http://dbpedia.org/resource/The_Watcher_(TV_series), http://dbpedia.org/resource/The_Fonz_and_the_Happy_Days_Gang, http://dbpedia.org/resource/B.A.D._Cats, http://dbpedia.org/resource/Perry_Mason_(TV_movies), http://dbpedia.org/resource/The_Firm_(2012_TV_series), http://dbpedia.org/resource/The_Odd_Couple_(1970_TV_series), http://dbpedia.org/resource/Townies, http://dbpedia.org/resource/The_Division, http://dbpedia.org/resource/The_Handler_(TV_series), http://dbpedia.org/resource/The_Heights_(TV_series), http://dbpedia.org/resource/2000_Malibu_Road, http://dbpedia.org/resource/Frank's_Place, http://dbpedia.org/resource/The_Round_Table_(TV_series), http://dbpedia.org/resource/The_Best_Little_Girl_in_the_World, http://dbpedia.org/resource/The_Hoop_Life, http://dbpedia.org/resource/Paper_Moon_(TV_series), http://dbpedia.org/resource/Good_Company_(TV_series), http://dbpedia.org/resource/Pirate_TV, http://dbpedia.org/resource/Charmed, http://dbpedia.org/resource/Frasier, http://dbpedia.org/resource/Here's_Boomer, http://dbpedia.org/resource/Star_Trek:_Enterprise, http://dbpedia.org/resource/Amanda's, http://dbpedia.org/resource/NCIS_(TV_series), http://dbpedia.org/resource/Wings_(1990_TV_series), http://dbpedia.org/resource/Serpico_(TV_series), http://dbpedia.org/resource/Duet_(TV_series), http://dbpedia.org/resource/Fired_Up_(TV_series), http://dbpedia.org/resource/Girlfriends, http://dbpedia.org/resource/In-Laws, http://dbpedia.org/resource/Out_of_Practice, http://dbpedia.org/resource/Leeza, http://dbpedia.org/resource/Marblehead_Manor, http://dbpedia.org/resource/Sunset_Beach_(TV_series), http://dbpedia.org/resource/Models_Inc., http://dbpedia.org/resource/Call_to_Glory, http://dbpedia.org/resource/Clueless_(TV_series), http://dbpedia.org/resource/DiResta, http://dbpedia.org/resource/Marker_(TV_series), http://dbpedia.org/resource/Pig_Sty, http://dbpedia.org/resource/Platypus_Man, http://dbpedia.org/resource/The_Secret_Diary_of_Desmond_Pfeiffer, http://dbpedia.org/resource/Angie_(TV_series), http://dbpedia.org/resource/Sister,_Sister_(TV_series), http://dbpedia.org/resource/The_Jesse_Owens_Story, http://dbpedia.org/resource/The_Winds_of_War_(miniseries), http://dbpedia.org/resource/Brandy:_Special_Delivery, http://dbpedia.org/resource/George_and_Leo, http://dbpedia.org/resource/3_South, http://dbpedia.org/resource/All_Souls_(TV_series), http://dbpedia.org/resource/Big_Apple_(TV_series), http://dbpedia.org/resource/Bram_&_Alice, http://dbpedia.org/resource/Buddy_Faro, http://dbpedia.org/resource/Dear_John_(U.S._TV_series), http://dbpedia.org/resource/Do_Over, http://dbpedia.org/resource/Dolphin_Cove_(TV_series), http://dbpedia.org/resource/E.A.R.T.H._Force, http://dbpedia.org/resource/Easy_Street_(TV_series), http://dbpedia.org/resource/Encore!_Encore!, http://dbpedia.org/resource/Flesh_'n'_Blood_(TV_series), http://dbpedia.org/resource/Grapevine_(TV_series), http://dbpedia.org/resource/Guy_Code, http://dbpedia.org/resource/Hearts_Are_Wild, http://dbpedia.org/resource/His_&_Hers_(TV_series), http://dbpedia.org/resource/Hitz, http://dbpedia.org/resource/Hometown_(TV_series), http://dbpedia.org/resource/Jenny_(TV_series), http://dbpedia.org/resource/Life_on_a_Stick, http://dbpedia.org/resource/Maggie_(1998_TV_series), http://dbpedia.org/resource/Making_the_Grade_(TV_series), http://dbpedia.org/resource/Manhunt_(2001_TV_series), http://dbpedia.org/resource/Men,_Women_&_Dogs, http://dbpedia.org/resource/Mr._Smith_(TV_series), http://dbpedia.org/resource/Mulligan's_Stew, http://dbpedia.org/resource/Nightingales_(U.S._TV_series), http://dbpedia.org/resource/Oh_Yeah!_Cartoons, http://dbpedia.org/resource/Queens_Supreme, http://dbpedia.org/resource/Raising_Dad, http://dbpedia.org/resource/Safe_Harbor_(TV_series), http://dbpedia.org/resource/Some_of_My_Best_Friends, http://dbpedia.org/resource/South_of_Sunset, http://dbpedia.org/resource/Struck_by_Lightning_(TV_series), http://dbpedia.org/resource/Szysznyk, http://dbpedia.org/resource/The_Bronx_Zoo_(TV_series), http://dbpedia.org/resource/The_Cavanaughs_(TV_series), http://dbpedia.org/resource/The_Marshal, http://dbpedia.org/resource/The_Random_Years, http://dbpedia.org/resource/The_Royal_Family_(TV_series), http://dbpedia.org/resource/The_Sandy_Duncan_Show, http://dbpedia.org/resource/The_Fear_Inside_(film), http://dbpedia.org/resource/The_Lazarus_Syndrome, http://dbpedia.org/resource/All_is_Forgiven, http://dbpedia.org/resource/The_Brothers_Grunt, http://dbpedia.org/resource/The_Master_(TV_series), http://dbpedia.org/resource/Vinyl_(TV_series), http://dbpedia.org/resource/Working_Stiffs_(TV_series), http://dbpedia.org/resource/Special_Unit_2, http://dbpedia.org/resource/VH1's_Legends, http://dbpedia.org/resource/A_Very_Brady_Christmas, http://dbpedia.org/resource/Day_by_Day_(TV_series), http://dbpedia.org/resource/Ferris_Bueller_(TV_series), http://dbpedia.org/resource/The_Brady_Kids, http://dbpedia.org/resource/The_Bradys, http://dbpedia.org/resource/Assault_on_the_Wayne, http://dbpedia.org/resource/Higher_Ground_(TV_series), http://dbpedia.org/resource/The_Powers_of_Matthew_Star, http://dbpedia.org/resource/Love,_Inc._(TV_series), http://dbpedia.org/resource/Girl_Code, http://dbpedia.org/resource/South_Beach_(2006_TV_series), http://dbpedia.org/resource/10-8:_Officers_on_Duty, http://dbpedia.org/resource/Clubhouse_(TV_series), http://dbpedia.org/resource/Second_Time_Around_(TV_series), http://dbpedia.org/resource/Barefoot_in_the_Park_(TV_series), http://dbpedia.org/resource/Blansky's_Beauties, http://dbpedia.org/resource/Hollywood_Beat, http://dbpedia.org/resource/It's_All_Relative, http://dbpedia.org/resource/Madigan_Men, http://dbpedia.org/resource/Makin'_It_(TV_series), http://dbpedia.org/resource/Mr._Sunshine_(1986_TV_series), http://dbpedia.org/resource/Philly_(TV_series), http://dbpedia.org/resource/The_New_Odd_Couple, http://dbpedia.org/resource/The_Oddball_Couple, http://dbpedia.org/resource/The_Renegades, http://dbpedia.org/resource/The_Trouble_with_Normal_(TV_series), http://dbpedia.org/resource/But_I_Don't_Want_to_Get_Married!, http://dbpedia.org/resource/University_Hospital_(TV_series), http://dbpedia.org/resource/Quarantined_(film), http://dbpedia.org/resource/America's_Dumbest_Criminals, http://dbpedia.org/resource/Legend_(TV_series), http://dbpedia.org/resource/Open_House_(TV_series), http://dbpedia.org/resource/The_Brady_Bunch_Hour, http://dbpedia.org/resource/Viper_(TV_series), http://dbpedia.org/resource/Moesha, http://dbpedia.org/resource/Threshold_(TV_series), http://dbpedia.org/resource/Orleans_(TV_series), http://dbpedia.org/resource/Faking_It_(2014_TV_series), http://dbpedia.org/resource/Making_of_a_Male_Model, http://dbpedia.org/resource/Mission:_Impossible_(1988_TV_series), http://dbpedia.org/resource/Pacific_Palisades_(TV_series), http://dbpedia.org/resource/Resurrection_Blvd., http://dbpedia.org/resource/Bob_(TV_series), http://dbpedia.org/resource/Project_U.F.O., http://dbpedia.org/resource/The_Beat_(TV_series), http://dbpedia.org/resource/Malibu_Shores, http://dbpedia.org/resource/At_Ease_(TV_series), http://dbpedia.org/resource/Petrocelli, http://dbpedia.org/resource/Kindred:_The_Embraced, http://dbpedia.org/resource/Rescue_77, http://dbpedia.org/resource/How_Awful_About_Allan, http://dbpedia.org/resource/Medical_Investigation, http://dbpedia.org/resource/Hot_Ticket, http://dbpedia.org/resource/Foul_Play_(TV_series), http://dbpedia.org/resource/10,000_Black_Men_Named_George, http://dbpedia.org/resource/Ryan's_Four, http://dbpedia.org/resource/Down_Home_(TV_series)] [INFO] Predicted Answer: [http://dbpedia.org/resource/A_Colbert_Christmas:_The_Greatest_Gift_of_All!] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 755, P: 0.496, R: 0.549, macro F1: 0.503, macro F1*: 0.521 -[INFO] Current time: 1824 ms, average: 2861.09 ms/ques, entity detection: 477.75 ms/ques (linking tool: 384.97 ms/ques), relation detection: 924.85 ms/ques (similarity: 699.91 ms/ques), EDG generation: 46.36 ms/ques +[INFO] Cumulative metrics, sample: 755, P: 0.496, R: 0.549, macro F1: 0.521 +[INFO] Current question consumed time: 1573 ms, average: 2575.63 ms/ques, entity detection: 422.00 ms/ques (linking tool: 343.79 ms/ques), relation detection: 852.78 ms/ques (similarity: 691.77 ms/ques), EDG generation: 46.40 ms/ques [INFO] Entity Linking P: 0.7397350993377484, R: 0.6927152317880795, micro F1: 0.7077262693156743, macro F1: 0.715453457149593 [INFO] Relation Linking P: 0.5952538631346578, R: 0.5397350993377483, micro F1: 0.5556732891832238, macro F1: 0.5661366120253771 [INFO] Type Linking P: 0.5483443708609271, R: 0.5483443708609271, micro F1: 0.5483443708609271, macro F1: 0.5483443708609271 @@ -10601,8 +10601,8 @@ Question 755: How many people used to play in the youthclub of FC Barcelona? [INFO] Golden Answer: [319] [INFO] Predicted Answer: [150] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 756, P: 0.495, R: 0.548, macro F1: 0.502, macro F1*: 0.521 -[INFO] Current time: 5026 ms, average: 2863.96 ms/ques, entity detection: 478.43 ms/ques (linking tool: 385.66 ms/ques), relation detection: 925.68 ms/ques (similarity: 700.39 ms/ques), EDG generation: 46.42 ms/ques +[INFO] Cumulative metrics, sample: 756, P: 0.495, R: 0.548, macro F1: 0.521 +[INFO] Current question consumed time: 4428 ms, average: 2578.08 ms/ques, entity detection: 422.60 ms/ques (linking tool: 344.40 ms/ques), relation detection: 853.52 ms/ques (similarity: 692.23 ms/ques), EDG generation: 46.46 ms/ques [INFO] Entity Linking P: 0.7400793650793651, R: 0.6931216931216931, micro F1: 0.7081128747795424, macro F1: 0.7158312640546145 [INFO] Relation Linking P: 0.5951278659611993, R: 0.5403439153439153, micro F1: 0.5558201058201067, macro F1: 0.5664142896693134 [INFO] Type Linking P: 0.548941798941799, R: 0.548941798941799, micro F1: 0.548941798941799, macro F1: 0.548941798941799 @@ -10615,8 +10615,8 @@ Question 756: Quebec born ice hockey players have stopped playing for which team [INFO] Golden Answer: [http://dbpedia.org/resource/Montreal_Canadiens, http://dbpedia.org/resource/National_Hockey_League, http://dbpedia.org/resource/American_Hockey_League, http://dbpedia.org/resource/St._Louis_Blues, http://dbpedia.org/resource/Buffalo_Bisons_(AHL), http://dbpedia.org/resource/Toronto_Maple_Leafs, http://dbpedia.org/resource/Edmonton_Eskimos_(ice_hockey), http://dbpedia.org/resource/Detroit_Red_Wings, http://dbpedia.org/resource/Boston_Bruins, http://dbpedia.org/resource/Chicago_Blackhawks, http://dbpedia.org/resource/Toronto_Blueshirts, http://dbpedia.org/resource/Montreal_Victorias, http://dbpedia.org/resource/Kansas_City_Scouts, http://dbpedia.org/resource/New_York_Islanders, http://dbpedia.org/resource/Los_Angeles_Kings, http://dbpedia.org/resource/Pittsburgh_Penguins, http://dbpedia.org/resource/Baltimore_Skipjacks, http://dbpedia.org/resource/New_Haven_Nighthawks, http://dbpedia.org/resource/Victoria_Cougars, http://dbpedia.org/resource/Montreal_Wanderers, http://dbpedia.org/resource/New_York_Rangers, http://dbpedia.org/resource/Seattle_Metropolitans, http://dbpedia.org/resource/Renfrew_Creamery_Kings, http://dbpedia.org/resource/Brandon_Wheat_Cities, http://dbpedia.org/resource/Spokane_Canaries, http://dbpedia.org/resource/Ottawa_Senators, http://dbpedia.org/resource/Colorado_Rockies_(NHL), http://dbpedia.org/resource/Quebec_Nordiques, http://dbpedia.org/resource/Colorado_Avalanche, http://dbpedia.org/resource/Florida_Panthers, http://dbpedia.org/resource/Traktor_Chelyabinsk, http://dbpedia.org/resource/Ajoie, http://dbpedia.org/resource/HC_TWK_Innsbruck, http://dbpedia.org/resource/New_Jersey_Devils, http://dbpedia.org/resource/Jokerit, http://dbpedia.org/resource/Vålerenga_Ishockey, http://dbpedia.org/resource/Pittsburgh_Hornets, http://dbpedia.org/resource/Vancouver_Canucks, http://dbpedia.org/resource/Philadelphia_Flyers, http://dbpedia.org/resource/Calgary_Flames, http://dbpedia.org/resource/San_Jose_Sharks, http://dbpedia.org/resource/Tampa_Bay_Lightning, http://dbpedia.org/resource/Edmonton_Oilers, http://dbpedia.org/resource/Atlanta_Thrashers, http://dbpedia.org/resource/Vancouver_Canucks_(WHL), http://dbpedia.org/resource/Western_Hockey_League_(1952–74), http://dbpedia.org/resource/Baltimore_Clippers, http://dbpedia.org/resource/Hartford_Whalers, http://dbpedia.org/resource/Anaheim_Ducks, http://dbpedia.org/resource/Dinamo_Riga, http://dbpedia.org/resource/Hamilton_Tigers, http://dbpedia.org/resource/Quebec_Bulldogs, http://dbpedia.org/resource/Waterloo_Colts, http://dbpedia.org/resource/SC_Bern, http://dbpedia.org/resource/Buffalo_Sabres, http://dbpedia.org/resource/Carolina_Hurricanes, http://dbpedia.org/resource/HC_Lev_Praha, http://dbpedia.org/resource/Minnesota_North_Stars, http://dbpedia.org/resource/Minnesota_Wild, http://dbpedia.org/resource/Dallas_Stars, http://dbpedia.org/resource/EV_Zug, http://dbpedia.org/resource/Sheffield_Steelers, http://dbpedia.org/resource/Binghamton_Senators, http://dbpedia.org/resource/Manitoba_Moose, http://dbpedia.org/resource/San_Antonio_Rampage, http://dbpedia.org/resource/Cincinnati_Mighty_Ducks, http://dbpedia.org/resource/Nashville_Predators, http://dbpedia.org/resource/Washington_Capitals, http://dbpedia.org/resource/Winnipeg_Jets_(1972–96), http://dbpedia.org/resource/Fredericton_Express, http://dbpedia.org/resource/Lowell_Devils, http://dbpedia.org/resource/Halifax_Citadels, http://dbpedia.org/resource/Ottawa_Senators_(original), http://dbpedia.org/resource/Providence_Reds, http://dbpedia.org/resource/Duluth_Hornets, http://dbpedia.org/resource/Tulsa_Oilers, http://dbpedia.org/resource/New_York_Americans, http://dbpedia.org/resource/Pittsburgh_Pirates_(NHL), http://dbpedia.org/resource/Canadian-American_Hockey_League, http://dbpedia.org/resource/American_Hockey_Association_(1926–42), http://dbpedia.org/resource/National_Hockey_Association, http://dbpedia.org/resource/Toronto_Arenas, http://dbpedia.org/resource/Portland_Pirates, http://dbpedia.org/resource/Rochester_Americans, http://dbpedia.org/resource/Milwaukee_Admirals, http://dbpedia.org/resource/Houston_Aeros_(1994–2013), http://dbpedia.org/resource/Cincinnati_Cyclones, http://dbpedia.org/resource/Pittsburgh_Yellow_Jackets, http://dbpedia.org/resource/Canadian_Professional_Hockey_League, http://dbpedia.org/resource/Niagara_Falls_Cataracts, http://dbpedia.org/resource/International_Hockey_League_(1929–36), http://dbpedia.org/resource/New_Haven_Eagles, http://dbpedia.org/resource/St._Louis_Flyers, http://dbpedia.org/resource/Buffalo_Americans, http://dbpedia.org/resource/Montreal_Hockey_Club, http://dbpedia.org/resource/California_Golden_Seals, http://dbpedia.org/resource/Philadelphia_Quakers_(NHL), http://dbpedia.org/resource/Winnipeg_Victorias, http://dbpedia.org/resource/Toronto_Professional_Hockey_Club, http://dbpedia.org/resource/Montreal_Shamrocks, http://dbpedia.org/resource/Sault_Ste._Marie_Marlboros, http://dbpedia.org/resource/Haileybury_Comets, http://dbpedia.org/resource/Smiths_Falls_Seniors, http://dbpedia.org/resource/Canadian_Hockey_Association_(1909–10), http://dbpedia.org/resource/Federal_Amateur_Hockey_League, http://dbpedia.org/resource/Toronto_Ontarios, http://dbpedia.org/resource/Saskatoon_Sheiks, http://dbpedia.org/resource/Springfield_Indians, http://dbpedia.org/resource/Hartford_Wolf_Pack, http://dbpedia.org/resource/Chicago_Wolves, http://dbpedia.org/resource/Worcester_Sharks, http://dbpedia.org/resource/HC_Kometa_Brno, http://dbpedia.org/resource/HC_Fribourg-Gottéron, http://dbpedia.org/resource/Hamburg_Freezers, http://dbpedia.org/resource/Wheeling_Nailers, http://dbpedia.org/resource/Norfolk_Admirals_(AHL), http://dbpedia.org/resource/Wilkes-Barre/Scranton_Penguins, http://dbpedia.org/resource/Frisk_Asker_Ishockey, http://dbpedia.org/resource/Spektrum_Flyers, http://dbpedia.org/resource/Frölunda_HC, http://dbpedia.org/resource/Brest_Albatros_Hockey, http://dbpedia.org/resource/HC_Lugano, http://dbpedia.org/resource/National_League_A, http://dbpedia.org/resource/Columbus_Blue_Jackets, http://dbpedia.org/resource/Adirondack_Red_Wings, http://dbpedia.org/resource/Düsseldorfer_EG, http://dbpedia.org/resource/Rapperswil-Jona_Lakers, http://dbpedia.org/resource/HC_CSKA_Moscow, http://dbpedia.org/resource/Oulun_Kärpät, http://dbpedia.org/resource/Frankfurt_Lions, http://dbpedia.org/resource/HC_Dinamo_Minsk, http://dbpedia.org/resource/Timrå_IK, http://dbpedia.org/resource/Torpedo_Nizhny_Novgorod, http://dbpedia.org/resource/Kassel_Huskies, http://dbpedia.org/resource/Quebec_Aces, http://dbpedia.org/resource/Manhattan_Arrows, http://dbpedia.org/resource/Baltimore_Orioles, http://dbpedia.org/resource/Buffalo_Bisons, http://dbpedia.org/resource/Philadelphia_Rockets, http://dbpedia.org/resource/Louisville_Blades, http://dbpedia.org/resource/Hollywood_Wolves, http://dbpedia.org/resource/New_York_Rovers, http://dbpedia.org/resource/Washington_Lions, http://dbpedia.org/resource/Johnstown_Jets, http://dbpedia.org/resource/Seattle_Ironmen, http://dbpedia.org/resource/HC_Slovan_Bratislava, http://dbpedia.org/resource/Manchester_Monarchs_(AHL), http://dbpedia.org/resource/Diables_Rouges_de_Briançon, http://dbpedia.org/resource/Cornwall_Aces, http://dbpedia.org/resource/VEU_Feldkirch, http://dbpedia.org/resource/Jonquière_Marquis, http://dbpedia.org/resource/WSV_Sterzing_Broncos, http://dbpedia.org/resource/Minnesota_Moose, http://dbpedia.org/resource/Worcester_IceCats, http://dbpedia.org/resource/Phoenix_Mustangs, http://dbpedia.org/resource/Springfield_Falcons, http://dbpedia.org/resource/Peoria_Rivermen_(IHL), http://dbpedia.org/resource/Detroit_Falcons_(CoHL), http://dbpedia.org/resource/Hershey_Bears, http://dbpedia.org/resource/Reading_Royals, http://dbpedia.org/resource/Thomas_Sabo_Ice_Tigers, http://dbpedia.org/resource/Vienna_Capitals, http://dbpedia.org/resource/USA_Hockey, http://dbpedia.org/resource/Portland_Rosebuds_(ice_hockey), http://dbpedia.org/resource/Ligue_Nord-Américaine_de_Hockey, http://dbpedia.org/resource/FC_Spartak_Moscow, http://dbpedia.org/resource/Amur_Khabarovsk, http://dbpedia.org/resource/Fort_Worth_Brahmas, http://dbpedia.org/resource/Fort_Worth_Fire, http://dbpedia.org/resource/Sorel-Tracy_Blackhawks, http://dbpedia.org/resource/Johnstown_Chiefs, http://dbpedia.org/resource/Augsburger_Panther, http://dbpedia.org/resource/HC_Sierre-Anniviers, http://dbpedia.org/resource/Prince_Edward_Island_Senators, http://dbpedia.org/resource/Syracuse_Crunch, http://dbpedia.org/resource/HC_La_Chaux-de-Fonds, http://dbpedia.org/resource/EHC_Olten, http://dbpedia.org/resource/EC_Bad_Tölz, http://dbpedia.org/resource/EHC_Freiburg, http://dbpedia.org/resource/Essen_Mosquitoes, http://dbpedia.org/resource/ECHL, http://dbpedia.org/resource/Sherbrooke_Canadiens, http://dbpedia.org/resource/Erie_Panthers, http://dbpedia.org/resource/Rockford_IceHogs, http://dbpedia.org/resource/Graz_99ers, http://dbpedia.org/resource/Stavanger_Oilers, http://dbpedia.org/resource/Straubing_Tigers, http://dbpedia.org/resource/United_Hockey_League, http://dbpedia.org/resource/Trenton_Titans, http://dbpedia.org/resource/Dayton_Bombers, http://dbpedia.org/resource/Bloomington_PrairieThunder, http://dbpedia.org/resource/Trois-Rivières_Blizzard, http://dbpedia.org/resource/Philadelphia_Phantoms, http://dbpedia.org/resource/Bakersfield_Condors_(1998–2015), http://dbpedia.org/resource/Vancouver_Blazers, http://dbpedia.org/resource/Minnesota_Fighting_Saints, http://dbpedia.org/resource/Bridgeport_Sound_Tigers, http://dbpedia.org/resource/Ritten_Sport, http://dbpedia.org/resource/Starbulls_Rosenheim, http://dbpedia.org/resource/SC_Bietigheim-Bissingen, http://dbpedia.org/resource/Fredericton_Canadiens, http://dbpedia.org/resource/Tallahassee_Tiger_Sharks, http://dbpedia.org/resource/New_Orleans_Brass, http://dbpedia.org/resource/Genève-Servette_HC, http://dbpedia.org/resource/ZSC_Lions, http://dbpedia.org/resource/Liiga, http://dbpedia.org/resource/JYP_Jyväskylä, http://dbpedia.org/resource/EHC_Basel, http://dbpedia.org/resource/National_League_B, http://dbpedia.org/resource/Lausanne_HC, http://dbpedia.org/resource/EHC_Biel, http://dbpedia.org/resource/EHC_Visp, http://dbpedia.org/resource/Indianapolis_Racers, http://dbpedia.org/resource/Tidewater_Sharks, http://dbpedia.org/resource/Mohawk_Valley_Comets, http://dbpedia.org/resource/Lake_Erie_Monsters, http://dbpedia.org/resource/Grand_Rapids_Griffins, http://dbpedia.org/resource/Iowa_Wild, http://dbpedia.org/resource/Hamilton_Bulldogs_(AHL), http://dbpedia.org/resource/Heilbronner_Falken, http://dbpedia.org/resource/Saint-Hyacinthe_Chiefs, http://dbpedia.org/resource/Saint-Hyacinthe_Laser, http://dbpedia.org/resource/Verdun_Dragons, http://dbpedia.org/resource/Dragons_de_Rouen, http://dbpedia.org/resource/Granby_Prédateurs, http://dbpedia.org/resource/Trois-Rivières_Draveurs, http://dbpedia.org/resource/EV_Landsberg, http://dbpedia.org/resource/HC_Fassa, http://dbpedia.org/resource/Greensboro_Monarchs, http://dbpedia.org/resource/Montreal_Roadrunners, http://dbpedia.org/resource/Krefeld_Pinguine, http://dbpedia.org/resource/Thetford_Mines_Isothermic, http://dbpedia.org/resource/TSV_Kottern, http://dbpedia.org/resource/Ice_hockey_in_Germany, http://dbpedia.org/resource/Muskegon_Lumberjacks_(1992–2010), http://dbpedia.org/resource/EHC_München, http://dbpedia.org/resource/SC_Riessersee, http://dbpedia.org/resource/Toronto_Tecumsehs, http://dbpedia.org/resource/Montreal_Montagnards, http://dbpedia.org/resource/Calumet_Miners, http://dbpedia.org/resource/ERC_Ingolstadt, http://dbpedia.org/resource/Iserlohn_Roosters, http://dbpedia.org/resource/Arizona_Coyotes, http://dbpedia.org/resource/Eastern_Hockey_League, http://dbpedia.org/resource/Erie_Blades, http://dbpedia.org/resource/HC_Davos, http://dbpedia.org/resource/Birmingham_Bulls, http://dbpedia.org/resource/Quebec_Major_Junior_Hockey_League, http://dbpedia.org/resource/Western_Women's_Hockey_League, http://dbpedia.org/resource/Minnesota_Whitecaps, http://dbpedia.org/resource/Pensacola_Ice_Pilots, http://dbpedia.org/resource/Reno_Renegades, http://dbpedia.org/resource/West_Coast_Hockey_League, http://dbpedia.org/resource/Austrian_National_League, http://dbpedia.org/resource/Flint_Generals, http://dbpedia.org/resource/Ottawa_Wheels, http://dbpedia.org/resource/Port_Huron_Icehawks, http://dbpedia.org/resource/International_Hockey_League_(1945–2001), http://dbpedia.org/resource/Las_Vegas_Thunder, http://dbpedia.org/resource/Roller_Hockey_International, http://dbpedia.org/resource/Knoxville_Cherokees, http://dbpedia.org/resource/Atlanta_Knights, http://dbpedia.org/resource/St._Paul_Rangers, http://dbpedia.org/resource/Central_Professional_Hockey_League, http://dbpedia.org/resource/Los_Angeles_Blades_(WHL), http://dbpedia.org/resource/Cleveland_Barons_(1937–73), http://dbpedia.org/resource/Nova_Scotia_Voyageurs, http://dbpedia.org/resource/Maine_Mariners, http://dbpedia.org/resource/Durham_Wasps, http://dbpedia.org/resource/Salt_Lake_Golden_Eagles, http://dbpedia.org/resource/British_Hockey_League, http://dbpedia.org/resource/World_Hockey_Association, http://dbpedia.org/resource/Modo_Hockey, http://dbpedia.org/resource/Muskegon_Mohawks, http://dbpedia.org/resource/Quebec_Senior_Hockey_League, http://dbpedia.org/resource/Jersey_Devils, http://dbpedia.org/resource/Memphis_South_Stars, http://dbpedia.org/resource/Memphis_Wings, http://dbpedia.org/resource/Shawinigan-Falls_Cataracts, http://dbpedia.org/resource/Montreal_Maroons, http://dbpedia.org/resource/Syracuse_Orange_women's_ice_hockey, http://dbpedia.org/resource/HC_Mont_Blanc, http://dbpedia.org/resource/HC_Tours, http://dbpedia.org/resource/Megeve_(ice_hockey), http://dbpedia.org/resource/Texas_Stars, http://dbpedia.org/resource/Lahti_Pelicans, http://dbpedia.org/resource/Hockey_Milano_Rossoblu, http://dbpedia.org/resource/1998–99_NHL_season, http://dbpedia.org/resource/2000–01_NHL_season, http://dbpedia.org/resource/Seattle_Totems, http://dbpedia.org/resource/Charlotte_Checkers_(1956–77), http://dbpedia.org/resource/Syracuse_Blazers, http://dbpedia.org/resource/Atlanta_Flames, http://dbpedia.org/resource/Providence_Bruins, http://dbpedia.org/resource/Adirondack_Phantoms, http://dbpedia.org/resource/Lehigh_Valley_Phantoms, http://dbpedia.org/resource/Kontinental_Hockey_League, http://dbpedia.org/resource/Deutsche_Eishockey_Liga, http://dbpedia.org/resource/Quad_City_Mallards, http://dbpedia.org/resource/Chesapeake_Icebreakers, http://dbpedia.org/resource/Elite_Ice_Hockey_League, http://dbpedia.org/resource/Braehead_Clan, http://dbpedia.org/resource/Chicoutimi_Saguenéens, http://dbpedia.org/resource/Sherbrooke_Castors, http://dbpedia.org/resource/Louisiana_IceGators_(SPHL), http://dbpedia.org/resource/Fort_Wayne_Komets, http://dbpedia.org/resource/Wichita_Thunder, http://dbpedia.org/resource/Nottingham_Panthers, http://dbpedia.org/resource/Central_Hockey_League, http://dbpedia.org/resource/Danbury_Trashers, http://dbpedia.org/resource/Dornbirner_EC, http://dbpedia.org/resource/Hampton_Roads_Admirals, http://dbpedia.org/resource/Halifax_Mooseheads, http://dbpedia.org/resource/Victoriaville_Tigres, http://dbpedia.org/resource/Beauport_Harfangs, http://dbpedia.org/resource/Drummondville_Voltigeurs, http://dbpedia.org/resource/Long_Beach_Ice_Dogs, http://dbpedia.org/resource/Tacoma_Sabercats, http://dbpedia.org/resource/Kloten_Flyers, http://dbpedia.org/resource/Detroit_Vipers, http://dbpedia.org/resource/Hull_Thunder, http://dbpedia.org/resource/Kalamazoo_Wings_(1974–2000), http://dbpedia.org/resource/Pont_Rouge_Lois_Jeans, http://dbpedia.org/resource/Manchester_Storm_(1995–2002), http://dbpedia.org/resource/Saint-Georges_Cool_FM_103.5, http://dbpedia.org/resource/St._Marie_Poutrelles_Delta, http://dbpedia.org/resource/Cleveland_Lumberjacks, http://dbpedia.org/resource/Phoenix_Roadrunners_(IHL), http://dbpedia.org/resource/EHC_Black_Wings_Linz, http://dbpedia.org/resource/Adler_Mannheim, http://dbpedia.org/resource/Füchse_Duisburg, http://dbpedia.org/resource/Slovak_Extraliga, http://dbpedia.org/resource/HPK, http://dbpedia.org/resource/Columbus_Cottonmouths, http://dbpedia.org/resource/Lexington_Men_O'_War, http://dbpedia.org/resource/Utah_Grizzlies, http://dbpedia.org/resource/Quebec_Citadelles, http://dbpedia.org/resource/Michigan_Stags, http://dbpedia.org/resource/EC_Red_Bull_Salzburg, http://dbpedia.org/resource/EC_VSV, http://dbpedia.org/resource/Laval_Titan, http://dbpedia.org/resource/KHL_Medveščak_Zagreb, http://dbpedia.org/resource/Western_Professional_Hockey_League, http://dbpedia.org/resource/Adirondack_Frostbite, http://dbpedia.org/resource/Columbia_Inferno, http://dbpedia.org/resource/El_Paso_Buzzards, http://dbpedia.org/resource/San_Diego_Gulls_(1995–2006), http://dbpedia.org/resource/Cornell_Big_Red_women's_ice_hockey, http://dbpedia.org/resource/SG_Cortina, http://dbpedia.org/resource/HC_Varese, http://dbpedia.org/resource/HC_Gherdëina, http://dbpedia.org/resource/HCB_South_Tyrol, http://dbpedia.org/resource/HC_Milano, http://dbpedia.org/resource/HC_Pustertal_Wölfe, http://dbpedia.org/resource/ESV_Kaufbeuren, http://dbpedia.org/resource/SaiPa, http://dbpedia.org/resource/Toronto_Marlies, http://dbpedia.org/resource/Asia_League_Ice_Hockey, http://dbpedia.org/resource/Quad_City_Flames, http://dbpedia.org/resource/Nippon_Paper_Cranes, http://dbpedia.org/resource/Texas_Wildcatters, http://dbpedia.org/resource/Las_Vegas_Wranglers, http://dbpedia.org/resource/Chicago_Express, http://dbpedia.org/resource/Atlanta_Gladiators, http://dbpedia.org/resource/South_Carolina_Stingrays, http://dbpedia.org/resource/Dresdner_Eislöwen, http://dbpedia.org/resource/Ravensburg_Towerstars, http://dbpedia.org/resource/Charlotte_Checkers, http://dbpedia.org/resource/Utica_Comets, http://dbpedia.org/resource/Albany_River_Rats, http://dbpedia.org/resource/Toronto_St._Patricks, http://dbpedia.org/resource/Rapid_City_Rush, http://dbpedia.org/resource/Stockton_Thunder, http://dbpedia.org/resource/Arizona_Sundogs, http://dbpedia.org/resource/Denver_Cutthroats, http://dbpedia.org/resource/Chicago_Cougars, http://dbpedia.org/resource/Rhode_Island_Eagles, http://dbpedia.org/resource/Long_Island_Cougars, http://dbpedia.org/resource/Long_Island_Ducks_(ice_hockey), http://dbpedia.org/resource/Des_Moines_Oak_Leafs, http://dbpedia.org/resource/Trail_Smoke_Eaters, http://dbpedia.org/resource/Montreal_Royals, http://dbpedia.org/resource/Houston_Aeros_(WHA), http://dbpedia.org/resource/New_York_Golden_Blades, http://dbpedia.org/resource/Maine_Nordiques, http://dbpedia.org/resource/North_American_Hockey_League_(1973–77), http://dbpedia.org/resource/Halifax_Crescents, http://dbpedia.org/resource/Montreal_Le_National, http://dbpedia.org/resource/Austin_Ice_Bats, http://dbpedia.org/resource/Kansas_City_Blades, http://dbpedia.org/resource/Austrian_Hockey_League, http://dbpedia.org/resource/EC_KAC, http://dbpedia.org/resource/Binghamton_Rangers, http://dbpedia.org/resource/Cincinnati_Mohawks, http://dbpedia.org/resource/Cincinnati_Stingers, http://dbpedia.org/resource/Eisbären_Berlin, http://dbpedia.org/resource/St._Louis_Eagles, http://dbpedia.org/resource/Serie_A, http://dbpedia.org/resource/Asiago_Hockey_1935, http://dbpedia.org/resource/Boston_Braves_(AHL), http://dbpedia.org/resource/Dayton_Gems, http://dbpedia.org/resource/Binghamton_Dusters, http://dbpedia.org/resource/Diables_Noirs_de_Tours, http://dbpedia.org/resource/Alleghe_Hockey, http://dbpedia.org/resource/Phoenix_RoadRunners, http://dbpedia.org/resource/Shawinigan_Cataractes, http://dbpedia.org/resource/Peoria_Rivermen_(ECHL), http://dbpedia.org/resource/Dundee_Stars, http://dbpedia.org/resource/Adirondack_Flames, http://dbpedia.org/resource/Frederikshavn_White_Hawks, http://dbpedia.org/resource/Florida_Everblades, http://dbpedia.org/resource/Odense_Bulldogs, http://dbpedia.org/resource/Iowa_Stars, http://dbpedia.org/resource/Oklahoma_City_Blazers, http://dbpedia.org/resource/Saginaw_Gears_(UHL), http://dbpedia.org/resource/Toledo_Goaldiggers, http://dbpedia.org/resource/Milano_Vipers, http://dbpedia.org/resource/Calgary_Cowboys, http://dbpedia.org/resource/Western_Canada_Hockey_League, http://dbpedia.org/resource/Quebec_Castors, http://dbpedia.org/resource/Ontario_Professional_Hockey_League, http://dbpedia.org/resource/Phoenix_Roadrunners_(WHA), http://dbpedia.org/resource/Knoxville_Knights, http://dbpedia.org/resource/Nashville_Dixie_Flyers, http://dbpedia.org/resource/San_Francisco_Seals_(ice_hockey), http://dbpedia.org/resource/Ducs_d'Angers, http://dbpedia.org/resource/Abbotsford_Heat, http://dbpedia.org/resource/Oklahoma_City_Barons, http://dbpedia.org/resource/New_Brunswick_Hawks, http://dbpedia.org/resource/Sherbrooke_Jets, http://dbpedia.org/resource/Los_Angeles_Sharks, http://dbpedia.org/resource/Brûleurs_de_Loups, http://dbpedia.org/resource/Toronto_Toros, http://dbpedia.org/resource/Ligue_Magnus, http://dbpedia.org/resource/IF_Frisk_Asker, http://dbpedia.org/resource/GET-ligaen, http://dbpedia.org/resource/Missouri_Mavericks, http://dbpedia.org/resource/Sunshine_Hockey_League, http://dbpedia.org/resource/Daytona_Beach_Breakers, http://dbpedia.org/resource/HC_Ambrì-Piotta, http://dbpedia.org/resource/HC_Font-Romeu, http://dbpedia.org/resource/San_Diego_Gulls_(1966–74), http://dbpedia.org/resource/FFHG_Division_1, http://dbpedia.org/resource/Laredo_Bucks, http://dbpedia.org/resource/Colorado_Eagles, http://dbpedia.org/resource/Jackson_Bandits, http://dbpedia.org/resource/Bloomington_Thunder_(SPHL), http://dbpedia.org/resource/Topeka_ScareCrows, http://dbpedia.org/resource/New_Haven_Knights, http://dbpedia.org/resource/Bisons_de_Neuilly-sur-Marne, http://dbpedia.org/resource/Elmira_Jackals, http://dbpedia.org/resource/Monroe_Moccasins, http://dbpedia.org/resource/Kölner_Haie, http://dbpedia.org/resource/Lakeland_Prowlers, http://dbpedia.org/resource/St._Thomas_Wildcats, http://dbpedia.org/resource/Jacksonville_Bullets, http://dbpedia.org/resource/Green_Bay_Ice, http://dbpedia.org/resource/Carolina_Thunderbirds, http://dbpedia.org/resource/Philadelphia_Blazers, http://dbpedia.org/resource/Cape_Breton_Oilers, http://dbpedia.org/resource/Ak_Bars_Kazan, http://dbpedia.org/resource/Hamilton_Canucks, http://dbpedia.org/resource/Avtomobilist_Yekaterinburg, http://dbpedia.org/resource/Avangard_Omsk, http://dbpedia.org/resource/Kalamazoo_Wings, http://dbpedia.org/resource/Kansas_City_Red_Wings, http://dbpedia.org/resource/Montreal_Westmount, http://dbpedia.org/resource/Örebro_HK, http://dbpedia.org/resource/Pittsburgh_Shamrocks, http://dbpedia.org/resource/Brynäs_IF, http://dbpedia.org/resource/Raleigh_IceCaps, http://dbpedia.org/resource/Tappara, http://dbpedia.org/resource/Victoria_Salmon_Kings, http://dbpedia.org/resource/Ayr_Scottish_Eagles, http://dbpedia.org/resource/Orlando_Solar_Bears_(IHL), http://dbpedia.org/resource/HC_Merano, http://dbpedia.org/resource/Coventry_Blaze, http://dbpedia.org/resource/Indianapolis_Ice, http://dbpedia.org/resource/Toledo_Storm, http://dbpedia.org/resource/Cleveland_Crusaders, http://dbpedia.org/resource/HDD_Olimpija_Ljubljana, http://dbpedia.org/resource/Winnipeg_Maple_Leafs, http://dbpedia.org/resource/Montreal_Crystals, http://dbpedia.org/resource/Pittsburgh_Athletic_Club, http://dbpedia.org/resource/Portage_Lakes_Hockey_Club, http://dbpedia.org/resource/Pittsburgh_Bankers, http://dbpedia.org/resource/Pittsburgh_Professionals, http://dbpedia.org/resource/Cobalt_Silver_Kings, http://dbpedia.org/resource/Hannover_Scorpions, http://dbpedia.org/resource/HIFK_(ice_hockey), http://dbpedia.org/resource/SCL_Tigers, http://dbpedia.org/resource/Wembley_Lions, http://dbpedia.org/resource/Brighton_Tigers, http://dbpedia.org/resource/Earls_Court_Rangers, http://dbpedia.org/resource/Royal_Montreal_Hockey_Club, http://dbpedia.org/resource/Ottawa_Senators_(senior_hockey), http://dbpedia.org/resource/Jacksonville_Rockets, http://dbpedia.org/resource/New_Haven_Blades, http://dbpedia.org/resource/Winnipeg_Jets, http://dbpedia.org/resource/Dundee_Rockets, http://dbpedia.org/resource/McGill_University, http://dbpedia.org/resource/Saint_John_Sea_Dogs, http://dbpedia.org/resource/Canadian_Interuniversity_Sport, http://dbpedia.org/resource/Baie-Comeau_Drakkar, http://dbpedia.org/resource/Swindon_Wildcats, http://dbpedia.org/resource/Murrayfield_Racers, http://dbpedia.org/resource/Fife_Flyers, http://dbpedia.org/resource/San_Antonio_Dragons, http://dbpedia.org/resource/Tupelo_T-Rex, http://dbpedia.org/resource/San_Angelo_Saints, http://dbpedia.org/resource/Rivière-du-Loup_Promutuel, http://dbpedia.org/resource/Alaska_Aces_(ECHL), http://dbpedia.org/resource/New_Mexico_Scorpions, http://dbpedia.org/resource/Long_Beach_Sharks/Rockets, http://dbpedia.org/resource/Johnstown_Red_Wings, http://dbpedia.org/resource/Phoenix_Roadrunners_(PHL), http://dbpedia.org/resource/Dallas_Black_Hawks, http://dbpedia.org/resource/Pacific_Hockey_League, http://dbpedia.org/resource/Clinton_Comets, http://dbpedia.org/resource/Long_Island_Ducks, http://dbpedia.org/resource/Philadelphia_Ramblers, http://dbpedia.org/resource/Metallurg_Magnitogorsk, http://dbpedia.org/resource/Lørenskog_IK, http://dbpedia.org/resource/HC_Thurgau, http://dbpedia.org/resource/HC_Martigny, http://dbpedia.org/resource/SC_Langenthal, http://dbpedia.org/resource/Brantford_Smoke, http://dbpedia.org/resource/Winston-Salem_IceHawks, http://dbpedia.org/resource/HC_Slavia_Praha, http://dbpedia.org/resource/Peoria_Rivermen_(AHL), http://dbpedia.org/resource/Mississippi_Sea_Wolves, http://dbpedia.org/resource/San_Diego_Mariners, http://dbpedia.org/resource/St._John's_Maple_Leafs, http://dbpedia.org/resource/Vancouver_Millionaires, http://dbpedia.org/resource/Michigan_Soo_Indians, http://dbpedia.org/resource/Cleveland_Barons_(NHL), http://dbpedia.org/resource/Colorado_Rockies, http://dbpedia.org/resource/Binghamton_Whalers, http://dbpedia.org/resource/Serie_A_(ice_hockey), http://dbpedia.org/resource/Herisau_HC, http://dbpedia.org/resource/Revier_Lions, http://dbpedia.org/resource/Sorel_Éperviers, http://dbpedia.org/resource/Montreal_Junior_Canadiens, http://dbpedia.org/resource/Philadelphia_Firebirds_(ice_hockey), http://dbpedia.org/resource/HC_Lada_Togliatti, http://dbpedia.org/resource/SønderjyskE_Ishockey, http://dbpedia.org/resource/Oberliga_(ice_hockey), http://dbpedia.org/resource/Stjernen_Hockey, http://dbpedia.org/resource/Leksands_IF, http://dbpedia.org/resource/IF_Troja/Ljungby, http://dbpedia.org/resource/HockeyAllsvenskan, http://dbpedia.org/resource/Malmö_Redhawks, http://dbpedia.org/resource/Atlant_Moscow_Oblast, http://dbpedia.org/resource/HC_Dynamo_Moscow, http://dbpedia.org/resource/Lukko, http://dbpedia.org/resource/Minneapolis_Bruins, http://dbpedia.org/resource/Kansas_City_Blues_(ice_hockey), http://dbpedia.org/resource/Amarillo_Wranglers_(1968–71), http://dbpedia.org/resource/Oklahoma_City_Blazers_(1965–77), http://dbpedia.org/resource/Cincinnati_Swords, http://dbpedia.org/resource/Färjestad_BK, http://dbpedia.org/resource/Munich_Barons, http://dbpedia.org/resource/Belfast_Giants, http://dbpedia.org/resource/Brampton_Beast, http://dbpedia.org/resource/Hampton_Gulls, http://dbpedia.org/resource/Wembley_Monarchs, http://dbpedia.org/resource/Ayr_Raiders_(1946_—_1956), http://dbpedia.org/resource/IK_Viking, http://dbpedia.org/resource/Muskegon_Lumberjacks_(1984–92), http://dbpedia.org/resource/Idaho_Steelheads, http://dbpedia.org/resource/Louisville_RiverFrogs, http://dbpedia.org/resource/Waco_Wizards, http://dbpedia.org/resource/HC_TPS, http://dbpedia.org/resource/HC_Plzeň, http://dbpedia.org/resource/HC_Eppan_Pirates, http://dbpedia.org/resource/Southern_Hockey_League_(1973–77), http://dbpedia.org/resource/Macon_Whoopees_(SHL), http://dbpedia.org/resource/Greensboro_Generals, http://dbpedia.org/resource/Baltimore_Bandits, http://dbpedia.org/resource/Moncton_Hawks, http://dbpedia.org/resource/Fort_Worth_Texans, http://dbpedia.org/resource/New_Westminster_Royals, http://dbpedia.org/resource/Newark_Bulldogs, http://dbpedia.org/resource/Muskegon_Zephyrs, http://dbpedia.org/resource/Valleyfield_Braves, http://dbpedia.org/resource/Twin_City_Cyclones, http://dbpedia.org/resource/Southern_Professional_Hockey_League, http://dbpedia.org/resource/Baton_Rouge_Kingfish, http://dbpedia.org/resource/Greenville_Grrrowl, http://dbpedia.org/resource/Jacksonville_Lizard_Kings, http://dbpedia.org/resource/Pee_Dee_Pride, http://dbpedia.org/resource/Moncton_Wildcats, http://dbpedia.org/resource/Roanoke_Valley_Vipers, http://dbpedia.org/resource/Port_Huron_Border_Cats, http://dbpedia.org/resource/HC_Valpellice, http://dbpedia.org/resource/Gatineau_Olympiques, http://dbpedia.org/resource/Manchester_Phoenix, http://dbpedia.org/resource/Kentucky_Thoroughblades, http://dbpedia.org/resource/Charlotte_Checkers_(1993–2010), http://dbpedia.org/resource/Port_Huron_Beacons, http://dbpedia.org/resource/Newcastle_Vipers, http://dbpedia.org/resource/San_Francisco_Spiders, http://dbpedia.org/resource/HC_Sibir_Novosibirsk, http://dbpedia.org/resource/Södertälje_SK, http://dbpedia.org/resource/Omaha_Knights, http://dbpedia.org/resource/Toledo_Blades, http://dbpedia.org/resource/Indianapolis_Checkers, http://dbpedia.org/resource/Rochester_Cardinals, http://dbpedia.org/resource/Richmond_Robins, http://dbpedia.org/resource/Pittsburgh_Lyceum, http://dbpedia.org/resource/Ottawa_Nationals, http://dbpedia.org/resource/Broome_Dusters, http://dbpedia.org/resource/St._John's_IceCaps, http://dbpedia.org/resource/Toledo_Walleye, http://dbpedia.org/resource/Ducs_de_Dijon, http://dbpedia.org/resource/Albany_Choppers, http://dbpedia.org/resource/Pittsburgh_Phantoms_(RHI), http://dbpedia.org/resource/Montana_Magic, http://dbpedia.org/resource/Tucson_Gila_Monsters, http://dbpedia.org/resource/Minnesota–Duluth_Bulldogs_women's_ice_hockey, http://dbpedia.org/resource/Boston_University_Terriers_women's_ice_hockey, http://dbpedia.org/resource/SG_Pontebba, http://dbpedia.org/resource/Arlan_Kokshetau, http://dbpedia.org/resource/Columbus_Owls, http://dbpedia.org/resource/Ässät, http://dbpedia.org/resource/Cornwall_Cookies, http://dbpedia.org/resource/Quebec_Provincial_Hockey_League, http://dbpedia.org/resource/Rögle_BK, http://dbpedia.org/resource/EHC_Wolfsburg, http://dbpedia.org/resource/Tønsberg_Vikings, http://dbpedia.org/resource/Jonquière_Condors, http://dbpedia.org/resource/Saguenay_Fjord_(hockey), http://dbpedia.org/resource/Ottawa_Civics, http://dbpedia.org/resource/Columbus_Checkers, http://dbpedia.org/resource/Portland_Buckaroos, http://dbpedia.org/resource/Utica_Devils, http://dbpedia.org/resource/New_Haven_Senators, http://dbpedia.org/resource/HC_Milano_Saima, http://dbpedia.org/resource/Flint_Generals_(1969–85), http://dbpedia.org/resource/HC_Milano_24, http://dbpedia.org/resource/HC_Auronzo, http://dbpedia.org/resource/Montreal_Lacrosse_Club, http://dbpedia.org/resource/Beauce_Jaros, http://dbpedia.org/resource/Ilves, http://dbpedia.org/resource/Corpus_Christi_IceRays_(1998–2010), http://dbpedia.org/resource/Rivière-du-Loup_3L, http://dbpedia.org/resource/Sherbrooke_Saint-François, http://dbpedia.org/resource/St._Charles_Chill, http://dbpedia.org/resource/Rødovre_Mighty_Bulls, http://dbpedia.org/resource/VIK_Västerås_HK, http://dbpedia.org/resource/British_National_League_(1996–2005), http://dbpedia.org/resource/Bracknell_Bees, http://dbpedia.org/resource/Providence_Friars_women's_ice_hockey, http://dbpedia.org/resource/Montreal_Axion, http://dbpedia.org/resource/Concordia_Stingers_women's_ice_hockey, http://dbpedia.org/resource/Odessa_Jackalopes, http://dbpedia.org/resource/Cardiff_Devils, http://dbpedia.org/resource/HK_MK_Bled, http://dbpedia.org/resource/HK_Vojvodina, http://dbpedia.org/resource/Friesland_Flyers, http://dbpedia.org/resource/Bowdoin_College, http://dbpedia.org/resource/Missouri_River_Otters, http://dbpedia.org/resource/Swedish_Hockey_League, http://dbpedia.org/resource/Fresno_Falcons, http://dbpedia.org/resource/Skellefteå_AIK, http://dbpedia.org/resource/Richmond_Renegades, http://dbpedia.org/resource/Knoxville_Ice_Bears, http://dbpedia.org/resource/Colorado_Gold_Kings, http://dbpedia.org/resource/Alabama_Slammers, http://dbpedia.org/resource/Atlantic_Coast_Hockey_League_(2002–03), http://dbpedia.org/resource/World_Hockey_Association_2, http://dbpedia.org/resource/Central_Texas_Stampede, http://dbpedia.org/resource/Quad_City_Mallards_(1995–2007), http://dbpedia.org/resource/San_Antonio_Iguanas, http://dbpedia.org/resource/Hull-Ottawa_Canadiens, http://dbpedia.org/resource/Eastern_Professional_Hockey_League_(1959–63), http://dbpedia.org/resource/Phoenix_Roadrunners_(WHL), http://dbpedia.org/resource/Victoria_Maple_Leafs, http://dbpedia.org/resource/Djurgårdens_IF_Hockey, http://dbpedia.org/resource/2nd_Bundesliga_(ice_hockey), http://dbpedia.org/resource/Vaasan_Sport, http://dbpedia.org/resource/Sainte-Marie_Poutrelles_Delta, http://dbpedia.org/resource/Utah_Grizzlies_(1995–2005), http://dbpedia.org/resource/Kansas_City_Outlaws, http://dbpedia.org/resource/Mestis, http://dbpedia.org/resource/Mora_IK, http://dbpedia.org/resource/Augusta_Lynx, http://dbpedia.org/resource/Lillehammer_IK, http://dbpedia.org/resource/Lausitzer_Füchse, http://dbpedia.org/resource/LaSalle_Rapides, http://dbpedia.org/resource/Lachute_Rapides, http://dbpedia.org/resource/Flint_Spirits, http://dbpedia.org/resource/International_Professional_Hockey_League, http://dbpedia.org/resource/Western_Pennsylvania_Hockey_League, http://dbpedia.org/resource/Maritime_Professional_Hockey_League, http://dbpedia.org/resource/STS_Sanok, http://dbpedia.org/resource/Toros_Neftekamsk, http://dbpedia.org/resource/GKS_Katowice_(ice_hockey), http://dbpedia.org/resource/Orlando_Solar_Bears_(ECHL), http://dbpedia.org/resource/Tulsa_Oilers_(1964–84), http://dbpedia.org/resource/McGill_Redmen, http://dbpedia.org/resource/Port_Huron_Flags, http://dbpedia.org/resource/Dayton_Owls, http://dbpedia.org/resource/Grand_Rapids_Owls, http://dbpedia.org/resource/Albuquerque_Six-Guns, http://dbpedia.org/resource/Denver_Spurs, http://dbpedia.org/resource/Schwenninger_Wild_Wings, http://dbpedia.org/resource/Houston_Apollos, http://dbpedia.org/resource/Trois-Rivières_Lions, http://dbpedia.org/resource/Los_Angeles_Blades, http://dbpedia.org/resource/Saskatoon_Quakers, http://dbpedia.org/resource/Calgary_Stampeders, http://dbpedia.org/resource/Saskatoon/St._Paul_Regals, http://dbpedia.org/resource/Glace_Bay_Jr._Miners, http://dbpedia.org/resource/Grand_Rapids_Grizzlies] [INFO] Predicted Answer: [http://dbpedia.org/resource/André_Bureau, http://dbpedia.org/resource/Anke_Engelke, http://dbpedia.org/resource/Don_Ferguson_(actor), http://dbpedia.org/resource/Henry_Brant, http://dbpedia.org/resource/Jon_Lajoie, http://dbpedia.org/resource/Jérôme_Choquette, http://dbpedia.org/resource/Kevin_McDonald, http://dbpedia.org/resource/Lucio_Amanti, http://dbpedia.org/resource/Mike_Ward_(comedian), http://dbpedia.org/resource/Pierre_Camu, http://dbpedia.org/resource/Sugar_Sammy, http://dbpedia.org/resource/André_Arthur, http://dbpedia.org/resource/Anthony_Mantha, http://dbpedia.org/resource/Bryan_Murray_(ice_hockey), http://dbpedia.org/resource/Denis_Héroux, http://dbpedia.org/resource/Dickie_Moore_(ice_hockey), http://dbpedia.org/resource/Duke_Keats, http://dbpedia.org/resource/Edgar_Bronfman,_Sr., http://dbpedia.org/resource/Emmanuelle_Chriqui, http://dbpedia.org/resource/Ernie_McLea, http://dbpedia.org/resource/Germain_Gagnon, http://dbpedia.org/resource/Ian_Turnbull_(ice_hockey), http://dbpedia.org/resource/James_E._Norris, http://dbpedia.org/resource/John_Colicos, http://dbpedia.org/resource/Josée_Verner, http://dbpedia.org/resource/Kim_St-Pierre, http://dbpedia.org/resource/Laurence_Leboeuf, http://dbpedia.org/resource/Lester_Patrick, http://dbpedia.org/resource/Luc_Robitaille, http://dbpedia.org/resource/Lucien_Bouchard, http://dbpedia.org/resource/Mario_Dumont, http://dbpedia.org/resource/Marius_Barbeau, http://dbpedia.org/resource/Martin_Lapointe, http://dbpedia.org/resource/Michel_Plasse, http://dbpedia.org/resource/Mike_Bossy, http://dbpedia.org/resource/Monique_Miller, http://dbpedia.org/resource/Patrick_Roy, http://dbpedia.org/resource/Paul_Bloom_(psychologist), http://dbpedia.org/resource/Pierre_Dagenais, http://dbpedia.org/resource/Pit_Martin, http://dbpedia.org/resource/Serge_Lajeunesse, http://dbpedia.org/resource/Stéphane_Gendron, http://dbpedia.org/resource/Yves_Racine, http://dbpedia.org/resource/Kimveer_Gill, http://dbpedia.org/resource/Peter_Cullen, http://dbpedia.org/resource/Raymond_Garneau, http://dbpedia.org/resource/Renaude_Lapointe, http://dbpedia.org/resource/Albert_S._Ruddy, http://dbpedia.org/resource/Albert_Tessier, http://dbpedia.org/resource/Anais_Pouliot, http://dbpedia.org/resource/Aurore_Gagnon, http://dbpedia.org/resource/Bernard_Lemaire, http://dbpedia.org/resource/Bobby_Bazini, http://dbpedia.org/resource/Brian_Macdonald_(choreographer), http://dbpedia.org/resource/Camille_Laurin, http://dbpedia.org/resource/Caroline_Dhavernas, http://dbpedia.org/resource/Christine_Robinson, http://dbpedia.org/resource/Claude_Forget, http://dbpedia.org/resource/Claude_Raymond_(baseball), http://dbpedia.org/resource/Clément_Gignac, http://dbpedia.org/resource/David_Perron, http://dbpedia.org/resource/Elias_Koteas, http://dbpedia.org/resource/Francis_Lessard, http://dbpedia.org/resource/Gilles_Villemure, http://dbpedia.org/resource/Guy_Provost, http://dbpedia.org/resource/Henry_Corden, http://dbpedia.org/resource/Hugues_Lapointe, http://dbpedia.org/resource/Jamil_Azzaoui, http://dbpedia.org/resource/Janet_Anderson_Perkin, http://dbpedia.org/resource/Jean-Guy_Cardinal, http://dbpedia.org/resource/Jean-Sébastien_Giguère, http://dbpedia.org/resource/Jean_Cournoyer, http://dbpedia.org/resource/Jean_Lapierre, http://dbpedia.org/resource/Jeff_Deslauriers, http://dbpedia.org/resource/Joe_Malone_(ice_hockey), http://dbpedia.org/resource/Joseph_Wiseman, http://dbpedia.org/resource/Justine_Dufour-Lapointe, http://dbpedia.org/resource/L._Denis_Desautels, http://dbpedia.org/resource/Laurent_Barré, http://dbpedia.org/resource/Laurent_Beaudoin, http://dbpedia.org/resource/Liza_Frulla, http://dbpedia.org/resource/Lou_Brouillard, http://dbpedia.org/resource/Louis_Siminovitch, http://dbpedia.org/resource/Marc-André_Gragnani, http://dbpedia.org/resource/Marc-André_Grondin, http://dbpedia.org/resource/Marc-Édouard_Vlasic, http://dbpedia.org/resource/Marc_Bureau_(ice_hockey), http://dbpedia.org/resource/Marc_Chouinard, http://dbpedia.org/resource/Marc_Dos_Santos, http://dbpedia.org/resource/Martin_Brodeur, http://dbpedia.org/resource/Muffy_Calder, http://dbpedia.org/resource/Pascal_Denis, http://dbpedia.org/resource/Paul_Dozois, http://dbpedia.org/resource/Phil_Devey, http://dbpedia.org/resource/Pierre_Gauthier, http://dbpedia.org/resource/Pierre_Lapointe, http://dbpedia.org/resource/Richard_Desjardins, http://dbpedia.org/resource/Ruta_Lee, http://dbpedia.org/resource/Serge_Dupire, http://dbpedia.org/resource/Solange_Chaput-Rolland, http://dbpedia.org/resource/Stéphane_Bédard, http://dbpedia.org/resource/Stéphane_Robidas, http://dbpedia.org/resource/Yoan_Garneau, http://dbpedia.org/resource/Yves_Jacques, http://dbpedia.org/resource/Yves_Séguin, http://dbpedia.org/resource/Yvon_Lambert, http://dbpedia.org/resource/Édouard_Rinfret, http://dbpedia.org/resource/Charles_Doherty, http://dbpedia.org/resource/Lysiane_Gagnon, http://dbpedia.org/resource/Willis_Adcock, http://dbpedia.org/resource/Alex_Anthopoulos, http://dbpedia.org/resource/Lucile_Watson, http://dbpedia.org/resource/Robert_Lepage, http://dbpedia.org/resource/Cairine_Wilson, http://dbpedia.org/resource/Félix_Auger-Aliassime, http://dbpedia.org/resource/Lawrence_Arthur_Dumoulin_Cannon, http://dbpedia.org/resource/Luc-Normand_Tellier, http://dbpedia.org/resource/Michael_Applebaum, http://dbpedia.org/resource/Joseph_Rouleau, http://dbpedia.org/resource/Mark_Lutz_(actor), http://dbpedia.org/resource/Nicole_Jaffe, http://dbpedia.org/resource/Alexis_Wawanoloath, http://dbpedia.org/resource/Dark_Cloud_(actor), http://dbpedia.org/resource/Réal_Ménard, http://dbpedia.org/resource/Sean_Patrick_Maloney, http://dbpedia.org/resource/Jacques_La_Ramee, http://dbpedia.org/resource/Leo_Margolis, http://dbpedia.org/resource/Aline_Chrétien, http://dbpedia.org/resource/Céline_Hervieux-Payette, http://dbpedia.org/resource/Eddie_Goldenberg, http://dbpedia.org/resource/J._Michel_Doyon, http://dbpedia.org/resource/Louis-Joseph_Pigeon, http://dbpedia.org/resource/Louis_LeBel, http://dbpedia.org/resource/Marie_Deschamps, http://dbpedia.org/resource/Warren_Kinsella, http://dbpedia.org/resource/Edward_William_Archibald, http://dbpedia.org/resource/Paul_Schoeffler, http://dbpedia.org/resource/Gilles_Turcot, http://dbpedia.org/resource/Alexandre_Cloutier, http://dbpedia.org/resource/Benoit_Charette, http://dbpedia.org/resource/Guy_Chevrette, http://dbpedia.org/resource/Jonathan_Valois, http://dbpedia.org/resource/Karen_Cliche, http://dbpedia.org/resource/Lisette_Lapointe, http://dbpedia.org/resource/Martine_Ouellet, http://dbpedia.org/resource/Hélène_David, http://dbpedia.org/resource/Cory_Pecker, http://dbpedia.org/resource/David_Loiseau, http://dbpedia.org/resource/Guy_Hemmings, http://dbpedia.org/resource/Hy_Buller, http://dbpedia.org/resource/Jean-Michel_Ménard, http://dbpedia.org/resource/Kalyna_Roberge, http://dbpedia.org/resource/Kathy_Tremblay, http://dbpedia.org/resource/Marie-France_Larouche, http://dbpedia.org/resource/Ron_Crevier, http://dbpedia.org/resource/Ronnie_Stern, http://dbpedia.org/resource/Stacy_Harris, http://dbpedia.org/resource/Steve_Dubinsky, http://dbpedia.org/resource/Stewart_Granger_(basketball), http://dbpedia.org/resource/Charles_J._Ross, http://dbpedia.org/resource/Raymond_Castilloux, http://dbpedia.org/resource/Claude_Lanthier, http://dbpedia.org/resource/François_X._Matthieu, http://dbpedia.org/resource/Jim_Hrivnak, http://dbpedia.org/resource/Joseph_Gervais, http://dbpedia.org/resource/Lise_Zarac, http://dbpedia.org/resource/John_L._Jolley, http://dbpedia.org/resource/Martin_Wilk, http://dbpedia.org/resource/Maurice_Clermont, http://dbpedia.org/resource/Wilbert_Keon, http://dbpedia.org/resource/Colleen_Murphy, http://dbpedia.org/resource/Emily_Hampshire, http://dbpedia.org/resource/George_Trakas, http://dbpedia.org/resource/Gila_Martow, http://dbpedia.org/resource/John_Brassard, http://dbpedia.org/resource/John_McCallum, http://dbpedia.org/resource/Lloyd_Chadburn, http://dbpedia.org/resource/Mike_Hough, http://dbpedia.org/resource/George_Carlyle_Marler, http://dbpedia.org/resource/Idola_Saint-Jean, http://dbpedia.org/resource/Joseph_Bernier, http://dbpedia.org/resource/Rosemary_Sullivan, http://dbpedia.org/resource/Costas_Menegakis, http://dbpedia.org/resource/Jeremy_Kinsman, http://dbpedia.org/resource/Max_M._Teitelbaum, http://dbpedia.org/resource/Pierre_Bussières, http://dbpedia.org/resource/Roy_Cullen, http://dbpedia.org/resource/William_Campbell_James_Meredith, http://dbpedia.org/resource/Alain_Rochat, http://dbpedia.org/resource/Billy_Bell_(ice_hockey), http://dbpedia.org/resource/Charlie_Langlois, http://dbpedia.org/resource/Dave_Ritchie_(ice_hockey), http://dbpedia.org/resource/Dylan_Hunter, http://dbpedia.org/resource/Edmond_Bouchard, http://dbpedia.org/resource/Ernie_Russell, http://dbpedia.org/resource/François_Lacombe, http://dbpedia.org/resource/Harry_Hyland, http://dbpedia.org/resource/Hib_Milks, http://dbpedia.org/resource/Jack_Marshall_(ice_hockey), http://dbpedia.org/resource/James_Power_(ice_hockey), http://dbpedia.org/resource/Jeff_Malone_(ice_hockey), http://dbpedia.org/resource/Odie_Cleghorn, http://dbpedia.org/resource/Paddy_Moran_(ice_hockey), http://dbpedia.org/resource/Percy_LeSueur, http://dbpedia.org/resource/Percy_Quinn, http://dbpedia.org/resource/Phil_Stevens_(ice_hockey), http://dbpedia.org/resource/Tara_Spencer-Nairn, http://dbpedia.org/resource/Walter_Rooney, http://dbpedia.org/resource/Jason_Della_Rocca, http://dbpedia.org/resource/Michele_Scarabelli, http://dbpedia.org/resource/Alan_Haworth, http://dbpedia.org/resource/Alex_Bourret, http://dbpedia.org/resource/Dan_Ruimy, http://dbpedia.org/resource/Gordie_Haworth, http://dbpedia.org/resource/Guillaume_Tremblay, http://dbpedia.org/resource/Jacques_Cloutier, http://dbpedia.org/resource/Jean-François_Bérubé, http://dbpedia.org/resource/Jean_Berthiaume, http://dbpedia.org/resource/Louise_Bédard, http://dbpedia.org/resource/Marc-André_Cliche, http://dbpedia.org/resource/Mario_Laframboise, http://dbpedia.org/resource/Michel_Ouellet, http://dbpedia.org/resource/Nancy_Drolet, http://dbpedia.org/resource/Pierre-Luc_Létourneau-Leblond, http://dbpedia.org/resource/Rick_Genest, http://dbpedia.org/resource/Roland_Doré_(administrator), http://dbpedia.org/resource/Serge_Boisvert, http://dbpedia.org/resource/Sébastien_Demers, http://dbpedia.org/resource/Angelo_Tsarouchas, http://dbpedia.org/resource/Dan_Geoffrion, http://dbpedia.org/resource/Connie_Forey, http://dbpedia.org/resource/Alex_Gaumond, http://dbpedia.org/resource/Henry_Edward_Burstall, http://dbpedia.org/resource/Alexandre_Boulerice, http://dbpedia.org/resource/Isabelle_Morin, http://dbpedia.org/resource/Maxime_Arseneau, http://dbpedia.org/resource/Raynald_Blais, http://dbpedia.org/resource/Roger_Pomerleau, http://dbpedia.org/resource/Louise_Robey, http://dbpedia.org/resource/Albert_Gervais, http://dbpedia.org/resource/Annie_Bovaird, http://dbpedia.org/resource/Arthur_Larouche, http://dbpedia.org/resource/Danièle_Tremblay-Lamer, http://dbpedia.org/resource/Henry_Archer_Ekers, http://dbpedia.org/resource/Jacques_Dumoulin, http://dbpedia.org/resource/James_John_Edmund_Guerin, http://dbpedia.org/resource/Louis_Payette, http://dbpedia.org/resource/Pierre_Édouard_Blondin, http://dbpedia.org/resource/Réjean_Hébert, http://dbpedia.org/resource/Étienne-Théodore_Pâquet, http://dbpedia.org/resource/Benoit_Groulx_(ice_hockey), http://dbpedia.org/resource/Bob_Perreault, http://dbpedia.org/resource/Guy_Rousseau, http://dbpedia.org/resource/Nicolas_Lesage, http://dbpedia.org/resource/Christian_Dubé_(ice_hockey), http://dbpedia.org/resource/Stéphane_Beauregard, http://dbpedia.org/resource/Fabienne_Thibeault, http://dbpedia.org/resource/Howard_Alper, http://dbpedia.org/resource/Saira_Mohan, http://dbpedia.org/resource/Claude_Verret, http://dbpedia.org/resource/Denis_Hamel, http://dbpedia.org/resource/Steve_Maltais, http://dbpedia.org/resource/Francis_Scarpaleggia, http://dbpedia.org/resource/Joseph-Édouard-Émile_Léonard, http://dbpedia.org/resource/Lysane_Blanchette-Lamothe, http://dbpedia.org/resource/Léopold_Demers, http://dbpedia.org/resource/Nicole_Demers, http://dbpedia.org/resource/Paul_Paquet, http://dbpedia.org/resource/Rosane_Doré_Lefebvre, http://dbpedia.org/resource/Anju_Dhillon, http://dbpedia.org/resource/Anne_Minh-Thu_Quach, http://dbpedia.org/resource/Barry_Blitt, http://dbpedia.org/resource/Brandon_Reid, http://dbpedia.org/resource/Daniel_Corso, http://dbpedia.org/resource/Ed_Janiszewski, http://dbpedia.org/resource/Emma_Robinson_(rower), http://dbpedia.org/resource/Emmanuelle_Blais, http://dbpedia.org/resource/Fred_Burchell_(ice_hockey), http://dbpedia.org/resource/Jason_Demers, http://dbpedia.org/resource/Karine_Sergerie, http://dbpedia.org/resource/Larry_Archambeault, http://dbpedia.org/resource/Louis_Leblanc, http://dbpedia.org/resource/Marc_Denis_(radio_personality), http://dbpedia.org/resource/Matthiew_Klinck, http://dbpedia.org/resource/Pierre_Michaud, http://dbpedia.org/resource/Pierre_Paquette, http://dbpedia.org/resource/René_Matte, http://dbpedia.org/resource/Roger_Lapointe, http://dbpedia.org/resource/Yves_Ducharme, http://dbpedia.org/resource/Dany_Roussin, http://dbpedia.org/resource/Henri_Courtemanche, http://dbpedia.org/resource/Roch_Pinard, http://dbpedia.org/resource/Rodolphe_Monty, http://dbpedia.org/resource/Barbara_Dunkelman, http://dbpedia.org/resource/Bernard_Bigras, http://dbpedia.org/resource/Claude_DeBellefeuille, http://dbpedia.org/resource/Diane_Bourgeois, http://dbpedia.org/resource/Gilles_Morin, http://dbpedia.org/resource/Jean-Yves_Laforest, http://dbpedia.org/resource/Luc_Guindon, http://dbpedia.org/resource/Paule_Brunelle, http://dbpedia.org/resource/Robert_Bouchard, http://dbpedia.org/resource/Flora_Martínez, http://dbpedia.org/resource/Mylène_Dinh-Robic, http://dbpedia.org/resource/Denis_Chalifoux, http://dbpedia.org/resource/Derek_Frenette, http://dbpedia.org/resource/Eric_Lacroix, http://dbpedia.org/resource/Eric_Lamaze, http://dbpedia.org/resource/Frédéric_Cassivi, http://dbpedia.org/resource/Marco_Rizi, http://dbpedia.org/resource/Nicholas_Vachon, http://dbpedia.org/resource/Normand_Lacombe, http://dbpedia.org/resource/Shiraz_Adam, http://dbpedia.org/resource/Georges_Groulx, http://dbpedia.org/resource/Peter_Peter, http://dbpedia.org/resource/Daniel_Goneau, http://dbpedia.org/resource/Danny_Dupont, http://dbpedia.org/resource/Edward_Bronfman, http://dbpedia.org/resource/Frederic_Deschênes, http://dbpedia.org/resource/Joseph_McCormick_(ice_hockey), http://dbpedia.org/resource/Mathieu_Dandenault, http://dbpedia.org/resource/Steve_Plouffe, http://dbpedia.org/resource/Sylvain_Blouin, http://dbpedia.org/resource/Yanick_Dube, http://dbpedia.org/resource/David_De_La_Peralle, http://dbpedia.org/resource/Jabari_Arthur, http://dbpedia.org/resource/Michel_de_Broin, http://dbpedia.org/resource/Alain_Raymond, http://dbpedia.org/resource/André_Doré, http://dbpedia.org/resource/Guy_Rouleau_(ice_hockey), http://dbpedia.org/resource/Jean-Marc_Routhier, http://dbpedia.org/resource/Jean-Michel_Daoust, http://dbpedia.org/resource/Julie_Parisien, http://dbpedia.org/resource/Marc_LaBelle, http://dbpedia.org/resource/Martin_Simard, http://dbpedia.org/resource/Mathieu_Brunelle, http://dbpedia.org/resource/Steven_Fletcher_(ice_hockey), http://dbpedia.org/resource/Jen_Gould, http://dbpedia.org/resource/Jean_Tétreault, http://dbpedia.org/resource/Bob_Wood_(MP), http://dbpedia.org/resource/Boom_Desjardins, http://dbpedia.org/resource/Mortimer_Davis, http://dbpedia.org/resource/Warren_Sharples, http://dbpedia.org/resource/Ed_Masterson, http://dbpedia.org/resource/John_Mahaffy_(ice_hockey), http://dbpedia.org/resource/Martin_Crête, http://dbpedia.org/resource/Pierre_Daviault, http://dbpedia.org/resource/Brendan_Clouston, http://dbpedia.org/resource/Maude_Smith_Gagnon, http://dbpedia.org/resource/Nic_Romm, http://dbpedia.org/resource/Alain_Dansereau, http://dbpedia.org/resource/Chantal_Payer, http://dbpedia.org/resource/Fleurette_Campeau, http://dbpedia.org/resource/Marc_Blouin, http://dbpedia.org/resource/Michel_Vaillancourt, http://dbpedia.org/resource/Serge_Proulx, http://dbpedia.org/resource/Susan_Stewart_(fencer), http://dbpedia.org/resource/Charles_Albert_Boynton, http://dbpedia.org/resource/Kevin_Bizier, http://dbpedia.org/resource/Pierre-Luc_Hinse, http://dbpedia.org/resource/Evens_Gravel, http://dbpedia.org/resource/Gino_Brousseau, http://dbpedia.org/resource/Renée_Aubin, http://dbpedia.org/resource/Annick_Papillon, http://dbpedia.org/resource/Charles_Cannon_(Quebec_politician), http://dbpedia.org/resource/Claude_Cyr, http://dbpedia.org/resource/France_Dionne, http://dbpedia.org/resource/Gilles_Boisvert, http://dbpedia.org/resource/LuFisto, http://dbpedia.org/resource/Maurice_Rinfret, http://dbpedia.org/resource/Roger_Parizeau, http://dbpedia.org/resource/Frederic_Cloutier, http://dbpedia.org/resource/Jean-Francois_Houle, http://dbpedia.org/resource/José_Charbonneau, http://dbpedia.org/resource/Marty_Raymond, http://dbpedia.org/resource/Maxime_Macenauer, http://dbpedia.org/resource/Achille_Bergevin, http://dbpedia.org/resource/Alfred_Thibaudeau, http://dbpedia.org/resource/Daniel_Ratthé, http://dbpedia.org/resource/Stu_Phillips_(country_singer), http://dbpedia.org/resource/Christina_Broccolini, http://dbpedia.org/resource/Éric_Deslauriers, http://dbpedia.org/resource/Edgard_Jules_Wermenlinger, http://dbpedia.org/resource/Elphège_Marier, http://dbpedia.org/resource/Eugène_Desrochers, http://dbpedia.org/resource/Frederick_Robert_Cromwell, http://dbpedia.org/resource/Jean-Louis_Frenette, http://dbpedia.org/resource/Maurice_Hallé, http://dbpedia.org/resource/Vital_Mallette, http://dbpedia.org/resource/Walter_George_Mitchell, http://dbpedia.org/resource/Joseph_Veleno, http://dbpedia.org/resource/Yannick_Riendeau, http://dbpedia.org/resource/Alex_Biega_(ice_hockey), http://dbpedia.org/resource/Danny_Biega, http://dbpedia.org/resource/Eli_Langer, http://dbpedia.org/resource/Jonathan_Racine, http://dbpedia.org/resource/Marc-Antoine_(singer), http://dbpedia.org/resource/Marc_Calixte, http://dbpedia.org/resource/Maxime_Fortunus, http://dbpedia.org/resource/Mélissa_Laveaux, http://dbpedia.org/resource/Obed_Cétoute, http://dbpedia.org/resource/Pierre-Cédric_Labrie, http://dbpedia.org/resource/Pierre_Gage, http://dbpedia.org/resource/William_Henry_Pferinger_Elkins, http://dbpedia.org/resource/Caroline_Olivier, http://dbpedia.org/resource/Kelly_Wiltshire, http://dbpedia.org/resource/Gilles_Zolty, http://dbpedia.org/resource/Maurice_Bélanger, http://dbpedia.org/resource/Larry_Fairholm, http://dbpedia.org/resource/Jim_Chambers, http://dbpedia.org/resource/Charles_Otis, http://dbpedia.org/resource/Rhoda_Martin, http://dbpedia.org/resource/Adolphe_Marcoux, http://dbpedia.org/resource/Grégoire_Bélanger, http://dbpedia.org/resource/Rouville_Beaudry, http://dbpedia.org/resource/Martin_Gilbert_(cyclist), http://dbpedia.org/resource/Shawn_Delierre, http://dbpedia.org/resource/Barbara_J._Minshall, http://dbpedia.org/resource/Nico_Archambault, http://dbpedia.org/resource/Alexandre_Tremblay, http://dbpedia.org/resource/Charles_Constantin_(ice_hockey), http://dbpedia.org/resource/Michel_Dubois_(ice_hockey), http://dbpedia.org/resource/Randy_Burchell, http://dbpedia.org/resource/Joël_Chouinard, http://dbpedia.org/resource/Melissa_Galianos, http://dbpedia.org/resource/Marcel_Duval, http://dbpedia.org/resource/Peter_Roizen, http://dbpedia.org/resource/Antonin_Décarie, http://dbpedia.org/resource/Louis_Lagasse, http://dbpedia.org/resource/François-Joseph_Bissot, http://dbpedia.org/resource/Nicolas_Pinto, http://dbpedia.org/resource/Philippe-Andre_Moreau, http://dbpedia.org/resource/Robert_Alexander_Fyfe, http://dbpedia.org/resource/Gilbert_Larose, http://dbpedia.org/resource/Stephen_Allan_Scott, http://dbpedia.org/resource/Manix_Auriantal, http://dbpedia.org/resource/Janet_Jones_(artist), http://dbpedia.org/resource/Francis_Lemieux, http://dbpedia.org/resource/Éric_Bellerose, http://dbpedia.org/resource/Claude-Gilles_Gosselin, http://dbpedia.org/resource/Félix_Allard, http://dbpedia.org/resource/Michaël_Dubuc, http://dbpedia.org/resource/François-Xavier_Bélanger, http://dbpedia.org/resource/Frederick_Arthur_McKenzie, http://dbpedia.org/resource/Anne_Robillard, http://dbpedia.org/resource/Julie-Anne_Derome, http://dbpedia.org/resource/Keke_Mortson, http://dbpedia.org/resource/Irving_Corey, http://dbpedia.org/resource/Alex_Genest, http://dbpedia.org/resource/Alexandre_Émond, http://dbpedia.org/resource/Émilie_Fournel, http://dbpedia.org/resource/Robert_Ouellet, http://dbpedia.org/resource/Maurice_E._Lagacé, http://dbpedia.org/resource/Éric_Dandenault, http://dbpedia.org/resource/Jean_de_Brabant, http://dbpedia.org/resource/Moe_Segal, http://dbpedia.org/resource/Nick_Rosen_(filmmaker), http://dbpedia.org/resource/Alodie-Virginie_Paradis, http://dbpedia.org/resource/Dave_Noel-Bernier, http://dbpedia.org/resource/Disappearance_of_Cédrika_Provencher, http://dbpedia.org/resource/Ed_Longfellow, http://dbpedia.org/resource/Fred_Strike, http://dbpedia.org/resource/Wilf_Talbot, http://dbpedia.org/resource/Alfred_Lépine, http://dbpedia.org/resource/Anne_Hébert, http://dbpedia.org/resource/Bob_Fillion, http://dbpedia.org/resource/Camillien_Houde, http://dbpedia.org/resource/Colleen_Dewhurst, http://dbpedia.org/resource/Connie_Broden, http://dbpedia.org/resource/Daniel_Paillé, http://dbpedia.org/resource/David_Rakoff, http://dbpedia.org/resource/Dollard_St._Laurent, http://dbpedia.org/resource/Erik_Mongrain, http://dbpedia.org/resource/Félix_Leclerc, http://dbpedia.org/resource/Guy_Laliberté, http://dbpedia.org/resource/Guy_Trottier, http://dbpedia.org/resource/Gérald_Beaudoin, http://dbpedia.org/resource/Haji_(actress), http://dbpedia.org/resource/Jack_Layton, http://dbpedia.org/resource/Jacques_Parizeau, http://dbpedia.org/resource/Jean-Pierre_Dumont, http://dbpedia.org/resource/Jean_Le_Moyne, http://dbpedia.org/resource/Jennifer_Finnigan, http://dbpedia.org/resource/Jimmy_Waite, http://dbpedia.org/resource/Kevin_Lowe, http://dbpedia.org/resource/Marc_Denis, http://dbpedia.org/resource/Marcel_Dionne, http://dbpedia.org/resource/Marcel_Pronovost, http://dbpedia.org/resource/Mary_Pierce, http://dbpedia.org/resource/Michel_Barrette, http://dbpedia.org/resource/Michel_Chartrand, http://dbpedia.org/resource/Michel_Larocque, http://dbpedia.org/resource/Nicolas_Macrozonaris, http://dbpedia.org/resource/Paul_Almond, http://dbpedia.org/resource/Paul_Comtois, http://dbpedia.org/resource/Philippe_Panneton, http://dbpedia.org/resource/Pierre_Aubry, http://dbpedia.org/resource/Pierre_Pagé, http://dbpedia.org/resource/Ray_Condo, http://dbpedia.org/resource/Reginald_Fessenden, http://dbpedia.org/resource/Sylvain_Simard, http://dbpedia.org/resource/Vic_Vogel, http://dbpedia.org/resource/Albert_Chartier, http://dbpedia.org/resource/Alexandre_Daigle, http://dbpedia.org/resource/André_D'Allemagne, http://dbpedia.org/resource/Bernie_Geoffrion, http://dbpedia.org/resource/Eugenie_Bouchard, http://dbpedia.org/resource/Eva_Avila, http://dbpedia.org/resource/Fernand_Grenier, http://dbpedia.org/resource/François_Girard, http://dbpedia.org/resource/Gaston_Gingras, http://dbpedia.org/resource/Gregory_Charles, http://dbpedia.org/resource/Hans_Schmidt_(wrestler), http://dbpedia.org/resource/Harry_Saltzman, http://dbpedia.org/resource/Hazel_McCallion, http://dbpedia.org/resource/Henri_Richard, http://dbpedia.org/resource/Hélène_Dorion, http://dbpedia.org/resource/Jacques_Duchesneau, http://dbpedia.org/resource/Jean-François_Lisée, http://dbpedia.org/resource/Jean-Jacques_Bertrand, http://dbpedia.org/resource/Jim_Corcoran, http://dbpedia.org/resource/Julien_Chouinard, http://dbpedia.org/resource/Kate_McGarrigle, http://dbpedia.org/resource/Kim_Poirier, http://dbpedia.org/resource/Louis_Rasminsky, http://dbpedia.org/resource/Louise_Arbour, http://dbpedia.org/resource/Manon_Rhéaume, http://dbpedia.org/resource/Marc_Dufour, http://dbpedia.org/resource/Marianne_St-Gelais, http://dbpedia.org/resource/Mario_Pelchat, http://dbpedia.org/resource/Marlène_Harnois, http://dbpedia.org/resource/Mike_Ribeiro, http://dbpedia.org/resource/Patrick_Côté_(fighter), http://dbpedia.org/resource/Paul-Émile_Borduas, http://dbpedia.org/resource/Paul_Gérin-Lajoie, http://dbpedia.org/resource/Pierre_Bouchard, http://dbpedia.org/resource/Pierre_Laporte, http://dbpedia.org/resource/Randy_Velischek, http://dbpedia.org/resource/Raôul_Duguay, http://dbpedia.org/resource/Rina_Lasnier, http://dbpedia.org/resource/Serge_Aubin, http://dbpedia.org/resource/Serge_Joyal, http://dbpedia.org/resource/Simon_Gagné, http://dbpedia.org/resource/Sébastien_Lefebvre, http://dbpedia.org/resource/Art_Phillips, http://dbpedia.org/resource/Azellus_Denis, http://dbpedia.org/resource/Ben_Mulroney, http://dbpedia.org/resource/Benjamin_Pâquet, http://dbpedia.org/resource/Betty_Goodwin, http://dbpedia.org/resource/Bill_Graham_(Canadian_politician), http://dbpedia.org/resource/Dave_Keon, http://dbpedia.org/resource/Don_Marshall, http://dbpedia.org/resource/Francis_Affleck, http://dbpedia.org/resource/Georges_Vézina, http://dbpedia.org/resource/Gilles_Baril_(PQ), http://dbpedia.org/resource/Gilles_Gilbert, http://dbpedia.org/resource/Guy_Carbonneau, http://dbpedia.org/resource/Guy_Lapointe, http://dbpedia.org/resource/Jack_Goldstein, http://dbpedia.org/resource/Jean-François_Pouliot_(politician), http://dbpedia.org/resource/Jean_Beetz, http://dbpedia.org/resource/Jehane_Benoît, http://dbpedia.org/resource/John_Samuel_Bourque, http://dbpedia.org/resource/Martin_Pasko, http://dbpedia.org/resource/Maxim_Lapierre, http://dbpedia.org/resource/Mitch_Melnick, http://dbpedia.org/resource/Nicole_Léger, http://dbpedia.org/resource/Patrick_Bordeleau, http://dbpedia.org/resource/Patrick_Lebeau, http://dbpedia.org/resource/Patrick_Traverse, http://dbpedia.org/resource/Pauline_Marois, http://dbpedia.org/resource/Philippe_Boucher, http://dbpedia.org/resource/Pierre_Harvey, http://dbpedia.org/resource/Richard_Séguin, http://dbpedia.org/resource/Rick_Martel, http://dbpedia.org/resource/Robert_Fletcher_Shaw, http://dbpedia.org/resource/Royce_Frith, http://dbpedia.org/resource/Steve_Bernier, http://dbpedia.org/resource/Stéphane_Matteau, http://dbpedia.org/resource/Stéphane_Morin, http://dbpedia.org/resource/Yanic_Truesdale, http://dbpedia.org/resource/Yasmeen_Ghauri, http://dbpedia.org/resource/Yvan_Ducharme, http://dbpedia.org/resource/Étienne_Boulay, http://dbpedia.org/resource/Joby_Baker, http://dbpedia.org/resource/André_Pronovost, http://dbpedia.org/resource/Art_Alexandre, http://dbpedia.org/resource/Bertrand_Boissonnault, http://dbpedia.org/resource/Carl_Goldenberg, http://dbpedia.org/resource/Daphna_Kastner, http://dbpedia.org/resource/Dayle_Haddon, http://dbpedia.org/resource/Earl_Robinson_(ice_hockey), http://dbpedia.org/resource/George_Chenier, http://dbpedia.org/resource/Georges_Mantha, http://dbpedia.org/resource/Laurie_Kingsbury, http://dbpedia.org/resource/Marcel_Vincent, http://dbpedia.org/resource/Maurice_Bourget, http://dbpedia.org/resource/Napoléon_Turcot, http://dbpedia.org/resource/Nicolas_Roy, http://dbpedia.org/resource/Philippe_Valois, http://dbpedia.org/resource/Pierre_Bohémier, http://dbpedia.org/resource/Prudence_Heward, http://dbpedia.org/resource/Walter_Buswell, http://dbpedia.org/resource/Arthur_Philemon_Coleman, http://dbpedia.org/resource/Arthur_Plante, http://dbpedia.org/resource/Charles-Avila_Wilson, http://dbpedia.org/resource/Guillaume_Boivin, http://dbpedia.org/resource/Henri_Sévérin_Béland, http://dbpedia.org/resource/Lucile_Wheeler, http://dbpedia.org/resource/Marc-Antoine_Pouliot, http://dbpedia.org/resource/Maurice_Riel, http://dbpedia.org/resource/Paulin_Bordeleau, http://dbpedia.org/resource/Kenneth_Gilbert, http://dbpedia.org/resource/Norm_Hiscock, http://dbpedia.org/resource/Yves_Simoneau, http://dbpedia.org/resource/Romeo_Saganash, http://dbpedia.org/resource/Angelo_Esposito, http://dbpedia.org/resource/Benoît_Hogue, http://dbpedia.org/resource/Bill_Dineen, http://dbpedia.org/resource/Bob_Errey, http://dbpedia.org/resource/Bob_Graham_(ice_hockey), http://dbpedia.org/resource/Dan_Blackburn, http://dbpedia.org/resource/Dave_Hodge, http://dbpedia.org/resource/Eddie_Johnston, http://dbpedia.org/resource/Emmett_Quinn, http://dbpedia.org/resource/Frank_Eddolls, http://dbpedia.org/resource/Garry_Galley, http://dbpedia.org/resource/Gaëtan_Dugas, http://dbpedia.org/resource/Hubert_Lacroix, http://dbpedia.org/resource/Jean-Guy_Gendron, http://dbpedia.org/resource/Kevin_Dineen, http://dbpedia.org/resource/Kris_Letang, http://dbpedia.org/resource/Laurent_Picard, http://dbpedia.org/resource/Lorne_Chabot, http://dbpedia.org/resource/Marc-André_Bergeron, http://dbpedia.org/resource/Norm_Gratton, http://dbpedia.org/resource/P._J._Stock, http://dbpedia.org/resource/Scotty_Morrison, http://dbpedia.org/resource/Steven_Delisle, http://dbpedia.org/resource/Sébastien_Caron, http://dbpedia.org/resource/Terry_Murray, http://dbpedia.org/resource/Alexandre_Picard_(ice_hockey), http://dbpedia.org/resource/Jonathan_Bernier, http://dbpedia.org/resource/Leo_Housakos, http://dbpedia.org/resource/Suzanne_Côté, http://dbpedia.org/resource/Georges-Antoine_Belcourt, http://dbpedia.org/resource/André_Lamy, http://dbpedia.org/resource/François_N._Macerola, http://dbpedia.org/resource/Lee_Demarbre, http://dbpedia.org/resource/Bruce_Richardson, http://dbpedia.org/resource/Roméo_Beaudry, http://dbpedia.org/resource/Steve_Anthony, http://dbpedia.org/resource/Susan_Fennell, http://dbpedia.org/resource/James_R._Wright, http://dbpedia.org/resource/John_Thompson_(TD_banker), http://dbpedia.org/resource/Alexandra_Luke, http://dbpedia.org/resource/Erin_O'Toole, http://dbpedia.org/resource/Geoffrey_Kelley, http://dbpedia.org/resource/George_Springate, http://dbpedia.org/resource/Michael_Soles, http://dbpedia.org/resource/Philip_A._Cutler, http://dbpedia.org/resource/William_Hope_(actor), http://dbpedia.org/resource/Caroline_St-Hilaire, http://dbpedia.org/resource/Charles_Ernest_Gault, http://dbpedia.org/resource/Charles_Gavan_Power, http://dbpedia.org/resource/Claude_Gladu, http://dbpedia.org/resource/Dany_Morin, http://dbpedia.org/resource/Gerry_Weiner, http://dbpedia.org/resource/Gilles_Ste-Croix, http://dbpedia.org/resource/Guillaume_Desbiens, http://dbpedia.org/resource/James_Gelfand, http://dbpedia.org/resource/Jean_Tremblay, http://dbpedia.org/resource/Louis-Tancrède_Bouthillier, http://dbpedia.org/resource/Max_Campbell, http://dbpedia.org/resource/Normand_Cherry, http://dbpedia.org/resource/Paul_Pratt, http://dbpedia.org/resource/Pierre_Nantel, http://dbpedia.org/resource/Réal_Bouvier, http://dbpedia.org/resource/Télesphore-Damien_Bouchard, http://dbpedia.org/resource/Yves_Gabias, http://dbpedia.org/resource/Alfred_La_Liberté, http://dbpedia.org/resource/Murdo_Martin, http://dbpedia.org/resource/Norm_Letnick, http://dbpedia.org/resource/Jennifer_Baichwal, http://dbpedia.org/resource/Dan_Jason, http://dbpedia.org/resource/Danny_Maciocia, http://dbpedia.org/resource/Dimitri_Soudas, http://dbpedia.org/resource/Pierre-Hugues_Boisvenu, http://dbpedia.org/resource/Brenda_Frazier, http://dbpedia.org/resource/Valérie_Tétreault, http://dbpedia.org/resource/Marcelle_Deschênes, http://dbpedia.org/resource/Scott_Yaphe, http://dbpedia.org/resource/Albert_Dumouchel, http://dbpedia.org/resource/Henry_Saxe, http://dbpedia.org/resource/Louis_Archambault, http://dbpedia.org/resource/Frédéric_Chabot, http://dbpedia.org/resource/Nathaniel_Miller, http://dbpedia.org/resource/Serge_Houde, http://dbpedia.org/resource/Dan_Daoust, http://dbpedia.org/resource/Joanne_Vannicola, http://dbpedia.org/resource/Kathryn_Zenna, http://dbpedia.org/resource/Krista_Sutton, http://dbpedia.org/resource/Miguel_Figueroa, http://dbpedia.org/resource/Prudent_Beaudry, http://dbpedia.org/resource/Robert_Poëti, http://dbpedia.org/resource/Ross_Hull, http://dbpedia.org/resource/Yoine_Goldstein, http://dbpedia.org/resource/NŌVA, http://dbpedia.org/resource/Olivier_Dion, http://dbpedia.org/resource/Pierre_Fortier, http://dbpedia.org/resource/Salem_Bland, http://dbpedia.org/resource/Sophie_Vaillancourt, http://dbpedia.org/resource/Étienne_Drapeau, http://dbpedia.org/resource/Carlo_Rossi_(politician), http://dbpedia.org/resource/David_Agnew_(president), http://dbpedia.org/resource/Jean-François_Larose, http://dbpedia.org/resource/Justin_Trottier, http://dbpedia.org/resource/Ève_Péclet, http://dbpedia.org/resource/Arnold_Scaasi, http://dbpedia.org/resource/Kelly_Kruger, http://dbpedia.org/resource/Barclay_Hope, http://dbpedia.org/resource/Alexandre_Giroux, http://dbpedia.org/resource/André_Plourde, http://dbpedia.org/resource/Charles_Deblois, http://dbpedia.org/resource/Eric_Lavigne, http://dbpedia.org/resource/Fernand_Jourdenais, http://dbpedia.org/resource/Francis_Bélanger, http://dbpedia.org/resource/François_Groleau, http://dbpedia.org/resource/James_Desmarais, http://dbpedia.org/resource/Jean-Guy_Guilbault, http://dbpedia.org/resource/Joe_Rullier, http://dbpedia.org/resource/Marc-André_Thinel, http://dbpedia.org/resource/Marc_Ferland_(politician), http://dbpedia.org/resource/Maurice_Tremblay, http://dbpedia.org/resource/Mirko_Murovic, http://dbpedia.org/resource/Philippe_DeRouville, http://dbpedia.org/resource/Simon_Lajeunesse, http://dbpedia.org/resource/Yanick_Dupré, http://dbpedia.org/resource/Arthur_Westover, http://dbpedia.org/resource/Claude_Boivin, http://dbpedia.org/resource/David_Desharnais, http://dbpedia.org/resource/Denis_Chassé, http://dbpedia.org/resource/Eric_Germain, http://dbpedia.org/resource/Jan_Alston, http://dbpedia.org/resource/Jason_Doig, http://dbpedia.org/resource/M.A._Fortin, http://dbpedia.org/resource/Martin_St._Amour, http://dbpedia.org/resource/Éric_Houde, http://dbpedia.org/resource/Bob_Roselle, http://dbpedia.org/resource/Mado_Lamotte, http://dbpedia.org/resource/Paul_Larose, http://dbpedia.org/resource/Tarzan_Tyler, http://dbpedia.org/resource/Trevor_Eyton, http://dbpedia.org/resource/George_Wardrope, http://dbpedia.org/resource/Louis_Domingue, http://dbpedia.org/resource/Serge_Marcil, http://dbpedia.org/resource/Élaine_Michaud, http://dbpedia.org/resource/Nathaniel_Pettes, http://dbpedia.org/resource/Alma_Ricard, http://dbpedia.org/resource/Alexandre_Grenier, http://dbpedia.org/resource/Benoît_Mondou, http://dbpedia.org/resource/Clément_Jodoin, http://dbpedia.org/resource/François_Sasseville, http://dbpedia.org/resource/Gabriel_Bourque, http://dbpedia.org/resource/Jean-Philippe_Côté, http://dbpedia.org/resource/Jonathan_Ferland, http://dbpedia.org/resource/Jérôme_Samson, http://dbpedia.org/resource/Marco_Cousineau, http://dbpedia.org/resource/Mathieu_Carle, http://dbpedia.org/resource/Patrick_Couture, http://dbpedia.org/resource/Philippe_Dupuis, http://dbpedia.org/resource/Pierre-Luc_Lessard, http://dbpedia.org/resource/Eugène_Durocher, http://dbpedia.org/resource/Lauriane_Rougeau, http://dbpedia.org/resource/Maxime_Daigneault, http://dbpedia.org/resource/Arthur_Vineberg, http://dbpedia.org/resource/George_Kourounis, http://dbpedia.org/resource/Nicholas_Fekete, http://dbpedia.org/resource/Susan_Bonner, http://dbpedia.org/resource/Billy_Barlow, http://dbpedia.org/resource/Cecil_Blachford, http://dbpedia.org/resource/Giulio_Scandella, http://dbpedia.org/resource/Jim_Corsi_(ice_hockey), http://dbpedia.org/resource/Massimo_Di_Ioia, http://dbpedia.org/resource/Maurizio_Mansi, http://dbpedia.org/resource/Mélissa_Désormeaux-Poulin, http://dbpedia.org/resource/Nicholas_La_Monaca, http://dbpedia.org/resource/Sol_Zanetti, http://dbpedia.org/resource/Stefano_Giliati, http://dbpedia.org/resource/Alexandre_Taché_(politician), http://dbpedia.org/resource/Guy_Rouleau_(politician), http://dbpedia.org/resource/Gérard_Lebel, http://dbpedia.org/resource/Jean-Pierre_Saintonge, http://dbpedia.org/resource/Martin_Ferland, http://dbpedia.org/resource/Pierre_Bélanger, http://dbpedia.org/resource/William_Dion, http://dbpedia.org/resource/Alcide_Côté, http://dbpedia.org/resource/Bluma_Appel, http://dbpedia.org/resource/Claude_Lajoie, http://dbpedia.org/resource/Eugène_Paquet, http://dbpedia.org/resource/Gabriel_Nadeau-Dubois, http://dbpedia.org/resource/John_Wildman_(actor), http://dbpedia.org/resource/Michael_Zelniker, http://dbpedia.org/resource/Paul_Martineau, http://dbpedia.org/resource/Samuel_Gobeil, http://dbpedia.org/resource/William_Frederic_Kay, http://dbpedia.org/resource/Yves_Leduc, http://dbpedia.org/resource/Sean_J._Harrington, http://dbpedia.org/resource/Robert_Wertheimer, http://dbpedia.org/resource/Mike_Madill, http://dbpedia.org/resource/Danick_Paquette, http://dbpedia.org/resource/Kevin_Marshall, http://dbpedia.org/resource/Mathieu_Aubin, http://dbpedia.org/resource/Stefan_Chaput, http://dbpedia.org/resource/W._Roger_Graham, http://dbpedia.org/resource/Stefan_Pleszczynski, http://dbpedia.org/resource/Camille_Dagenais, http://dbpedia.org/resource/François-Philippe_Brais, http://dbpedia.org/resource/Gilles_Cloutier, http://dbpedia.org/resource/Jack_McDonald_(ice_hockey_b._1887), http://dbpedia.org/resource/Peter_Reid_(triathlete), http://dbpedia.org/resource/Victor_Yarnell, http://dbpedia.org/resource/André_Barbeau, http://dbpedia.org/resource/Jean-Philip_Chabot, http://dbpedia.org/resource/Cindy_Busby, http://dbpedia.org/resource/Paul_Brodie, http://dbpedia.org/resource/Samantha_Albert, http://dbpedia.org/resource/Derek_Corneil, http://dbpedia.org/resource/Catherine_LeFrançois, http://dbpedia.org/resource/Pom_Klementieff, http://dbpedia.org/resource/Charlie_Bilodeau, http://dbpedia.org/resource/Nicolas_Nadeau, http://dbpedia.org/resource/Alexandre_Thurber, http://dbpedia.org/resource/Dick_Sarrazin, http://dbpedia.org/resource/Gary_Connelly, http://dbpedia.org/resource/Paul_Menard_(ice_hockey), http://dbpedia.org/resource/Pierre_Viau, http://dbpedia.org/resource/Richard_Dumas_(ice_hockey), http://dbpedia.org/resource/Chris_Stathopoulos, http://dbpedia.org/resource/Alain_Langlais, http://dbpedia.org/resource/Bob_Ritchie_(ice_hockey), http://dbpedia.org/resource/Gilles_Bilodeau, http://dbpedia.org/resource/Andy_Gilpin, http://dbpedia.org/resource/Anthony_Beauvillier, http://dbpedia.org/resource/Henri-André_Laperrière, http://dbpedia.org/resource/Mike_Lagace, http://dbpedia.org/resource/Denis_Chalifoux_(MNA), http://dbpedia.org/resource/Craig_Johnson_(ice_hockey_b._1971), http://dbpedia.org/resource/Pierre_Trottier, http://dbpedia.org/resource/Roy_Fournier, http://dbpedia.org/resource/Emilie_Demers_Boutin, http://dbpedia.org/resource/Pierre-Alexandre_Rousseau, http://dbpedia.org/resource/Vincent_Marquis, http://dbpedia.org/resource/Stéphane_Veilleux, http://dbpedia.org/resource/Jeff_Kuhner, http://dbpedia.org/resource/Renaldo_Sagesse, http://dbpedia.org/resource/Marc-Antoine_Gagnon, http://dbpedia.org/resource/Danielle_St-Amand, http://dbpedia.org/resource/Marie-Pier_Préfontaine, http://dbpedia.org/resource/Khem_Birch, http://dbpedia.org/resource/Daniel_Guérard, http://dbpedia.org/resource/Gabriel_Gagne, http://dbpedia.org/resource/Louis_Béland-Goyette, http://dbpedia.org/resource/Albiny_Paquette, http://dbpedia.org/resource/Camille-Eugène_Pouliot, http://dbpedia.org/resource/Maurice_Custeau, http://dbpedia.org/resource/Éloi_Guillemette, http://dbpedia.org/resource/Sam_Glucksberg, http://dbpedia.org/resource/Marisol_Grondin, http://dbpedia.org/resource/Ray_LaRose, http://dbpedia.org/resource/Adam_Lamhamedi, http://dbpedia.org/resource/Jean_Labonté, http://dbpedia.org/resource/Rob_Lobel, http://dbpedia.org/resource/William_R._C._Blundell, http://dbpedia.org/resource/Katrina_Monton, http://dbpedia.org/resource/Donnie_Ruiz, http://dbpedia.org/resource/Scott_Oke, http://dbpedia.org/resource/Laurie_Walker_(artist), http://dbpedia.org/resource/Jean-Claude_Garneau, http://dbpedia.org/resource/Howard_Reid_(admiral), http://dbpedia.org/resource/Geneviève_Morrison, http://dbpedia.org/resource/Jonathan_Beaulieu-Richard, http://dbpedia.org/resource/Jack_Grannary, http://dbpedia.org/resource/Jacques_Lemelin, http://dbpedia.org/resource/Norm_Descoteaux, http://dbpedia.org/resource/Patsy_Séguin, http://dbpedia.org/resource/Donald_Hillman, http://dbpedia.org/resource/Joseph-Alphonse-Paul_Cadotte, http://dbpedia.org/resource/Dan_Delisle, http://dbpedia.org/resource/Julia_Scott, http://dbpedia.org/resource/Kit_Lang, http://dbpedia.org/resource/Derek_Famulare, http://dbpedia.org/resource/Alain_Daigle, http://dbpedia.org/resource/Alfred_Rouleau, http://dbpedia.org/resource/André_Boisclair, http://dbpedia.org/resource/Andrée_Boucher, http://dbpedia.org/resource/August_Schellenberg, http://dbpedia.org/resource/Bat_Masterson, http://dbpedia.org/resource/Benoît_Brunet, http://dbpedia.org/resource/Benoît_Sauvageau, http://dbpedia.org/resource/Bryce_Mackasey, http://dbpedia.org/resource/Buddy_O'Connor, http://dbpedia.org/resource/Carol_Vadnais, http://dbpedia.org/resource/Carole_Laure, http://dbpedia.org/resource/Charles_Hamelin, http://dbpedia.org/resource/Claire_Martin_(writer), http://dbpedia.org/resource/Claude_Provost, http://dbpedia.org/resource/Cliff_Fletcher, http://dbpedia.org/resource/Dan_Cloutier, http://dbpedia.org/resource/Daniel_Johnson,_Sr., http://dbpedia.org/resource/Don_Boudria, http://dbpedia.org/resource/Dorothée_Berryman, http://dbpedia.org/resource/Erik_Guay, http://dbpedia.org/resource/Fabien_Roy, http://dbpedia.org/resource/Fifi_D'Orsay, http://dbpedia.org/resource/François_Leroux, http://dbpedia.org/resource/Félix-Antoine_Savard, http://dbpedia.org/resource/Gaétan_Duchesne, http://dbpedia.org/resource/Georges_Vanier, http://dbpedia.org/resource/Germaine_Guèvremont, http://dbpedia.org/resource/Gerry_McNeil, http://dbpedia.org/resource/Gilles_Rocheleau, http://dbpedia.org/resource/Ginette_Reno, http://dbpedia.org/resource/Guillaume_LeBlanc, http://dbpedia.org/resource/Gérard_Filion, http://dbpedia.org/resource/Heather_Reisman, http://dbpedia.org/resource/Jacob_Viner, http://dbpedia.org/resource/Jacques_Bouchard, http://dbpedia.org/resource/Jacques_Chagnon, http://dbpedia.org/resource/Jacques_Demers, http://dbpedia.org/resource/Jacques_Hétu, http://dbpedia.org/resource/Jasey-Jay_Anderson, http://dbpedia.org/resource/Jean-François_Jacques, http://dbpedia.org/resource/Jean_Béliveau, http://dbpedia.org/resource/Jean_Drapeau, http://dbpedia.org/resource/Jean_Gascon, http://dbpedia.org/resource/Jean_Marchand, http://dbpedia.org/resource/Jimmy_Mann_(ice_hockey), http://dbpedia.org/resource/Joe_Weider, http://dbpedia.org/resource/John_Lynch-Staunton, http://dbpedia.org/resource/Jos_LeDuc, http://dbpedia.org/resource/Joseph-Armand_Bombardier, http://dbpedia.org/resource/King_O'Malley, http://dbpedia.org/resource/Lise_Bacon, http://dbpedia.org/resource/Lorraine_Desmarais, http://dbpedia.org/resource/Lothaire_Bluteau, http://dbpedia.org/resource/Louise_Harel, http://dbpedia.org/resource/Marie-Louise_Meilleur, http://dbpedia.org/resource/Martin_Biron, http://dbpedia.org/resource/Martin_Cauchon, http://dbpedia.org/resource/Maryam_Henein, http://dbpedia.org/resource/Mathieu_Biron, http://dbpedia.org/resource/Maurice_Sauvé, http://dbpedia.org/resource/Michael_Sarrazin, http://dbpedia.org/resource/Michel_Brière, http://dbpedia.org/resource/Michel_Parizeau, http://dbpedia.org/resource/Nels_Stewart, http://dbpedia.org/resource/Northrop_Frye, http://dbpedia.org/resource/Oscar_Peterson, http://dbpedia.org/resource/Patrick_Poulin, http://dbpedia.org/resource/Pierre_Claude_Nolin, http://dbpedia.org/resource/Pierre_Guité, http://dbpedia.org/resource/Pierre_Pettigrew, http://dbpedia.org/resource/Pierre_Reid, http://dbpedia.org/resource/Pierre_Turgeon, http://dbpedia.org/resource/René_Lépine, http://dbpedia.org/resource/Robert_Joy, http://dbpedia.org/resource/Régine_Chassagne, http://dbpedia.org/resource/Sam_Pollock, http://dbpedia.org/resource/Serge_Bernier, http://dbpedia.org/resource/Sylvio_Mantha, http://dbpedia.org/resource/Terri_Clark, http://dbpedia.org/resource/Willie_Dunn, http://dbpedia.org/resource/Yves-François_Blanchet, http://dbpedia.org/resource/Édouard_Fabre, http://dbpedia.org/resource/Émile_Bouchard, http://dbpedia.org/resource/François_Arnaud_(actor), http://dbpedia.org/resource/Jean-Paul_Riopelle, http://dbpedia.org/resource/Lionel_Blair, http://dbpedia.org/resource/Mark_Aronoff, http://dbpedia.org/resource/Naomi_Klein, http://dbpedia.org/resource/Rodrigue_Tremblay, http://dbpedia.org/resource/Chester_Brown, http://dbpedia.org/resource/Jacques_Languirand, http://dbpedia.org/resource/Suzanne_Tremblay, http://dbpedia.org/resource/2014_shootings_at_Parliament_Hill,_Ottawa, http://dbpedia.org/resource/Al_Connelly, http://dbpedia.org/resource/Alex_Chiasson, http://dbpedia.org/resource/Christian_Duguay_(director), http://dbpedia.org/resource/Claude_Lapointe, http://dbpedia.org/resource/Daniel_Boucher_(musician), http://dbpedia.org/resource/Daniel_Brière, http://dbpedia.org/resource/Denis_Brodeur, http://dbpedia.org/resource/Fleming_Mackell, http://dbpedia.org/resource/Frank_Finnigan, http://dbpedia.org/resource/Gilles_Vigneault, http://dbpedia.org/resource/Grant_Crack, http://dbpedia.org/resource/J._J._Daigneault, http://dbpedia.org/resource/Jacques_Richard, http://dbpedia.org/resource/Jean-Marc_Fournier]... [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 757, P: 0.495, R: 0.548, macro F1: 0.502, macro F1*: 0.520 -[INFO] Current time: 12213 ms, average: 2876.31 ms/ques, entity detection: 479.05 ms/ques (linking tool: 386.36 ms/ques), relation detection: 930.69 ms/ques (similarity: 706.08 ms/ques), EDG generation: 46.39 ms/ques +[INFO] Cumulative metrics, sample: 757, P: 0.495, R: 0.548, macro F1: 0.520 +[INFO] Current question consumed time: 11299 ms, average: 2589.61 ms/ques, entity detection: 423.24 ms/ques (linking tool: 345.10 ms/ques), relation detection: 858.10 ms/ques (similarity: 697.90 ms/ques), EDG generation: 46.43 ms/ques [INFO] Entity Linking P: 0.7404227212681638, R: 0.6935270805812418, micro F1: 0.7084984588287108, macro F1: 0.7162080675555706 [INFO] Relation Linking P: 0.5956627036547776, R: 0.5402906208718626, micro F1: 0.5559665345662712, macro F1: 0.566627105250053 [INFO] Type Linking P: 0.5482166446499339, R: 0.5482166446499339, micro F1: 0.5482166446499339, macro F1: 0.5482166446499339 @@ -10629,8 +10629,8 @@ Question 757: Who is the chancellor of Falmouth University ? [INFO] Golden Answer: [http://dbpedia.org/resource/Dawn_French] [INFO] Predicted Answer: [http://dbpedia.org/resource/Dawn_French] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 758, P: 0.495, R: 0.548, macro F1: 0.502, macro F1*: 0.520 -[INFO] Current time: 868 ms, average: 2873.66 ms/ques, entity detection: 478.71 ms/ques (linking tool: 386.00 ms/ques), relation detection: 929.73 ms/ques (similarity: 705.32 ms/ques), EDG generation: 46.35 ms/ques +[INFO] Cumulative metrics, sample: 758, P: 0.495, R: 0.548, macro F1: 0.520 +[INFO] Current question consumed time: 719 ms, average: 2587.14 ms/ques, entity detection: 422.90 ms/ques (linking tool: 344.77 ms/ques), relation detection: 857.21 ms/ques (similarity: 697.16 ms/ques), EDG generation: 46.38 ms/ques [INFO] Entity Linking P: 0.7407651715039578, R: 0.6939313984168866, micro F1: 0.7088830255057178, macro F1: 0.7165838716525642 [INFO] Relation Linking P: 0.5961961301671064, R: 0.5408970976253298, micro F1: 0.5565523306948117, macro F1: 0.5672019646953812 [INFO] Type Linking P: 0.5488126649076517, R: 0.5488126649076517, micro F1: 0.5488126649076517, macro F1: 0.5488126649076517 @@ -10643,8 +10643,8 @@ Question 758: what kind of things play on WBIG FM? [INFO] Golden Answer: [http://dbpedia.org/resource/Asia, http://dbpedia.org/resource/Classic_rock] [INFO] Predicted Answer: [http://dbpedia.org/resource/Baltimore_Ravens] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 759, P: 0.495, R: 0.547, macro F1: 0.502, macro F1*: 0.520 -[INFO] Current time: 1309 ms, average: 2871.60 ms/ques, entity detection: 478.26 ms/ques (linking tool: 385.61 ms/ques), relation detection: 929.09 ms/ques (similarity: 704.95 ms/ques), EDG generation: 46.30 ms/ques +[INFO] Cumulative metrics, sample: 759, P: 0.495, R: 0.547, macro F1: 0.520 +[INFO] Current question consumed time: 1126 ms, average: 2585.21 ms/ques, entity detection: 422.50 ms/ques (linking tool: 344.42 ms/ques), relation detection: 856.60 ms/ques (similarity: 696.79 ms/ques), EDG generation: 46.33 ms/ques [INFO] Entity Linking P: 0.741106719367589, R: 0.69433465085639, micro F1: 0.7092665788317972, macro F1: 0.7169586803244046 [INFO] Relation Linking P: 0.5954106280193237, R: 0.5401844532279315, micro F1: 0.5558190601668871, macro F1: 0.566454663029116 [INFO] Type Linking P: 0.549407114624506, R: 0.549407114624506, micro F1: 0.549407114624506, macro F1: 0.549407114624506 @@ -10657,8 +10657,8 @@ Question 759: Name the are where South side Elevated Railroad operate and has pl [INFO] Golden Answer: [http://dbpedia.org/resource/Chicago] [INFO] Predicted Answer: [http://dbpedia.org/resource/Simon_City_Royals, http://dbpedia.org/resource/Jousters] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 760, P: 0.494, R: 0.547, macro F1: 0.501, macro F1*: 0.519 -[INFO] Current time: 6077 ms, average: 2875.81 ms/ques, entity detection: 478.23 ms/ques (linking tool: 385.59 ms/ques), relation detection: 932.01 ms/ques (similarity: 706.05 ms/ques), EDG generation: 46.28 ms/ques +[INFO] Cumulative metrics, sample: 760, P: 0.494, R: 0.547, macro F1: 0.519 +[INFO] Current question consumed time: 5482 ms, average: 2589.02 ms/ques, entity detection: 422.46 ms/ques (linking tool: 344.38 ms/ques), relation detection: 859.27 ms/ques (similarity: 697.90 ms/ques), EDG generation: 46.31 ms/ques [INFO] Entity Linking P: 0.7414473684210526, R: 0.694078947368421, micro F1: 0.7092105263157904, macro F1: 0.7169816440735202 [INFO] Relation Linking P: 0.5952850877192982, R: 0.5401315789473684, micro F1: 0.5557456140350886, macro F1: 0.5663687768431218 [INFO] Type Linking P: 0.55, R: 0.55, micro F1: 0.55, macro F1: 0.55 @@ -10671,8 +10671,8 @@ Question 760: In which fields are the graduates of Worcester College, Oxford wor [INFO] Golden Answer: [http://dbpedia.org/resource/Ornithology, http://dbpedia.org/resource/Parallel_computing, http://dbpedia.org/resource/Supercomputer, http://dbpedia.org/resource/E-Science, http://dbpedia.org/resource/Development_economics, http://dbpedia.org/resource/Economics, http://dbpedia.org/resource/Public_economics, http://dbpedia.org/resource/Chess, http://dbpedia.org/resource/Mathematics, http://dbpedia.org/resource/Writing] [INFO] Predicted Answer: [http://dbpedia.org/resource/Installation_art, http://dbpedia.org/resource/Sculpture] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 761, P: 0.493, R: 0.546, macro F1: 0.500, macro F1*: 0.518 -[INFO] Current time: 5955 ms, average: 2879.86 ms/ques, entity detection: 478.16 ms/ques (linking tool: 385.56 ms/ques), relation detection: 934.92 ms/ques (similarity: 707.50 ms/ques), EDG generation: 46.36 ms/ques +[INFO] Cumulative metrics, sample: 761, P: 0.493, R: 0.546, macro F1: 0.518 +[INFO] Current question consumed time: 5333 ms, average: 2592.63 ms/ques, entity detection: 422.37 ms/ques (linking tool: 344.34 ms/ques), relation detection: 861.94 ms/ques (similarity: 699.36 ms/ques), EDG generation: 46.41 ms/ques [INFO] Entity Linking P: 0.740473061760841, R: 0.6931668856767411, micro F1: 0.7082785808147184, macro F1: 0.7160394868539756 [INFO] Relation Linking P: 0.5951598773543583, R: 0.5400788436268068, micro F1: 0.5556723609286035, macro F1: 0.5662831127831957 [INFO] Type Linking P: 0.5505913272010512, R: 0.5505913272010512, micro F1: 0.5505913272010512, macro F1: 0.5505913272010512 @@ -10685,8 +10685,8 @@ Question 761: List the places where the relatives of Mark Donaldson died ? [INFO] Golden Answer: [http://dbpedia.org/resource/New_Zealand, http://dbpedia.org/resource/Palmerston_North] [INFO] Predicted Answer: [http://dbpedia.org/resource/Waratah,_New_South_Wales] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 762, P: 0.493, R: 0.545, macro F1: 0.500, macro F1*: 0.518 -[INFO] Current time: 1520 ms, average: 2878.08 ms/ques, entity detection: 477.81 ms/ques (linking tool: 385.21 ms/ques), relation detection: 934.28 ms/ques (similarity: 707.23 ms/ques), EDG generation: 46.41 ms/ques +[INFO] Cumulative metrics, sample: 762, P: 0.493, R: 0.545, macro F1: 0.518 +[INFO] Current question consumed time: 1316 ms, average: 2590.96 ms/ques, entity detection: 422.04 ms/ques (linking tool: 344.02 ms/ques), relation detection: 861.36 ms/ques (similarity: 699.11 ms/ques), EDG generation: 46.46 ms/ques [INFO] Entity Linking P: 0.739501312335958, R: 0.6922572178477691, micro F1: 0.7073490813648303, macro F1: 0.7150998024880254 [INFO] Relation Linking P: 0.5943788276465442, R: 0.5393700787401575, micro F1: 0.5549431321084872, macro F1: 0.5655399590918792 [INFO] Type Linking P: 0.5498687664041995, R: 0.5498687664041995, micro F1: 0.5498687664041995, macro F1: 0.5498687664041995 @@ -10699,8 +10699,8 @@ Question 762: What famous politician died in the Petersen House? [INFO] Golden Answer: [http://dbpedia.org/resource/Abraham_Lincoln] [INFO] Predicted Answer: [http://dbpedia.org/resource/Abraham_Lincoln] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 763, P: 0.493, R: 0.546, macro F1: 0.500, macro F1*: 0.518 -[INFO] Current time: 1129 ms, average: 2875.79 ms/ques, entity detection: 477.39 ms/ques (linking tool: 384.83 ms/ques), relation detection: 933.55 ms/ques (similarity: 706.83 ms/ques), EDG generation: 46.36 ms/ques +[INFO] Cumulative metrics, sample: 763, P: 0.493, R: 0.546, macro F1: 0.518 +[INFO] Current question consumed time: 1005 ms, average: 2588.88 ms/ques, entity detection: 421.66 ms/ques (linking tool: 343.67 ms/ques), relation detection: 860.69 ms/ques (similarity: 698.71 ms/ques), EDG generation: 46.42 ms/ques [INFO] Entity Linking P: 0.7398427260812582, R: 0.6926605504587156, micro F1: 0.7077326343381398, macro F1: 0.7154746216540622 [INFO] Relation Linking P: 0.5949104412407165, R: 0.5399737876802096, micro F1: 0.5555264307557893, macro F1: 0.5661124475977484 [INFO] Type Linking P: 0.5491480996068152, R: 0.5491480996068152, micro F1: 0.5491480996068152, macro F1: 0.5491480996068152 @@ -10713,8 +10713,8 @@ Question 763: What are some artists on the show whose opening theme is Send It O [INFO] Golden Answer: [http://dbpedia.org/resource/McClain_(band), http://dbpedia.org/resource/Bridgit_Mendler, Demi Lovato and Joe Jonas@en] [INFO] Predicted Answer: [http://dbpedia.org/resource/McClain_(band), http://dbpedia.org/resource/Bridgit_Mendler, Demi Lovato and Joe Jonas@en, http://dbpedia.org/resource/Send_It_On_(Disney's_Friends_for_Change_song)] [INFO] P: 0.75, R: 1.0, F: 0.8571428571428571 -[INFO] Cumulative metrics, sample: 764, P: 0.494, R: 0.546, macro F1: 0.501, macro F1*: 0.519 -[INFO] Current time: 1585 ms, average: 2874.10 ms/ques, entity detection: 476.95 ms/ques (linking tool: 384.45 ms/ques), relation detection: 933.18 ms/ques (similarity: 706.63 ms/ques), EDG generation: 46.34 ms/ques +[INFO] Cumulative metrics, sample: 764, P: 0.494, R: 0.546, macro F1: 0.519 +[INFO] Current question consumed time: 1388 ms, average: 2587.30 ms/ques, entity detection: 421.25 ms/ques (linking tool: 343.32 ms/ques), relation detection: 860.34 ms/ques (similarity: 698.55 ms/ques), EDG generation: 46.39 ms/ques [INFO] Entity Linking P: 0.7401832460732984, R: 0.6930628272251309, micro F1: 0.7081151832460741, macro F1: 0.7158484544215735 [INFO] Relation Linking P: 0.5954406631762652, R: 0.5405759162303665, micro F1: 0.5561082024432819, macro F1: 0.5666834232744254 [INFO] Type Linking P: 0.5484293193717278, R: 0.5484293193717278, micro F1: 0.5484293193717278, macro F1: 0.5484293193717278 @@ -10727,8 +10727,8 @@ Question 764: What is the citizenship of Saima Chowdhury? [INFO] Golden Answer: [http://dbpedia.org/resource/United_Kingdom] [INFO] Predicted Answer: [http://dbpedia.org/resource/United_Kingdom] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 765, P: 0.494, R: 0.547, macro F1: 0.502, macro F1*: 0.519 -[INFO] Current time: 755 ms, average: 2871.33 ms/ques, entity detection: 476.56 ms/ques (linking tool: 384.06 ms/ques), relation detection: 932.21 ms/ques (similarity: 705.86 ms/ques), EDG generation: 46.29 ms/ques +[INFO] Cumulative metrics, sample: 765, P: 0.494, R: 0.547, macro F1: 0.519 +[INFO] Current question consumed time: 624 ms, average: 2584.74 ms/ques, entity detection: 420.88 ms/ques (linking tool: 342.96 ms/ques), relation detection: 859.44 ms/ques (similarity: 697.78 ms/ques), EDG generation: 46.34 ms/ques [INFO] Entity Linking P: 0.7405228758169935, R: 0.6934640522875817, micro F1: 0.7084967320261447, macro F1: 0.7162213046871742 [INFO] Relation Linking P: 0.5959694989106753, R: 0.5411764705882353, micro F1: 0.5566884531590423, macro F1: 0.5672528921521669 [INFO] Type Linking P: 0.5490196078431373, R: 0.5490196078431373, micro F1: 0.5490196078431373, macro F1: 0.5490196078431373 @@ -10741,8 +10741,8 @@ Question 765: What are the musical performances whose label is Celluloid Records [INFO] Golden Answer: [http://dbpedia.org/resource/The_Last_Poets, http://dbpedia.org/resource/Hustlers_Convention_(Lightnin'_Rod_album), http://dbpedia.org/resource/Shapeshifter_(Gong_album), http://dbpedia.org/resource/Horses_&_Trees, http://dbpedia.org/resource/Down_by_Law_(Deadline_album), http://dbpedia.org/resource/Fela_Kuti, http://dbpedia.org/resource/Massacre_(experimental_band), http://dbpedia.org/resource/The_Golden_Palominos_(album), http://dbpedia.org/resource/Jukebox_Babe, http://dbpedia.org/resource/Material_(band), http://dbpedia.org/resource/A_Dead_Horse, http://dbpedia.org/resource/Blast_of_Silence_(Axed_My_Baby_for_a_Nickel), http://dbpedia.org/resource/Ferdinand_Richard, http://dbpedia.org/resource/Collision_Drive, http://dbpedia.org/resource/Deadline_(band), http://dbpedia.org/resource/Into_the_Outlands, http://dbpedia.org/resource/SXL_(band), http://dbpedia.org/resource/Jody_Harris, http://dbpedia.org/resource/Killing_Time_(Massacre_album), http://dbpedia.org/resource/Western_Culture_(album), http://dbpedia.org/resource/Low_Life_(Peter_Brötzmann_and_Bill_Laswell_album), http://dbpedia.org/resource/Circle_X_(EP), http://dbpedia.org/resource/Red_Krayola, http://dbpedia.org/resource/Etron_Fou_Leloublan, http://dbpedia.org/resource/Visions_of_Excess] [INFO] Predicted Answer: [http://dbpedia.org/resource/The_Last_Poets, http://dbpedia.org/resource/Hustlers_Convention_(Lightnin'_Rod_album), http://dbpedia.org/resource/Shapeshifter_(Gong_album), http://dbpedia.org/resource/Horses_&_Trees, http://dbpedia.org/resource/Down_by_Law_(Deadline_album), http://dbpedia.org/resource/Fela_Kuti, http://dbpedia.org/resource/Massacre_(experimental_band), http://dbpedia.org/resource/The_Golden_Palominos_(album), http://dbpedia.org/resource/Jukebox_Babe, http://dbpedia.org/resource/Material_(band), http://dbpedia.org/resource/A_Dead_Horse, http://dbpedia.org/resource/Blast_of_Silence_(Axed_My_Baby_for_a_Nickel), http://dbpedia.org/resource/Ferdinand_Richard, http://dbpedia.org/resource/Collision_Drive, http://dbpedia.org/resource/Deadline_(band), http://dbpedia.org/resource/Into_the_Outlands, http://dbpedia.org/resource/SXL_(band), http://dbpedia.org/resource/Jody_Harris, http://dbpedia.org/resource/Killing_Time_(Massacre_album), http://dbpedia.org/resource/Western_Culture_(album), http://dbpedia.org/resource/Low_Life_(Peter_Brötzmann_and_Bill_Laswell_album), http://dbpedia.org/resource/Circle_X_(EP), http://dbpedia.org/resource/Red_Krayola, http://dbpedia.org/resource/Etron_Fou_Leloublan, http://dbpedia.org/resource/Visions_of_Excess] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 766, P: 0.495, R: 0.548, macro F1: 0.502, macro F1*: 0.520 -[INFO] Current time: 1510 ms, average: 2869.55 ms/ques, entity detection: 476.28 ms/ques (linking tool: 383.75 ms/ques), relation detection: 931.45 ms/ques (similarity: 705.21 ms/ques), EDG generation: 46.26 ms/ques +[INFO] Cumulative metrics, sample: 766, P: 0.495, R: 0.548, macro F1: 0.520 +[INFO] Current question consumed time: 1272 ms, average: 2583.02 ms/ques, entity detection: 420.62 ms/ques (linking tool: 342.67 ms/ques), relation detection: 858.73 ms/ques (similarity: 697.15 ms/ques), EDG generation: 46.30 ms/ques [INFO] Entity Linking P: 0.7408616187989556, R: 0.693864229765013, micro F1: 0.7088772845953012, macro F1: 0.7165931763269243 [INFO] Relation Linking P: 0.5964969538729329, R: 0.54177545691906, micro F1: 0.557267188859879, macro F1: 0.5678208602288456 [INFO] Type Linking P: 0.5496083550913838, R: 0.5496083550913838, micro F1: 0.5496083550913838, macro F1: 0.5496083550913838 @@ -10755,8 +10755,8 @@ Question 766: What is the craft of the Ralph Flanders which is also the professi [INFO] Golden Answer: [http://dbpedia.org/resource/United_States_Senate] [INFO] Predicted Answer: [http://dbpedia.org/resource/United_States_Senate] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 767, P: 0.496, R: 0.548, macro F1: 0.503, macro F1*: 0.521 -[INFO] Current time: 2570 ms, average: 2869.16 ms/ques, entity detection: 476.26 ms/ques (linking tool: 383.64 ms/ques), relation detection: 931.42 ms/ques (similarity: 705.01 ms/ques), EDG generation: 46.29 ms/ques +[INFO] Cumulative metrics, sample: 767, P: 0.496, R: 0.548, macro F1: 0.521 +[INFO] Current question consumed time: 2145 ms, average: 2582.45 ms/ques, entity detection: 420.56 ms/ques (linking tool: 342.56 ms/ques), relation detection: 858.72 ms/ques (similarity: 697.11 ms/ques), EDG generation: 46.33 ms/ques [INFO] Entity Linking P: 0.741199478487614, R: 0.6942633637548892, micro F1: 0.7092568448500661, macro F1: 0.7169640731964659 [INFO] Relation Linking P: 0.5970230334637113, R: 0.5423728813559322, micro F1: 0.557844415471535, macro F1: 0.568387333470172 [INFO] Type Linking P: 0.5501955671447197, R: 0.5501955671447197, micro F1: 0.5501955671447197, macro F1: 0.5501955671447197 @@ -10769,8 +10769,8 @@ Question 767: How many people have fought wars where Arthur St. Clair was a comm [INFO] Golden Answer: [7] [INFO] Predicted Answer: [8, 7, 4] [INFO] P: 0.3333333333333333, R: 1.0, F: 0.5 -[INFO] Cumulative metrics, sample: 768, P: 0.496, R: 0.549, macro F1: 0.503, macro F1*: 0.521 -[INFO] Current time: 2885 ms, average: 2869.18 ms/ques, entity detection: 475.83 ms/ques (linking tool: 383.25 ms/ques), relation detection: 931.35 ms/ques (similarity: 705.23 ms/ques), EDG generation: 46.26 ms/ques +[INFO] Cumulative metrics, sample: 768, P: 0.496, R: 0.549, macro F1: 0.521 +[INFO] Current question consumed time: 2327 ms, average: 2582.12 ms/ques, entity detection: 420.15 ms/ques (linking tool: 342.19 ms/ques), relation detection: 858.59 ms/ques (similarity: 697.32 ms/ques), EDG generation: 46.29 ms/ques [INFO] Entity Linking P: 0.7415364583333334, R: 0.6946614583333334, micro F1: 0.7096354166666675, macro F1: 0.7173339991311575 [INFO] Relation Linking P: 0.5975477430555555, R: 0.54296875, micro F1: 0.5584201388888897, macro F1: 0.5689523178099135 [INFO] Type Linking P: 0.55078125, R: 0.55078125, micro F1: 0.55078125, macro F1: 0.55078125 @@ -10783,8 +10783,8 @@ Question 768: Does Chicago own the Monroe Station? [INFO] Golden Answer: [true] [INFO] Predicted Answer: [true] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 769, P: 0.496, R: 0.549, macro F1: 0.504, macro F1*: 0.521 -[INFO] Current time: 2698 ms, average: 2868.96 ms/ques, entity detection: 476.53 ms/ques (linking tool: 383.89 ms/ques), relation detection: 930.62 ms/ques (similarity: 704.59 ms/ques), EDG generation: 46.21 ms/ques +[INFO] Cumulative metrics, sample: 769, P: 0.496, R: 0.549, macro F1: 0.521 +[INFO] Current question consumed time: 2156 ms, average: 2581.57 ms/ques, entity detection: 420.66 ms/ques (linking tool: 342.67 ms/ques), relation detection: 857.89 ms/ques (similarity: 696.68 ms/ques), EDG generation: 46.24 ms/ques [INFO] Entity Linking P: 0.741222366710013, R: 0.694408322496749, micro F1: 0.7093628088426537, macro F1: 0.7170520721433822 [INFO] Relation Linking P: 0.5980710879930645, R: 0.5435630689206762, micro F1: 0.5589943649761603, macro F1: 0.5695158191501114 [INFO] Type Linking P: 0.5513654096228868, R: 0.5513654096228868, micro F1: 0.5513654096228868, macro F1: 0.5513654096228868 @@ -10797,8 +10797,8 @@ Question 769: Which team has had both Martin prochzka and garry unger as its pla [INFO] Golden Answer: [http://dbpedia.org/resource/Toronto_Maple_Leafs] [INFO] Predicted Answer: [http://dbpedia.org/resource/HC_Ajoie] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 770, P: 0.496, R: 0.549, macro F1: 0.503, macro F1*: 0.521 -[INFO] Current time: 3412 ms, average: 2869.66 ms/ques, entity detection: 476.31 ms/ques (linking tool: 383.61 ms/ques), relation detection: 930.86 ms/ques (similarity: 705.26 ms/ques), EDG generation: 46.23 ms/ques +[INFO] Cumulative metrics, sample: 770, P: 0.496, R: 0.549, macro F1: 0.521 +[INFO] Current question consumed time: 2971 ms, average: 2582.08 ms/ques, entity detection: 420.41 ms/ques (linking tool: 342.38 ms/ques), relation detection: 858.12 ms/ques (similarity: 697.37 ms/ques), EDG generation: 46.26 ms/ques [INFO] Entity Linking P: 0.7402597402597403, R: 0.6935064935064935, micro F1: 0.7084415584415593, macro F1: 0.7161208356860532 [INFO] Relation Linking P: 0.5972943722943722, R: 0.5428571428571428, micro F1: 0.5582683982683991, macro F1: 0.5687761882161502 [INFO] Type Linking P: 0.551948051948052, R: 0.551948051948052, micro F1: 0.551948051948052, macro F1: 0.551948051948052 @@ -10811,8 +10811,8 @@ Question 770: What are the awards received by spouse of Ellen Lundstrom? [INFO] Golden Answer: [http://dbpedia.org/resource/Academy_Awards, http://dbpedia.org/resource/Goethe_Prize, http://dbpedia.org/resource/Praemium_Imperiale] [INFO] Predicted Answer: [http://dbpedia.org/resource/Minnesota] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 771, P: 0.495, R: 0.548, macro F1: 0.502, macro F1*: 0.520 -[INFO] Current time: 730 ms, average: 2866.89 ms/ques, entity detection: 475.89 ms/ques (linking tool: 383.25 ms/ques), relation detection: 929.81 ms/ques (similarity: 704.44 ms/ques), EDG generation: 46.19 ms/ques +[INFO] Cumulative metrics, sample: 771, P: 0.495, R: 0.548, macro F1: 0.520 +[INFO] Current question consumed time: 568 ms, average: 2579.46 ms/ques, entity detection: 419.99 ms/ques (linking tool: 342.03 ms/ques), relation detection: 857.13 ms/ques (similarity: 696.56 ms/ques), EDG generation: 46.23 ms/ques [INFO] Entity Linking P: 0.7392996108949417, R: 0.6926070038910506, micro F1: 0.7075226977950723, macro F1: 0.7151920148874978 [INFO] Relation Linking P: 0.5965196714223951, R: 0.5421530479896238, micro F1: 0.5575443147427591, macro F1: 0.5680384759097739 [INFO] Type Linking P: 0.5525291828793775, R: 0.5525291828793775, micro F1: 0.5525291828793775, macro F1: 0.5525291828793775 @@ -10825,8 +10825,8 @@ Question 771: Who all have succeeded the kings buried in Rome? [INFO] Golden Answer: [http://dbpedia.org/resource/Tiberius, http://dbpedia.org/resource/Claudius, http://dbpedia.org/resource/Caligula, http://dbpedia.org/resource/Kingdom_of_Italy_(Napoleonic)] [INFO] Predicted Answer: [http://dbpedia.org/resource/Charles_Albert_of_Sardinia, http://dbpedia.org/resource/Umberto_I_of_Italy] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 772, P: 0.494, R: 0.547, macro F1: 0.502, macro F1*: 0.519 -[INFO] Current time: 9509 ms, average: 2875.49 ms/ques, entity detection: 476.76 ms/ques (linking tool: 384.09 ms/ques), relation detection: 933.57 ms/ques (similarity: 708.00 ms/ques), EDG generation: 46.29 ms/ques +[INFO] Cumulative metrics, sample: 772, P: 0.494, R: 0.547, macro F1: 0.519 +[INFO] Current question consumed time: 8333 ms, average: 2586.92 ms/ques, entity detection: 420.71 ms/ques (linking tool: 342.74 ms/ques), relation detection: 860.53 ms/ques (similarity: 700.20 ms/ques), EDG generation: 46.19 ms/ques [INFO] Entity Linking P: 0.7396373056994818, R: 0.6930051813471503, micro F1: 0.7079015544041459, macro F1: 0.7155623120239111 [INFO] Relation Linking P: 0.596394645941278, R: 0.542098445595855, micro F1: 0.5574697754749577, macro F1: 0.5679518179419927 [INFO] Type Linking P: 0.5518134715025906, R: 0.5518134715025906, micro F1: 0.5518134715025906, macro F1: 0.5518134715025906 @@ -10839,8 +10839,8 @@ Question 772: Through which states do the rivers flow which join in the republic [INFO] Golden Answer: [http://dbpedia.org/resource/Nebraska, http://dbpedia.org/resource/Colorado] [INFO] Predicted Answer: [http://dbpedia.org/resource/Missouri_River, http://dbpedia.org/resource/Nebraska, http://dbpedia.org/resource/Kansas, http://dbpedia.org/resource/Colorado] [INFO] P: 0.5, R: 1.0, F: 0.6666666666666666 -[INFO] Cumulative metrics, sample: 773, P: 0.494, R: 0.548, macro F1: 0.502, macro F1*: 0.520 -[INFO] Current time: 3806 ms, average: 2876.69 ms/ques, entity detection: 476.42 ms/ques (linking tool: 383.73 ms/ques), relation detection: 934.10 ms/ques (similarity: 709.03 ms/ques), EDG generation: 46.28 ms/ques +[INFO] Cumulative metrics, sample: 773, P: 0.494, R: 0.548, macro F1: 0.520 +[INFO] Current question consumed time: 3362 ms, average: 2587.92 ms/ques, entity detection: 420.40 ms/ques (linking tool: 342.42 ms/ques), relation detection: 860.99 ms/ques (similarity: 701.21 ms/ques), EDG generation: 46.18 ms/ques [INFO] Entity Linking P: 0.7399741267787839, R: 0.6934023285899095, micro F1: 0.7082794307891341, macro F1: 0.7159316461253217 [INFO] Relation Linking P: 0.596269943941354, R: 0.5420439844760673, micro F1: 0.5573954290642527, macro F1: 0.5678653808385506 [INFO] Type Linking P: 0.5510996119016818, R: 0.5510996119016818, micro F1: 0.5510996119016818, macro F1: 0.5510996119016818 @@ -10853,8 +10853,8 @@ Question 773: Give me the home town of all musical artists who uses Guitar as in [INFO] Golden Answer: [http://dbpedia.org/resource/Cologne, http://dbpedia.org/resource/North_Rhine-Westphalia, http://dbpedia.org/resource/West_Germany, http://dbpedia.org/resource/Chicago, http://dbpedia.org/resource/Columbia,_Missouri, http://dbpedia.org/resource/Kildare, http://dbpedia.org/resource/Mobile,_Alabama, http://dbpedia.org/resource/Wynnewood,_Pennsylvania, http://dbpedia.org/resource/Canada, http://dbpedia.org/resource/Montreal, http://dbpedia.org/resource/Quebec, http://dbpedia.org/resource/Godalming, http://dbpedia.org/resource/Brazil, http://dbpedia.org/resource/Rio_de_Janeiro, http://dbpedia.org/resource/Leesburg,_Georgia, http://dbpedia.org/resource/Cincinnati, http://dbpedia.org/resource/Ohio, http://dbpedia.org/resource/United_States, http://dbpedia.org/resource/Stockholm, http://dbpedia.org/resource/England, http://dbpedia.org/resource/Guildford, http://dbpedia.org/resource/Surrey, http://dbpedia.org/resource/Sydney, http://dbpedia.org/resource/Denmark, http://dbpedia.org/resource/Bornholm, http://dbpedia.org/resource/Canton,_North_Carolina, http://dbpedia.org/resource/Whitesville,_West_Virginia, http://dbpedia.org/resource/Norway, http://dbpedia.org/resource/California, http://dbpedia.org/resource/Los_Angeles, http://dbpedia.org/resource/Blue_Springs,_Missouri, http://dbpedia.org/resource/Medway, http://dbpedia.org/resource/Hoo_St_Werburgh, http://dbpedia.org/resource/Birmingham, http://dbpedia.org/resource/West_Midlands_(county), http://dbpedia.org/resource/Oakland,_California, http://dbpedia.org/resource/New_Rochelle,_New_York, http://dbpedia.org/resource/Brooklyn, http://dbpedia.org/resource/New_York_City, http://dbpedia.org/resource/Ontario, http://dbpedia.org/resource/Toronto, http://dbpedia.org/resource/Lubbock,_Texas, http://dbpedia.org/resource/Sunderland,_Tyne_and_Wear, http://dbpedia.org/resource/Memphis,_Tennessee, http://dbpedia.org/resource/Waveland,_Mississippi, http://dbpedia.org/resource/Greece, http://dbpedia.org/resource/Australia, http://dbpedia.org/resource/Gothenburg, http://dbpedia.org/resource/Houston, http://dbpedia.org/resource/Illinois, http://dbpedia.org/resource/Mainz, http://dbpedia.org/resource/Greenville,_South_Carolina, http://dbpedia.org/resource/Holstebro, http://dbpedia.org/resource/Cuba, http://dbpedia.org/resource/Havana, http://dbpedia.org/resource/Vancouver, http://dbpedia.org/resource/Bahia, http://dbpedia.org/resource/Juazeiro, http://dbpedia.org/resource/Mexico, http://dbpedia.org/resource/Easthampton,_Massachusetts, http://dbpedia.org/resource/Massachusetts, http://dbpedia.org/resource/San_Diego, http://dbpedia.org/resource/Belgrade, http://dbpedia.org/resource/Serbia, http://dbpedia.org/resource/Tennessee, http://dbpedia.org/resource/Boom,_Belgium, http://dbpedia.org/resource/Antwerp, http://dbpedia.org/resource/Belgium, http://dbpedia.org/resource/Flanders, http://dbpedia.org/resource/Seattle, http://dbpedia.org/resource/Dutch_East_Indies, http://dbpedia.org/resource/Batavia,_Dutch_East_Indies, http://dbpedia.org/resource/Java, http://dbpedia.org/resource/Marietta,_Georgia, http://dbpedia.org/resource/Boston, http://dbpedia.org/resource/New_York, http://dbpedia.org/resource/Maynardville,_Tennessee, http://dbpedia.org/resource/New_Jersey, http://dbpedia.org/resource/Indiana, http://dbpedia.org/resource/Indianapolis, http://dbpedia.org/resource/Dallas, http://dbpedia.org/resource/Berkeley,_California, http://dbpedia.org/resource/Morton_Grove,_Illinois, http://dbpedia.org/resource/Lincolnshire, http://dbpedia.org/resource/Scunthorpe, http://dbpedia.org/resource/Comanche,_Oklahoma, http://dbpedia.org/resource/London, http://dbpedia.org/resource/Holland,_Michigan, http://dbpedia.org/resource/Texas, http://dbpedia.org/resource/Denton,_Texas, http://dbpedia.org/resource/Sainte-Foy,_Quebec_City, http://dbpedia.org/resource/Waldorf,_Maryland, http://dbpedia.org/resource/Washington,_D.C., http://dbpedia.org/resource/Melbourne, http://dbpedia.org/resource/Bristol, http://dbpedia.org/resource/Italy, http://dbpedia.org/resource/Terrazzo,_Veneto, http://dbpedia.org/resource/Buffalo,_New_York, http://dbpedia.org/resource/Adelaide, http://dbpedia.org/resource/San_Francisco, http://dbpedia.org/resource/Oklahoma, http://dbpedia.org/resource/Oklahoma_City, http://dbpedia.org/resource/Oxfordshire, http://dbpedia.org/resource/Abingdon-on-Thames, http://dbpedia.org/resource/Pennsylvania, http://dbpedia.org/resource/Philadelphia, http://dbpedia.org/resource/Garden_Grove,_California, http://dbpedia.org/resource/Riverdale,_Bronx, http://dbpedia.org/resource/Redlands,_California, http://dbpedia.org/resource/Wheeling,_West_Virginia, http://dbpedia.org/resource/Abilene,_Texas, http://dbpedia.org/resource/Netherlands, http://dbpedia.org/resource/The_Hague, http://dbpedia.org/resource/Istanbul, http://dbpedia.org/resource/Turkey, http://dbpedia.org/resource/Germany, http://dbpedia.org/resource/Bad_Homburg_vor_der_Höhe, http://dbpedia.org/resource/Waco,_Texas, http://dbpedia.org/resource/Princeton,_New_Jersey, http://dbpedia.org/resource/Rochester,_New_York, http://dbpedia.org/resource/Sweden, http://dbpedia.org/resource/New_Zealand, http://dbpedia.org/resource/Wellington, http://dbpedia.org/resource/Idaho, http://dbpedia.org/resource/Boise,_Idaho, http://dbpedia.org/resource/Goirle, http://dbpedia.org/resource/Crowley,_Louisiana, http://dbpedia.org/resource/Austria, http://dbpedia.org/resource/Vienna, http://dbpedia.org/resource/Dalserf, http://dbpedia.org/resource/Scotland, http://dbpedia.org/resource/South_Lanarkshire, http://dbpedia.org/resource/Gold_Coast,_Queensland, http://dbpedia.org/resource/Chile, http://dbpedia.org/resource/Catemu, http://dbpedia.org/resource/Skive,_Denmark, http://dbpedia.org/resource/Copenhagen, http://dbpedia.org/resource/Cape_Verde, http://dbpedia.org/resource/Mindelo, http://dbpedia.org/resource/São_Vicente,_Cape_Verde, http://dbpedia.org/resource/United_Kingdom, http://dbpedia.org/resource/Lancashire, http://dbpedia.org/resource/Darwen, http://dbpedia.org/resource/Athens,_Georgia, http://dbpedia.org/resource/Madrid, http://dbpedia.org/resource/Spain, http://dbpedia.org/resource/Greenwich_Village, http://dbpedia.org/resource/Carrboro,_North_Carolina, http://dbpedia.org/resource/Finland, http://dbpedia.org/resource/Helsinki, http://dbpedia.org/resource/Keiser,_Arkansas, http://dbpedia.org/resource/Concord,_North_Carolina, http://dbpedia.org/resource/Freeport,_New_York, http://dbpedia.org/resource/Harman,_West_Virginia, http://dbpedia.org/resource/Puerto_Rico, http://dbpedia.org/resource/Vega_Alta,_Puerto_Rico, http://dbpedia.org/resource/Tervo, http://dbpedia.org/resource/Auckland, http://dbpedia.org/resource/Gadsden,_Alabama, http://dbpedia.org/resource/Florida, http://dbpedia.org/resource/Sacramento,_California, http://dbpedia.org/resource/Basque_Country_(autonomous_community), http://dbpedia.org/resource/Florence,_Alabama, http://dbpedia.org/resource/Center_Point,_Alabama, http://dbpedia.org/resource/Loachapoka,_Alabama, http://dbpedia.org/resource/Etne, http://dbpedia.org/resource/Columbus,_Ohio, http://dbpedia.org/resource/Tottenham, http://dbpedia.org/resource/Menlo_Park,_California, http://dbpedia.org/resource/Killen,_Alabama, http://dbpedia.org/resource/South_Windsor,_Connecticut, http://dbpedia.org/resource/Panama_City,_Florida, http://dbpedia.org/resource/Pooler,_Georgia, http://dbpedia.org/resource/Morrison,_Colorado, http://dbpedia.org/resource/Marysville,_California, http://dbpedia.org/resource/Albany,_Georgia, http://dbpedia.org/resource/Honolulu, http://dbpedia.org/resource/Kansas, http://dbpedia.org/resource/Kinsley,_Kansas, http://dbpedia.org/resource/Iowa_City,_Iowa, http://dbpedia.org/resource/Nashville,_Tennessee, http://dbpedia.org/resource/Graceville,_Minnesota, http://dbpedia.org/resource/Royal_Oak,_Michigan, http://dbpedia.org/resource/Hernando,_Mississippi, http://dbpedia.org/resource/Corinth,_Mississippi, http://dbpedia.org/resource/Mississippi, http://dbpedia.org/resource/Amersfoort, http://dbpedia.org/resource/Muskegon,_Michigan, http://dbpedia.org/resource/Hopewell,_New_Jersey, http://dbpedia.org/resource/Boonton,_New_Jersey, http://dbpedia.org/resource/Jamestown,_New_York, http://dbpedia.org/resource/Long_Island, http://dbpedia.org/resource/Gillette,_New_Jersey, http://dbpedia.org/resource/Carteret,_New_Jersey, http://dbpedia.org/resource/Fairport,_New_York, http://dbpedia.org/resource/New_Brunswick,_New_Jersey, http://dbpedia.org/resource/Waverly,_Tennessee, http://dbpedia.org/resource/Sabina,_Ohio, http://dbpedia.org/resource/Canby,_Oregon, http://dbpedia.org/resource/Kettering,_Ohio, http://dbpedia.org/resource/Glen_Dale,_West_Virginia, http://dbpedia.org/resource/Bethel_Acres,_Oklahoma, http://dbpedia.org/resource/Bandera,_Texas, http://dbpedia.org/resource/Pampa,_Texas, http://dbpedia.org/resource/Utah, http://dbpedia.org/resource/Salt_Lake_City, http://dbpedia.org/resource/Austin,_Texas, http://dbpedia.org/resource/Pulaski,_Tennessee, http://dbpedia.org/resource/France, http://dbpedia.org/resource/Toulouse, http://dbpedia.org/resource/Fort_Irwin_National_Training_Center, http://dbpedia.org/resource/El_Paso,_Texas, http://dbpedia.org/resource/Godley,_Texas, http://dbpedia.org/resource/Yonkers,_New_York, http://dbpedia.org/resource/Puryear,_Tennessee, http://dbpedia.org/resource/Gainesville,_Texas, http://dbpedia.org/resource/Sumter,_South_Carolina, http://dbpedia.org/resource/Wise_County,_Texas, http://dbpedia.org/resource/Bath,_Somerset, http://dbpedia.org/resource/Bible_Hill,_Nova_Scotia, http://dbpedia.org/resource/Oslo, http://dbpedia.org/resource/Sheffield, http://dbpedia.org/resource/Hillsborough,_Sheffield, http://dbpedia.org/resource/San_Antonio, http://dbpedia.org/resource/Olathe,_Kansas, http://dbpedia.org/resource/Providence,_Rhode_Island, http://dbpedia.org/resource/Stavanger, http://dbpedia.org/resource/Oshawa, http://dbpedia.org/resource/Minneapolis, http://dbpedia.org/resource/Minnesota, http://dbpedia.org/resource/Wiener_Neustadt, http://dbpedia.org/resource/Kentucky, http://dbpedia.org/resource/Louisville,_Kentucky, http://dbpedia.org/resource/Newcastle_upon_Tyne, http://dbpedia.org/resource/British_Columbia, http://dbpedia.org/resource/Martin_County,_Kentucky, http://dbpedia.org/resource/Arkansas, http://dbpedia.org/resource/Fayetteville,_Arkansas, http://dbpedia.org/resource/Rothesay,_Bute, http://dbpedia.org/resource/Okemah,_Oklahoma, http://dbpedia.org/resource/Alberta, http://dbpedia.org/resource/Calgary, http://dbpedia.org/resource/Copeville,_Texas, http://dbpedia.org/resource/Archerwill,_Saskatchewan, http://dbpedia.org/resource/Fremont,_Nebraska, http://dbpedia.org/resource/Karnes_City,_Texas, http://dbpedia.org/resource/Phoenix,_Arizona, http://dbpedia.org/resource/Daytona_Beach,_Florida, http://dbpedia.org/resource/South_Australia, http://dbpedia.org/resource/Georgetown,_Texas, http://dbpedia.org/resource/Sherman,_Texas, http://dbpedia.org/resource/Columbia,_South_Carolina, http://dbpedia.org/resource/Uxbridge,_Ontario, http://dbpedia.org/resource/Success,_Missouri, http://dbpedia.org/resource/Asheville,_North_Carolina, http://dbpedia.org/resource/Fort_Wayne,_Indiana, http://dbpedia.org/resource/Devon, http://dbpedia.org/resource/Sparta,_Illinois, http://dbpedia.org/resource/Woodside,_California, http://dbpedia.org/resource/Minas_Gerais, http://dbpedia.org/resource/Sete_Lagoas, http://dbpedia.org/resource/Baytown,_Texas, http://dbpedia.org/resource/Seneca,_Missouri, http://dbpedia.org/resource/Maple_Ridge,_British_Columbia, http://dbpedia.org/resource/Caddo,_Oklahoma, http://dbpedia.org/resource/Breckenridge,_Texas, http://dbpedia.org/resource/Hiroshima_Prefecture, http://dbpedia.org/resource/Japan, http://dbpedia.org/resource/Takehara,_Hiroshima, http://dbpedia.org/resource/Lake_Charles,_Louisiana, http://dbpedia.org/resource/Roblin,_Manitoba, http://dbpedia.org/resource/Binghamton,_New_York, http://dbpedia.org/resource/Midwest_City,_Oklahoma, http://dbpedia.org/resource/Duncan,_Arizona, http://dbpedia.org/resource/Karachi, http://dbpedia.org/resource/Pakistan, http://dbpedia.org/resource/Jordan, http://dbpedia.org/resource/State_of_Palestine, http://dbpedia.org/resource/Little_Rock,_Arkansas, http://dbpedia.org/resource/Valletta, http://dbpedia.org/resource/Palma,_Majorca, http://dbpedia.org/resource/Balearic_Islands, http://dbpedia.org/resource/Santa_Barbara,_California, http://dbpedia.org/resource/Porto_Alegre, http://dbpedia.org/resource/Portland,_Oregon, http://dbpedia.org/resource/Oregon, http://dbpedia.org/resource/South_Africa, http://dbpedia.org/resource/KwaZulu-Natal, http://dbpedia.org/resource/Midlands_of_KwaZulu-Natal, http://dbpedia.org/resource/Bennington,_Vermont, http://dbpedia.org/resource/Orlando,_Florida, http://dbpedia.org/resource/Fries,_Virginia, http://dbpedia.org/resource/Wantagh,_New_York, http://dbpedia.org/resource/The_Bellmores,_New_York, http://dbpedia.org/resource/Queens, http://dbpedia.org/resource/Tulsa,_Oklahoma, http://dbpedia.org/resource/Hawaii, http://dbpedia.org/resource/Ames,_Iowa, http://dbpedia.org/resource/Bronaugh,_Missouri, http://dbpedia.org/resource/New_South_Wales, http://dbpedia.org/resource/London_Borough_of_Hackney, http://dbpedia.org/resource/Durban, http://dbpedia.org/resource/Tuusula, http://dbpedia.org/resource/Nepal, http://dbpedia.org/resource/Pittsburgh, http://dbpedia.org/resource/Glendale,_California, http://dbpedia.org/resource/Ivory_Coast, http://dbpedia.org/resource/Jamaica, http://dbpedia.org/resource/North_Carolina, http://dbpedia.org/resource/Research_Triangle, http://dbpedia.org/resource/Montevideo, http://dbpedia.org/resource/Uruguay, http://dbpedia.org/resource/Niagara_Falls,_New_York, http://dbpedia.org/resource/Switzerland, http://dbpedia.org/resource/Zürich, http://dbpedia.org/resource/Fort_Worth,_Texas, http://dbpedia.org/resource/Fresno,_California, http://dbpedia.org/resource/Vallejo,_California, http://dbpedia.org/resource/Essex, http://dbpedia.org/resource/St._Petersburg,_Florida, http://dbpedia.org/resource/New_Mexico, http://dbpedia.org/resource/South_Korea, http://dbpedia.org/resource/Brownsville,_Texas, http://dbpedia.org/resource/Kingston,_Ontario, http://dbpedia.org/resource/Castaic,_California, http://dbpedia.org/resource/Republic_of_Ireland, http://dbpedia.org/resource/County_Wicklow, http://dbpedia.org/resource/Lisbon, http://dbpedia.org/resource/Sleepy_Hollow,_New_York, http://dbpedia.org/resource/Tampa,_Florida, http://dbpedia.org/resource/Ridgewood,_New_Jersey, http://dbpedia.org/resource/Silver_Lake,_Los_Angeles, http://dbpedia.org/resource/Fraser_Canyon, http://dbpedia.org/resource/London,_Ontario, http://dbpedia.org/resource/Kasai,_Hyōgo, http://dbpedia.org/resource/Gilroy,_California, http://dbpedia.org/resource/Somers_Point,_New_Jersey, http://dbpedia.org/resource/Bad_Windsheim, http://dbpedia.org/resource/Barnesboro,_Pennsylvania, http://dbpedia.org/resource/Carleton_Place, http://dbpedia.org/resource/Fremont,_California, http://dbpedia.org/resource/Cheshire, http://dbpedia.org/resource/Philippines, http://dbpedia.org/resource/Cebu_City, http://dbpedia.org/resource/Union_City,_Tennessee, http://dbpedia.org/resource/Brisbane, http://dbpedia.org/resource/Queensland, http://dbpedia.org/resource/Lebanon, http://dbpedia.org/resource/Tripoli,_Lebanon, http://dbpedia.org/resource/Sebastopol,_Texas, http://dbpedia.org/resource/Hyattsville,_Maryland, http://dbpedia.org/resource/Woodbridge,_Virginia, http://dbpedia.org/resource/Brixton, http://dbpedia.org/resource/Bremerton,_Washington, http://dbpedia.org/resource/Kosovo, http://dbpedia.org/resource/Wollongong, http://dbpedia.org/resource/Cheongju, http://dbpedia.org/resource/North_Chungcheong_Province, http://dbpedia.org/resource/Vietnam, http://dbpedia.org/resource/Córdoba,_Andalusia, http://dbpedia.org/resource/Novi_Sad, http://dbpedia.org/resource/Atlanta, http://dbpedia.org/resource/Georgia_(U.S._state), http://dbpedia.org/resource/Russian_Soviet_Federative_Socialist_Republic, http://dbpedia.org/resource/Yekaterinburg, http://dbpedia.org/resource/Bodø, http://dbpedia.org/resource/Milan, http://dbpedia.org/resource/Shetland, http://dbpedia.org/resource/Grand_Rapids,_Michigan, http://dbpedia.org/resource/Spokane,_Washington, http://dbpedia.org/resource/Franklin_Lakes,_New_Jersey, http://dbpedia.org/resource/Grycksbo, http://dbpedia.org/resource/Falun, http://dbpedia.org/resource/Virginia, http://dbpedia.org/resource/Catalonia, http://dbpedia.org/resource/Sabadell, http://dbpedia.org/resource/Tokyo, http://dbpedia.org/resource/Ventura,_California, http://dbpedia.org/resource/Maryland, http://dbpedia.org/resource/Linz, http://dbpedia.org/resource/Europe, http://dbpedia.org/resource/Iran, http://dbpedia.org/resource/Tehran, http://dbpedia.org/resource/Tasmania, http://dbpedia.org/resource/Franklin,_Tasmania, http://dbpedia.org/resource/Harlem, http://dbpedia.org/resource/Winston-Salem,_North_Carolina, http://dbpedia.org/resource/Nacogdoches,_Texas, http://dbpedia.org/resource/Finglas, http://dbpedia.org/resource/County_Dublin, http://dbpedia.org/resource/Louisiana, http://dbpedia.org/resource/New_Orleans, http://dbpedia.org/resource/Iceland, http://dbpedia.org/resource/Amsterdam, http://dbpedia.org/resource/Homer,_Alaska, http://dbpedia.org/resource/Waterloo,_Ontario, http://dbpedia.org/resource/Forest_Lake,_Minnesota, http://dbpedia.org/resource/Southlake,_Texas, http://dbpedia.org/resource/Alaska, http://dbpedia.org/resource/Anchorage,_Alaska, http://dbpedia.org/resource/San_Juan,_Puerto_Rico, http://dbpedia.org/resource/Halstead, http://dbpedia.org/resource/Bay_Village,_Ohio, http://dbpedia.org/resource/Chicopee,_Massachusetts, http://dbpedia.org/resource/Hemel_Hempstead, http://dbpedia.org/resource/Hertfordshire, http://dbpedia.org/resource/Aarhus, http://dbpedia.org/resource/Germans, http://dbpedia.org/resource/Huế, http://dbpedia.org/resource/Dayton,_Ohio, http://dbpedia.org/resource/Argentina, http://dbpedia.org/resource/Buenos_Aires, http://dbpedia.org/resource/Richmond,_Virginia, http://dbpedia.org/resource/South_Lyon,_Michigan, http://dbpedia.org/resource/Moscow, http://dbpedia.org/resource/Soviet_Union, http://dbpedia.org/resource/Rodeo,_California, http://dbpedia.org/resource/Lexington,_Kentucky, http://dbpedia.org/resource/Portsmouth,_Virginia, http://dbpedia.org/resource/Chesapeake,_Virginia, http://dbpedia.org/resource/Castle_Hill,_New_South_Wales, http://dbpedia.org/resource/Carpentras, http://dbpedia.org/resource/Keithville,_Louisiana, http://dbpedia.org/resource/Caddo_Parish,_Louisiana, http://dbpedia.org/resource/Hawthorne,_California, http://dbpedia.org/resource/Beech_Mountain,_North_Carolina, http://dbpedia.org/resource/Beijing, http://dbpedia.org/resource/China, http://dbpedia.org/resource/San_Fernando_Valley, http://dbpedia.org/resource/Rock_Island,_Illinois, http://dbpedia.org/resource/Mayfield,_Kentucky, http://dbpedia.org/resource/Walla_Walla,_Washington, http://dbpedia.org/resource/Nagasaki, http://dbpedia.org/resource/Detroit, http://dbpedia.org/resource/Jacksonville,_Florida, http://dbpedia.org/resource/Wimbledon,_London, http://dbpedia.org/resource/Ruston,_Louisiana, http://dbpedia.org/resource/Las_Vegas_Valley, http://dbpedia.org/resource/Nevada, http://dbpedia.org/resource/Daisy,_Kentucky, http://dbpedia.org/resource/Russia, http://dbpedia.org/resource/Trenton,_New_Jersey, http://dbpedia.org/resource/Brest,_France, http://dbpedia.org/resource/Brittany, http://dbpedia.org/resource/Angeles,_Philippines, http://dbpedia.org/resource/Pampanga, http://dbpedia.org/resource/Perth, http://dbpedia.org/resource/Ann_Arbor,_Michigan, http://dbpedia.org/resource/Paris,_Illinois, http://dbpedia.org/resource/Marlton,_New_Jersey, http://dbpedia.org/resource/Caldwell_County,_North_Carolina, http://dbpedia.org/resource/Darby,_Pennsylvania, http://dbpedia.org/resource/Hamilton,_New_Zealand, http://dbpedia.org/resource/Dublin, http://dbpedia.org/resource/Glasgow, http://dbpedia.org/resource/Albury, http://dbpedia.org/resource/Poggio_Bustone, http://dbpedia.org/resource/Burbank,_California, http://dbpedia.org/resource/Hiroshima, http://dbpedia.org/resource/Leeds, http://dbpedia.org/resource/Oakville,_Ontario, http://dbpedia.org/resource/Newton,_Mississippi, http://dbpedia.org/resource/New_Brunswick, http://dbpedia.org/resource/Edmundston, http://dbpedia.org/resource/Oxford, http://dbpedia.org/resource/Elverum, http://dbpedia.org/resource/Hedmark, http://dbpedia.org/resource/Hollywood, http://dbpedia.org/resource/South_Orange,_New_Jersey, http://dbpedia.org/resource/Ottawa, http://dbpedia.org/resource/Berlin, http://dbpedia.org/resource/Chemnitz, http://dbpedia.org/resource/Saxony, http://dbpedia.org/resource/Kansas_City,_Kansas, http://dbpedia.org/resource/St_Kilda,_Victoria, http://dbpedia.org/resource/Stellenbosch, http://dbpedia.org/resource/Shreveport,_Louisiana, http://dbpedia.org/resource/Chapel_Hill,_North_Carolina, http://dbpedia.org/resource/West_Chester,_Pennsylvania, http://dbpedia.org/resource/Blenheim,_New_Zealand, http://dbpedia.org/resource/Belfast, http://dbpedia.org/resource/Northern_Ireland, http://dbpedia.org/resource/Kirghiz_Soviet_Socialist_Republic, http://dbpedia.org/resource/Poland, http://dbpedia.org/resource/Enville,_Oklahoma, http://dbpedia.org/resource/Aabyhøj, http://dbpedia.org/resource/South_Carolina, http://dbpedia.org/resource/Laurens,_South_Carolina, http://dbpedia.org/resource/Canterbury, http://dbpedia.org/resource/Pamplona, http://dbpedia.org/resource/Chechen-Ingush_Autonomous_Soviet_Socialist_Republic, http://dbpedia.org/resource/Grozny, http://dbpedia.org/resource/Humphreys_County,_Mississippi, http://dbpedia.org/resource/Winthrop,_Massachusetts, http://dbpedia.org/resource/Wuppertal, http://dbpedia.org/resource/Bowie,_Maryland, http://dbpedia.org/resource/Country_music, http://dbpedia.org/resource/Karlshamn, http://dbpedia.org/resource/Tomnavoulin, http://dbpedia.org/resource/Banffshire, http://dbpedia.org/resource/Geraardsbergen, http://dbpedia.org/resource/Kingston,_Jamaica, http://dbpedia.org/resource/Israel, http://dbpedia.org/resource/Jerusalem, http://dbpedia.org/resource/Gloucestershire, http://dbpedia.org/resource/Cheltenham, http://dbpedia.org/resource/Rugby,_Warwickshire, http://dbpedia.org/resource/Beersheba, http://dbpedia.org/resource/Farmerville,_Louisiana, http://dbpedia.org/resource/Cambridge,_Massachusetts, http://dbpedia.org/resource/Somerville,_Massachusetts, http://dbpedia.org/resource/Bastrop,_Louisiana, http://dbpedia.org/resource/New_Bedford,_Massachusetts, http://dbpedia.org/resource/Maine, http://dbpedia.org/resource/Portland,_Maine, http://dbpedia.org/resource/Knoxville,_Tennessee, http://dbpedia.org/resource/Lodi,_New_Jersey, http://dbpedia.org/resource/Booneville,_Mississippi, http://dbpedia.org/resource/Dumont,_New_Jersey, http://dbpedia.org/resource/Derbyshire, http://dbpedia.org/resource/Buxton, http://dbpedia.org/resource/Senatobia,_Mississippi, http://dbpedia.org/resource/St._Ignatius,_Montana, http://dbpedia.org/resource/Ramsey,_New_Jersey, http://dbpedia.org/resource/Sussex_County,_New_Jersey, http://dbpedia.org/resource/Hastings-on-Hudson,_New_York, http://dbpedia.org/resource/Dix_Hills,_New_York, http://dbpedia.org/resource/Fayetteville,_North_Carolina, http://dbpedia.org/resource/Venice,_Los_Angeles, http://dbpedia.org/resource/Derby, http://dbpedia.org/resource/Maplewood,_New_Jersey, http://dbpedia.org/resource/Cohoes,_New_York, http://dbpedia.org/resource/Cairo, http://dbpedia.org/resource/Egypt, http://dbpedia.org/resource/South_London, http://dbpedia.org/resource/West_Mifflin,_Pennsylvania, http://dbpedia.org/resource/Cleveland, http://dbpedia.org/resource/São_Paulo, http://dbpedia.org/resource/Voorheesville,_New_York, http://dbpedia.org/resource/Aguascalientes, http://dbpedia.org/resource/Newmarket,_Ontario, http://dbpedia.org/resource/Mexico_City, http://dbpedia.org/resource/Portugal, http://dbpedia.org/resource/Loures, http://dbpedia.org/resource/Cartago_Valle, http://dbpedia.org/resource/Colombia, http://dbpedia.org/resource/São_Caetano_do_Sul, http://dbpedia.org/resource/Hazard,_Kentucky, http://dbpedia.org/resource/Windsor,_Ontario, http://dbpedia.org/resource/Pacific_Palisades,_Los_Angeles, http://dbpedia.org/resource/Grafton,_New_South_Wales, http://dbpedia.org/resource/Seoul, http://dbpedia.org/resource/Northeastern_United_States, http://dbpedia.org/resource/Redondo_Beach,_California, http://dbpedia.org/resource/Moldova, http://dbpedia.org/resource/Bălți, http://dbpedia.org/resource/Nakano,_Tokyo, http://dbpedia.org/resource/Worthing, http://dbpedia.org/resource/West_Pubnico,_Nova_Scotia, http://dbpedia.org/resource/Brighton, http://dbpedia.org/resource/Barbados, http://dbpedia.org/resource/Hare_Bay,_Newfoundland_and_Labrador, http://dbpedia.org/resource/Guelph, http://dbpedia.org/resource/Richardson,_Texas, http://dbpedia.org/resource/Jackson,_Mississippi, http://dbpedia.org/resource/Reseda,_Los_Angeles, http://dbpedia.org/resource/Valparaíso, http://dbpedia.org/resource/Yorkshire, http://dbpedia.org/resource/Kingston_upon_Hull, http://dbpedia.org/resource/The_Bronx, http://dbpedia.org/resource/Xinjiang, http://dbpedia.org/resource/Norfolk,_Virginia, http://dbpedia.org/resource/Alicante, http://dbpedia.org/resource/West_Yorkshire, http://dbpedia.org/resource/Washington_(state), http://dbpedia.org/resource/South_Haven,_Michigan, http://dbpedia.org/resource/West_Midlands_(region), http://dbpedia.org/resource/Wolverhampton, http://dbpedia.org/resource/Nevada,_Missouri, http://dbpedia.org/resource/Southern_California, http://dbpedia.org/resource/Terneuzen, http://dbpedia.org/resource/Taylorsville,_Kentucky, http://dbpedia.org/resource/St._Louis, http://dbpedia.org/resource/Dharan,_Nepal, http://dbpedia.org/resource/Suffolk, http://dbpedia.org/resource/Bildeston, http://dbpedia.org/resource/Tampere, http://dbpedia.org/resource/Metropolitan_Borough_of_Oldham, http://dbpedia.org/resource/Greater_Manchester, http://dbpedia.org/resource/São_Paulo_(state), http://dbpedia.org/resource/Ribeirão_Preto, http://dbpedia.org/resource/Newfoundland_and_Labrador, http://dbpedia.org/resource/St._John's,_Newfoundland_and_Labrador, http://dbpedia.org/resource/Peru, http://dbpedia.org/resource/Callao, http://dbpedia.org/resource/Sri_Lanka, http://dbpedia.org/resource/Greater_Sudbury, http://dbpedia.org/resource/Mount_Dennis, http://dbpedia.org/resource/Kagoshima, http://dbpedia.org/resource/Kagoshima_Prefecture, http://dbpedia.org/resource/Terre_Haute,_Indiana, http://dbpedia.org/resource/County_Cavan, http://dbpedia.org/resource/Shercock, http://dbpedia.org/resource/Whittlesey, http://dbpedia.org/resource/Tajikistan, http://dbpedia.org/resource/Khorugh, http://dbpedia.org/resource/Airdrie,_Alberta, http://dbpedia.org/resource/Tacoma,_Washington, http://dbpedia.org/resource/Colorado, http://dbpedia.org/resource/Morrinsville, http://dbpedia.org/resource/Montana, http://dbpedia.org/resource/Venezuela, http://dbpedia.org/resource/Maracaibo, http://dbpedia.org/resource/Zulia, http://dbpedia.org/resource/Marseille, http://dbpedia.org/resource/Wall_Township,_New_Jersey, http://dbpedia.org/resource/Scarborough,_Toronto, http://dbpedia.org/resource/Kingston_upon_Thames, http://dbpedia.org/resource/Delhi, http://dbpedia.org/resource/India, http://dbpedia.org/resource/Haryana, http://dbpedia.org/resource/Bhiwani, http://dbpedia.org/resource/County_Kildare, http://dbpedia.org/resource/Celbridge, http://dbpedia.org/resource/Westchester_County,_New_York, http://dbpedia.org/resource/Malaysia, http://dbpedia.org/resource/Bukit_Jalil, http://dbpedia.org/resource/Carrara, http://dbpedia.org/resource/Tuscany, http://dbpedia.org/resource/Sullivan_County,_Pennsylvania, http://dbpedia.org/resource/Newport_Beach,_California, http://dbpedia.org/resource/Caloocan, http://dbpedia.org/resource/South_Holland, http://dbpedia.org/resource/Staten_Island, http://dbpedia.org/resource/Algeria, http://dbpedia.org/resource/Algiers, http://dbpedia.org/resource/Mohamed_Belouizdad, http://dbpedia.org/resource/Denver, http://dbpedia.org/resource/Erfurt, http://dbpedia.org/resource/Saint_Paul,_Minnesota, http://dbpedia.org/resource/Vantaa, http://dbpedia.org/resource/Beşiktaş, http://dbpedia.org/resource/Levent, http://dbpedia.org/resource/Valley_Stream,_New_York, http://dbpedia.org/resource/Campbell,_California, http://dbpedia.org/resource/Jacksonville,_Texas, http://dbpedia.org/resource/Canton,_Ohio, http://dbpedia.org/resource/Birmingham,_Alabama, http://dbpedia.org/resource/Tamaulipas, http://dbpedia.org/resource/Ciudad_Madero, http://dbpedia.org/resource/Korumburra, http://dbpedia.org/resource/New_Haven,_Connecticut, http://dbpedia.org/resource/Irvine,_North_Ayrshire, http://dbpedia.org/resource/Croatia, http://dbpedia.org/resource/Medicine_Hat, http://dbpedia.org/resource/Woodland,_Alabama, http://dbpedia.org/resource/Paris, http://dbpedia.org/resource/Fort_Scott,_Kansas, http://dbpedia.org/resource/Ankara, http://dbpedia.org/resource/Hamilton,_Ohio, http://dbpedia.org/resource/Kitee, http://dbpedia.org/resource/Paterson,_New_Jersey, http://dbpedia.org/resource/Surbiton, http://dbpedia.org/resource/Corpus_Christi,_Texas, http://dbpedia.org/resource/Pottstown,_Pennsylvania, http://dbpedia.org/resource/Fargo,_North_Dakota, http://dbpedia.org/resource/Cassine, http://dbpedia.org/resource/Stockton,_California, http://dbpedia.org/resource/Swillington, http://dbpedia.org/resource/Jujuy_Province, http://dbpedia.org/resource/Kyoto, http://dbpedia.org/resource/Mulberry,_Indiana, http://dbpedia.org/resource/Moreno_Valley,_California, http://dbpedia.org/resource/Albany,_New_York, http://dbpedia.org/resource/Bolton, http://dbpedia.org/resource/Ponca_City,_Oklahoma, http://dbpedia.org/resource/Santa_Monica,_California, http://dbpedia.org/resource/Clarion,_Pennsylvania, http://dbpedia.org/resource/Oromia_Region, http://dbpedia.org/resource/Dembidolo, http://dbpedia.org/resource/Utica,_New_York, http://dbpedia.org/resource/Wakefield, http://dbpedia.org/resource/Worsley, http://dbpedia.org/resource/Holtville,_Alabama, http://dbpedia.org/resource/Rayne,_Louisiana, http://dbpedia.org/resource/Madison,_Wisconsin, http://dbpedia.org/resource/Batangas, http://dbpedia.org/resource/Rosario,_Batangas, http://dbpedia.org/resource/Três_Pontas, http://dbpedia.org/resource/Kiryat_Gat, http://dbpedia.org/resource/Moose_Lake,_Minnesota, http://dbpedia.org/resource/Manila, http://dbpedia.org/resource/Shepherd's_Bush, http://dbpedia.org/resource/Dushanbe, http://dbpedia.org/resource/Andover,_Massachusetts, http://dbpedia.org/resource/Tver, http://dbpedia.org/resource/Michigan, http://dbpedia.org/resource/Duluth,_Minnesota, http://dbpedia.org/resource/Gifu_Prefecture, http://dbpedia.org/resource/Quantico,_Virginia, http://dbpedia.org/resource/New_Cross, http://dbpedia.org/resource/Morristown,_New_Jersey, http://dbpedia.org/resource/Neusiedl_am_See, http://dbpedia.org/resource/Santurce,_San_Juan,_Puerto_Rico, http://dbpedia.org/resource/Ardnacrusha, http://dbpedia.org/resource/Allegheny_County,_Pennsylvania, http://dbpedia.org/resource/Reykjavík, http://dbpedia.org/resource/Diamond_Bar,_California, http://dbpedia.org/resource/El_Dorado,_Arkansas, http://dbpedia.org/resource/Anaheim,_California, http://dbpedia.org/resource/Half_Moon_Bay,_California, http://dbpedia.org/resource/East_Bay_(San_Francisco_Bay_Area), http://dbpedia.org/resource/Northwest_Arkansas, http://dbpedia.org/resource/Ukiah,_California, http://dbpedia.org/resource/La_Habra,_California, http://dbpedia.org/resource/Rogers,_Arkansas, http://dbpedia.org/resource/Arizona, http://dbpedia.org/resource/Ørum_(Sønderlyng), http://dbpedia.org/resource/Yvelines, http://dbpedia.org/resource/Saint-Germain-en-Laye, http://dbpedia.org/resource/Île-de-France, http://dbpedia.org/resource/Osaka, http://dbpedia.org/resource/Widnes, http://dbpedia.org/resource/Nigeria, http://dbpedia.org/resource/Calabar, http://dbpedia.org/resource/Ghana, http://dbpedia.org/resource/Accra, http://dbpedia.org/resource/Harrisonburg,_Virginia, http://dbpedia.org/resource/Omaha,_Nebraska, http://dbpedia.org/resource/Nikšić, http://dbpedia.org/resource/Montenegro, http://dbpedia.org/resource/Missouri, http://dbpedia.org/resource/Western_Australia, http://dbpedia.org/resource/Albany,_Western_Australia, http://dbpedia.org/resource/Shepparton, http://dbpedia.org/resource/Amarillo,_Texas, http://dbpedia.org/resource/Eugene,_Oregon, http://dbpedia.org/resource/Acton,_Massachusetts, http://dbpedia.org/resource/County_Tipperary, http://dbpedia.org/resource/Carrick-on-Suir, http://dbpedia.org/resource/Belvidere,_Illinois, http://dbpedia.org/resource/Ayr,_Ontario, http://dbpedia.org/resource/Chișinău, http://dbpedia.org/resource/Hampshire, http://dbpedia.org/resource/Farnborough,_Hampshire, http://dbpedia.org/resource/North_Miami_Beach,_Florida, http://dbpedia.org/resource/Dieppe,_New_Brunswick, http://dbpedia.org/resource/Center_Moriches,_New_York, http://dbpedia.org/resource/Sindh, http://dbpedia.org/resource/Americans, http://dbpedia.org/resource/Lahore, http://dbpedia.org/resource/Punjab,_Pakistan, http://dbpedia.org/resource/Bærum, http://dbpedia.org/resource/Aomori_Prefecture, http://dbpedia.org/resource/Canberra, http://dbpedia.org/resource/Australian_Capital_Territory, http://dbpedia.org/resource/Oxford,_Mississippi, http://dbpedia.org/resource/Pasadena,_California, http://dbpedia.org/resource/Czechoslovakia, http://dbpedia.org/resource/Uherské_Hradiště, http://dbpedia.org/resource/Portrane, http://dbpedia.org/resource/Kamakura, http://dbpedia.org/resource/Kanagawa_Prefecture, http://dbpedia.org/resource/Arlington_County,_Virginia, http://dbpedia.org/resource/Wales, http://dbpedia.org/resource/Bongabong,_Oriental_Mindoro, http://dbpedia.org/resource/Oriental_Mindoro, http://dbpedia.org/resource/Ludvika, http://dbpedia.org/resource/Eastry, http://dbpedia.org/resource/Kent, http://dbpedia.org/resource/Aylesbury, http://dbpedia.org/resource/Buckinghamshire, http://dbpedia.org/resource/La_Habra_Heights,_California, http://dbpedia.org/resource/Marshall,_North_Carolina, http://dbpedia.org/resource/Revere,_North_Carolina, http://dbpedia.org/resource/Winnetka,_Illinois, http://dbpedia.org/resource/Orange_County,_California, http://dbpedia.org/resource/Paddington, http://dbpedia.org/resource/List_of_sub_regions_used_in_the_London_Plan, http://dbpedia.org/resource/Vermont, http://dbpedia.org/resource/Haiti, http://dbpedia.org/resource/Port-au-Prince, http://dbpedia.org/resource/Coppell,_Texas, http://dbpedia.org/resource/Alum_Creek,_West_Virginia, http://dbpedia.org/resource/Jansen,_Saskatchewan, http://dbpedia.org/resource/Knowlton_Township,_New_Jersey, http://dbpedia.org/resource/Halmstad, http://dbpedia.org/resource/Kristianstad, http://dbpedia.org/resource/Manhattan_Beach,_California, http://dbpedia.org/resource/St._Catharines, http://dbpedia.org/resource/Selangor, http://dbpedia.org/resource/Kajang, http://dbpedia.org/resource/Strängnäs, http://dbpedia.org/resource/Worcester,_Massachusetts, http://dbpedia.org/resource/Hamilton,_Ontario, http://dbpedia.org/resource/Cambridge, http://dbpedia.org/resource/Tamil_Nadu, http://dbpedia.org/resource/Georgia_(country), http://dbpedia.org/resource/Fuengirola, http://dbpedia.org/resource/Bowling_Green,_Kentucky, http://dbpedia.org/resource/Story_City,_Iowa, http://dbpedia.org/resource/Syrian_Americans, http://dbpedia.org/resource/Prague, http://dbpedia.org/resource/North_Vancouver_(city), http://dbpedia.org/resource/Katy,_Texas, http://dbpedia.org/resource/Westfield,_New_Jersey, http://dbpedia.org/resource/New_Ulm,_Minnesota, http://dbpedia.org/resource/Lagos, http://dbpedia.org/resource/Mount_Dora,_Florida, http://dbpedia.org/resource/Montpelier,_Vermont, http://dbpedia.org/resource/Totnes, http://dbpedia.org/resource/West_Hartford,_Connecticut, http://dbpedia.org/resource/Norton,_New_Brunswick, http://dbpedia.org/resource/Mount_Kisco,_New_York, http://dbpedia.org/resource/Shelby,_North_Carolina, http://dbpedia.org/resource/Indonesia, http://dbpedia.org/resource/Central_Java, http://dbpedia.org/resource/Franklin,_Tennessee, http://dbpedia.org/resource/Blackpool, http://dbpedia.org/resource/Manhattan, http://dbpedia.org/resource/Bradford, http://dbpedia.org/resource/Cwmaman, http://dbpedia.org/resource/Oran_Province, http://dbpedia.org/resource/Mission_Viejo,_California, http://dbpedia.org/resource/Nagasaki_Prefecture, http://dbpedia.org/resource/Des_Moines,_Iowa, http://dbpedia.org/resource/Iowa, http://dbpedia.org/resource/Crawley, http://dbpedia.org/resource/Tilburg, http://dbpedia.org/resource/Peterborough,_Ontario, http://dbpedia.org/resource/Mississauga, http://dbpedia.org/resource/DeKalb,_Illinois, http://dbpedia.org/resource/Wheaton,_Illinois, http://dbpedia.org/resource/Chickasaw_County,_Mississippi, http://dbpedia.org/resource/Kelowna, http://dbpedia.org/resource/Wikwemikong_First_Nation, http://dbpedia.org/resource/Jaffray,_British_Columbia, http://dbpedia.org/resource/Ware,_Massachusetts, http://dbpedia.org/resource/Linden,_Texas, http://dbpedia.org/resource/Sicily, http://dbpedia.org/resource/Riposto, http://dbpedia.org/resource/Waycross,_Georgia, http://dbpedia.org/resource/Stanthorpe,_Queensland, http://dbpedia.org/resource/Douglas,_Georgia, http://dbpedia.org/resource/Sabinal,_Texas, http://dbpedia.org/resource/Moorhead,_Mississippi, http://dbpedia.org/resource/Westland,_Michigan, http://dbpedia.org/resource/Cross_Lanes,_West_Virginia, http://dbpedia.org/resource/Sandy_Hook,_Kentucky, http://dbpedia.org/resource/Cairns, http://dbpedia.org/resource/Chatham-Kent, http://dbpedia.org/resource/Winnipeg, http://dbpedia.org/resource/Pikeville,_Kentucky, http://dbpedia.org/resource/Highland_Park,_Illinois, http://dbpedia.org/resource/Hawkesbury,_Ontario, http://dbpedia.org/resource/Hackleburg,_Alabama, http://dbpedia.org/resource/Frederiksberg, http://dbpedia.org/resource/Beauregard_Parish,_Louisiana, http://dbpedia.org/resource/San_Carlos,_Chile, http://dbpedia.org/resource/Morrisville,_Missouri, http://dbpedia.org/resource/Swarcliffe, http://dbpedia.org/resource/Streetsville,_Ontario, http://dbpedia.org/resource/Lyons,_New_York, http://dbpedia.org/resource/Henderson,_Nevada, http://dbpedia.org/resource/Lasker,_North_Carolina, http://dbpedia.org/resource/Grand_Saline,_Texas, http://dbpedia.org/resource/Springfield,_Virginia, http://dbpedia.org/resource/Murray,_Utah, http://dbpedia.org/resource/Hillsboro,_Oregon, http://dbpedia.org/resource/Meonstoke, http://dbpedia.org/resource/South_Shields, http://dbpedia.org/resource/Tyne_and_Wear, http://dbpedia.org/resource/Miyagi_Prefecture, http://dbpedia.org/resource/Montebello,_California, http://dbpedia.org/resource/West_Riding_of_Yorkshire, http://dbpedia.org/resource/Kirkstall, http://dbpedia.org/resource/Albuquerque,_New_Mexico, http://dbpedia.org/resource/Bryn_Mawr,_Pennsylvania, http://dbpedia.org/resource/Santa_Rosa,_California, http://dbpedia.org/resource/Englewood,_New_Jersey, http://dbpedia.org/resource/Barnsley, http://dbpedia.org/resource/Cawthorne, http://dbpedia.org/resource/Fukuoka, http://dbpedia.org/resource/Nova_Scotia, http://dbpedia.org/resource/Dartmouth,_Nova_Scotia, http://dbpedia.org/resource/Kazakh_Soviet_Socialist_Republic, http://dbpedia.org/resource/Dagenham, http://dbpedia.org/resource/Chūbu_region, http://dbpedia.org/resource/Rotterdam, http://dbpedia.org/resource/Erdington, http://dbpedia.org/resource/Essen, http://dbpedia.org/resource/Darien,_Connecticut, http://dbpedia.org/resource/Connecticut, http://dbpedia.org/resource/Sharon,_Massachusetts, http://dbpedia.org/resource/San_Jose,_California, http://dbpedia.org/resource/Kansas_City,_Missouri, http://dbpedia.org/resource/Estonia, http://dbpedia.org/resource/Tallinn, http://dbpedia.org/resource/Hokkaido, http://dbpedia.org/resource/Sapporo, http://dbpedia.org/resource/Rockville,_Maryland, http://dbpedia.org/resource/East_Brunswick,_New_Jersey, http://dbpedia.org/resource/Bothell,_Washington, http://dbpedia.org/resource/Akron,_Ohio, http://dbpedia.org/resource/Uppsala, http://dbpedia.org/resource/Italian_Americans, http://dbpedia.org/resource/Faroe_Islands, http://dbpedia.org/resource/Wildwood,_Florida, http://dbpedia.org/resource/Hanover_Park,_Illinois, http://dbpedia.org/resource/Offenburg, http://dbpedia.org/resource/Lippstadt, http://dbpedia.org/resource/Blackwood,_Caerphilly, http://dbpedia.org/resource/Dorset, http://dbpedia.org/resource/Little_Hanford, http://dbpedia.org/resource/Halifax,_West_Yorkshire, http://dbpedia.org/resource/Salta, http://dbpedia.org/resource/Bilbao, http://dbpedia.org/resource/Inglewood,_California, http://dbpedia.org/resource/Plasencia, http://dbpedia.org/resource/Sewell,_Chile, http://dbpedia.org/resource/Galway, http://dbpedia.org/resource/La_Rioja_(Spain), http://dbpedia.org/resource/Logroño, http://dbpedia.org/resource/Stuttgart, http://dbpedia.org/resource/Lincoln,_Nebraska, http://dbpedia.org/resource/Nebraska, http://dbpedia.org/resource/Clane, http://dbpedia.org/resource/Carlisle,_Cumbria, http://dbpedia.org/resource/Ise,_Mie, http://dbpedia.org/resource/Baltimore, http://dbpedia.org/resource/Healdsburg,_California, http://dbpedia.org/resource/Bandar-e_Anzali, http://dbpedia.org/resource/Gilan_Province, http://dbpedia.org/resource/Macon,_Georgia, http://dbpedia.org/resource/Madera,_California, http://dbpedia.org/resource/Paternò, http://dbpedia.org/resource/Bryne, http://dbpedia.org/resource/Beni_Yenni, http://dbpedia.org/resource/Chiba,_Chiba, http://dbpedia.org/resource/Oak_Park,_Illinois, http://dbpedia.org/resource/Kerala, http://dbpedia.org/resource/Kochi, http://dbpedia.org/resource/Chelyabinsk, http://dbpedia.org/resource/Västerås, http://dbpedia.org/resource/Pétionville, http://dbpedia.org/resource/Tel_Aviv, http://dbpedia.org/resource/Newark,_New_Jersey, http://dbpedia.org/resource/Nihonmatsu,_Fukushima, http://dbpedia.org/resource/Fukushima_Prefecture, http://dbpedia.org/resource/European_Union, http://dbpedia.org/resource/Zagreb, http://dbpedia.org/resource/Panama, http://dbpedia.org/resource/Cebu, http://dbpedia.org/resource/Bosnia_and_Herzegovina, http://dbpedia.org/resource/Socialist_Federal_Republic_of_Yugoslavia, http://dbpedia.org/resource/Bihać, http://dbpedia.org/resource/Tórshavn, http://dbpedia.org/resource/Bulgaria, http://dbpedia.org/resource/Morocco, http://dbpedia.org/resource/Oujda, http://dbpedia.org/resource/Makati, http://dbpedia.org/resource/Burnham-on-Crouch, http://dbpedia.org/resource/Columbia,_Maryland, http://dbpedia.org/resource/Nagano_Prefecture, http://dbpedia.org/resource/Chikuma,_Nagano, http://dbpedia.org/resource/Turin, http://dbpedia.org/resource/Yamatokōriyama, http://dbpedia.org/resource/Watford, http://dbpedia.org/resource/Lachine,_Quebec, http://dbpedia.org/resource/East_Los_Angeles,_California, http://dbpedia.org/resource/Charlestown,_Boston, http://dbpedia.org/resource/Mjölby, http://dbpedia.org/resource/Seminole_County,_Oklahoma, http://dbpedia.org/resource/Somerset, http://dbpedia.org/resource/Taunton, http://dbpedia.org/resource/Mumbai, http://dbpedia.org/resource/Edmonton, http://dbpedia.org/resource/Oulu, http://dbpedia.org/resource/Clark_Air_Base, http://dbpedia.org/resource/Islington, http://dbpedia.org/resource/Broussard,_Louisiana, http://dbpedia.org/resource/Singapore, http://dbpedia.org/resource/Quebec_City, http://dbpedia.org/resource/Northern_California, http://dbpedia.org/resource/Metairie,_Louisiana, http://dbpedia.org/resource/Brzeg, http://dbpedia.org/resource/Pleasantville,_New_York, http://dbpedia.org/resource/Spruce_Pine,_North_Carolina, http://dbpedia.org/resource/Hanover,_Ontario, http://dbpedia.org/resource/Victoria,_British_Columbia, http://dbpedia.org/resource/Leicestershire, http://dbpedia.org/resource/Sileby, http://dbpedia.org/resource/Manitoba, http://dbpedia.org/resource/Frankfurt, http://dbpedia.org/resource/Hamburg, http://dbpedia.org/resource/North_Vancouver_(district_municipality), http://dbpedia.org/resource/Norfolk, http://dbpedia.org/resource/Fort_Lauderdale,_Florida, http://dbpedia.org/resource/Elche, http://dbpedia.org/resource/Macarena,_Seville]... [INFO] Predicted Answer: [http://dbpedia.org/resource/Cologne, http://dbpedia.org/resource/North_Rhine-Westphalia, http://dbpedia.org/resource/West_Germany, http://dbpedia.org/resource/Chicago, http://dbpedia.org/resource/Columbia,_Missouri, http://dbpedia.org/resource/Kildare, http://dbpedia.org/resource/Mobile,_Alabama, http://dbpedia.org/resource/Wynnewood,_Pennsylvania, http://dbpedia.org/resource/Canada, http://dbpedia.org/resource/Montreal, http://dbpedia.org/resource/Quebec, http://dbpedia.org/resource/Godalming, http://dbpedia.org/resource/Brazil, http://dbpedia.org/resource/Rio_de_Janeiro, http://dbpedia.org/resource/Leesburg,_Georgia, http://dbpedia.org/resource/Cincinnati, http://dbpedia.org/resource/Ohio, http://dbpedia.org/resource/United_States, http://dbpedia.org/resource/Stockholm, http://dbpedia.org/resource/England, http://dbpedia.org/resource/Guildford, http://dbpedia.org/resource/Surrey, http://dbpedia.org/resource/Sydney, http://dbpedia.org/resource/Denmark, http://dbpedia.org/resource/Bornholm, http://dbpedia.org/resource/Canton,_North_Carolina, http://dbpedia.org/resource/Whitesville,_West_Virginia, http://dbpedia.org/resource/Norway, http://dbpedia.org/resource/California, http://dbpedia.org/resource/Los_Angeles, http://dbpedia.org/resource/Blue_Springs,_Missouri, http://dbpedia.org/resource/Medway, http://dbpedia.org/resource/Hoo_St_Werburgh, http://dbpedia.org/resource/Birmingham, http://dbpedia.org/resource/West_Midlands_(county), http://dbpedia.org/resource/Oakland,_California, http://dbpedia.org/resource/New_Rochelle,_New_York, http://dbpedia.org/resource/Brooklyn, http://dbpedia.org/resource/New_York_City, http://dbpedia.org/resource/Ontario, http://dbpedia.org/resource/Toronto, http://dbpedia.org/resource/Lubbock,_Texas, http://dbpedia.org/resource/Sunderland,_Tyne_and_Wear, http://dbpedia.org/resource/Memphis,_Tennessee, http://dbpedia.org/resource/Waveland,_Mississippi, http://dbpedia.org/resource/Greece, http://dbpedia.org/resource/Australia, http://dbpedia.org/resource/Gothenburg, http://dbpedia.org/resource/Houston, http://dbpedia.org/resource/Illinois, http://dbpedia.org/resource/Mainz, http://dbpedia.org/resource/Greenville,_South_Carolina, http://dbpedia.org/resource/Holstebro, http://dbpedia.org/resource/Cuba, http://dbpedia.org/resource/Havana, http://dbpedia.org/resource/Vancouver, http://dbpedia.org/resource/Bahia, http://dbpedia.org/resource/Juazeiro, http://dbpedia.org/resource/Mexico, http://dbpedia.org/resource/Easthampton,_Massachusetts, http://dbpedia.org/resource/Massachusetts, http://dbpedia.org/resource/Ithaca,_New_York, http://dbpedia.org/resource/San_Diego, http://dbpedia.org/resource/Belgrade, http://dbpedia.org/resource/Serbia, http://dbpedia.org/resource/Tennessee, http://dbpedia.org/resource/Boom,_Belgium, http://dbpedia.org/resource/Antwerp, http://dbpedia.org/resource/Belgium, http://dbpedia.org/resource/Flanders, http://dbpedia.org/resource/Seattle, http://dbpedia.org/resource/Dutch_East_Indies, http://dbpedia.org/resource/Batavia,_Dutch_East_Indies, http://dbpedia.org/resource/Java, http://dbpedia.org/resource/Marietta,_Georgia, http://dbpedia.org/resource/Boston, http://dbpedia.org/resource/New_York, http://dbpedia.org/resource/Maynardville,_Tennessee, http://dbpedia.org/resource/New_Jersey, http://dbpedia.org/resource/Indiana, http://dbpedia.org/resource/Indianapolis, http://dbpedia.org/resource/Dallas, http://dbpedia.org/resource/Berkeley,_California, http://dbpedia.org/resource/Morton_Grove,_Illinois, http://dbpedia.org/resource/Lincolnshire, http://dbpedia.org/resource/Scunthorpe, http://dbpedia.org/resource/Comanche,_Oklahoma, http://dbpedia.org/resource/London, http://dbpedia.org/resource/Holland,_Michigan, http://dbpedia.org/resource/Texas, http://dbpedia.org/resource/Denton,_Texas, http://dbpedia.org/resource/Sainte-Foy,_Quebec_City, http://dbpedia.org/resource/Waldorf,_Maryland, http://dbpedia.org/resource/Washington,_D.C., http://dbpedia.org/resource/Melbourne, http://dbpedia.org/resource/Bristol, http://dbpedia.org/resource/Italy, http://dbpedia.org/resource/Terrazzo,_Veneto, http://dbpedia.org/resource/Buffalo,_New_York, http://dbpedia.org/resource/Adelaide, http://dbpedia.org/resource/San_Francisco, http://dbpedia.org/resource/Oklahoma, http://dbpedia.org/resource/Oklahoma_City, http://dbpedia.org/resource/Oxfordshire, http://dbpedia.org/resource/Abingdon-on-Thames, http://dbpedia.org/resource/Pennsylvania, http://dbpedia.org/resource/Philadelphia, http://dbpedia.org/resource/Garden_Grove,_California, http://dbpedia.org/resource/Riverdale,_Bronx, http://dbpedia.org/resource/Redlands,_California, http://dbpedia.org/resource/Wheeling,_West_Virginia, http://dbpedia.org/resource/Abilene,_Texas, http://dbpedia.org/resource/Netherlands, http://dbpedia.org/resource/The_Hague, http://dbpedia.org/resource/Istanbul, http://dbpedia.org/resource/Turkey, http://dbpedia.org/resource/Germany, http://dbpedia.org/resource/Bad_Homburg_vor_der_Höhe, http://dbpedia.org/resource/Waco,_Texas, http://dbpedia.org/resource/Princeton,_New_Jersey, http://dbpedia.org/resource/Rochester,_New_York, http://dbpedia.org/resource/Sweden, http://dbpedia.org/resource/New_Zealand, http://dbpedia.org/resource/Wellington, http://dbpedia.org/resource/Idaho, http://dbpedia.org/resource/Boise,_Idaho, http://dbpedia.org/resource/Goirle, http://dbpedia.org/resource/Crowley,_Louisiana, http://dbpedia.org/resource/Austria, http://dbpedia.org/resource/Vienna, http://dbpedia.org/resource/Dalserf, http://dbpedia.org/resource/Scotland, http://dbpedia.org/resource/South_Lanarkshire, http://dbpedia.org/resource/Gold_Coast,_Queensland, http://dbpedia.org/resource/Chile, http://dbpedia.org/resource/Catemu, http://dbpedia.org/resource/Skive,_Denmark, http://dbpedia.org/resource/Copenhagen, http://dbpedia.org/resource/Cape_Verde, http://dbpedia.org/resource/Mindelo, http://dbpedia.org/resource/São_Vicente,_Cape_Verde, http://dbpedia.org/resource/United_Kingdom, http://dbpedia.org/resource/Lancashire, http://dbpedia.org/resource/Darwen, http://dbpedia.org/resource/Athens,_Georgia, http://dbpedia.org/resource/Madrid, http://dbpedia.org/resource/Spain, http://dbpedia.org/resource/Greenwich_Village, http://dbpedia.org/resource/Carrboro,_North_Carolina, http://dbpedia.org/resource/Finland, http://dbpedia.org/resource/Helsinki, http://dbpedia.org/resource/Keiser,_Arkansas, http://dbpedia.org/resource/Concord,_North_Carolina, http://dbpedia.org/resource/Freeport,_New_York, http://dbpedia.org/resource/Harman,_West_Virginia, http://dbpedia.org/resource/Puerto_Rico, http://dbpedia.org/resource/Vega_Alta,_Puerto_Rico, http://dbpedia.org/resource/Tervo, http://dbpedia.org/resource/Auckland, http://dbpedia.org/resource/Gadsden,_Alabama, http://dbpedia.org/resource/Florida, http://dbpedia.org/resource/Sacramento,_California, http://dbpedia.org/resource/Basque_Country_(autonomous_community), http://dbpedia.org/resource/Florence,_Alabama, http://dbpedia.org/resource/Center_Point,_Alabama, http://dbpedia.org/resource/Loachapoka,_Alabama, http://dbpedia.org/resource/Etne, http://dbpedia.org/resource/Columbus,_Ohio, http://dbpedia.org/resource/Tottenham, http://dbpedia.org/resource/Menlo_Park,_California, http://dbpedia.org/resource/Killen,_Alabama, http://dbpedia.org/resource/Birmingham,_Alabama, http://dbpedia.org/resource/South_Windsor,_Connecticut, http://dbpedia.org/resource/Panama_City,_Florida, http://dbpedia.org/resource/Pooler,_Georgia, http://dbpedia.org/resource/Morrison,_Colorado, http://dbpedia.org/resource/Marysville,_California, http://dbpedia.org/resource/Albany,_Georgia, http://dbpedia.org/resource/Honolulu, http://dbpedia.org/resource/Kansas, http://dbpedia.org/resource/Kinsley,_Kansas, http://dbpedia.org/resource/Iowa_City,_Iowa, http://dbpedia.org/resource/Nashville,_Tennessee, http://dbpedia.org/resource/Graceville,_Minnesota, http://dbpedia.org/resource/Royal_Oak,_Michigan, http://dbpedia.org/resource/Hernando,_Mississippi, http://dbpedia.org/resource/Corinth,_Mississippi, http://dbpedia.org/resource/Mississippi, http://dbpedia.org/resource/Amersfoort, http://dbpedia.org/resource/Muskegon,_Michigan, http://dbpedia.org/resource/Hopewell,_New_Jersey, http://dbpedia.org/resource/Boonton,_New_Jersey, http://dbpedia.org/resource/Jamestown,_New_York, http://dbpedia.org/resource/Long_Island, http://dbpedia.org/resource/Gillette,_New_Jersey, http://dbpedia.org/resource/Carteret,_New_Jersey, http://dbpedia.org/resource/Fairport,_New_York, http://dbpedia.org/resource/New_Brunswick,_New_Jersey, http://dbpedia.org/resource/Waverly,_Tennessee, http://dbpedia.org/resource/Sabina,_Ohio, http://dbpedia.org/resource/Canby,_Oregon, http://dbpedia.org/resource/Kettering,_Ohio, http://dbpedia.org/resource/Glen_Dale,_West_Virginia, http://dbpedia.org/resource/Bethel_Acres,_Oklahoma, http://dbpedia.org/resource/Bandera,_Texas, http://dbpedia.org/resource/Pampa,_Texas, http://dbpedia.org/resource/Utah, http://dbpedia.org/resource/Salt_Lake_City, http://dbpedia.org/resource/Austin,_Texas, http://dbpedia.org/resource/Pulaski,_Tennessee, http://dbpedia.org/resource/France, http://dbpedia.org/resource/Toulouse, http://dbpedia.org/resource/Fort_Irwin_National_Training_Center, http://dbpedia.org/resource/El_Paso,_Texas, http://dbpedia.org/resource/Godley,_Texas, http://dbpedia.org/resource/Yonkers,_New_York, http://dbpedia.org/resource/Puryear,_Tennessee, http://dbpedia.org/resource/Gainesville,_Texas, http://dbpedia.org/resource/Sumter,_South_Carolina, http://dbpedia.org/resource/Wise_County,_Texas, http://dbpedia.org/resource/Bath,_Somerset, http://dbpedia.org/resource/Bible_Hill,_Nova_Scotia, http://dbpedia.org/resource/Oslo, http://dbpedia.org/resource/Sheffield, http://dbpedia.org/resource/Hillsborough,_Sheffield, http://dbpedia.org/resource/San_Antonio, http://dbpedia.org/resource/Olathe,_Kansas, http://dbpedia.org/resource/Providence,_Rhode_Island, http://dbpedia.org/resource/Stavanger, http://dbpedia.org/resource/Oshawa, http://dbpedia.org/resource/Minneapolis, http://dbpedia.org/resource/Minnesota, http://dbpedia.org/resource/Wiener_Neustadt, http://dbpedia.org/resource/Kentucky, http://dbpedia.org/resource/Louisville,_Kentucky, http://dbpedia.org/resource/Newcastle_upon_Tyne, http://dbpedia.org/resource/British_Columbia, http://dbpedia.org/resource/Martin_County,_Kentucky, http://dbpedia.org/resource/Arkansas, http://dbpedia.org/resource/Fayetteville,_Arkansas, http://dbpedia.org/resource/Rothesay,_Bute, http://dbpedia.org/resource/Okemah,_Oklahoma, http://dbpedia.org/resource/Alberta, http://dbpedia.org/resource/Calgary, http://dbpedia.org/resource/Copeville,_Texas, http://dbpedia.org/resource/Archerwill,_Saskatchewan, http://dbpedia.org/resource/Fremont,_Nebraska, http://dbpedia.org/resource/Karnes_City,_Texas, http://dbpedia.org/resource/Phoenix,_Arizona, http://dbpedia.org/resource/Daytona_Beach,_Florida, http://dbpedia.org/resource/South_Australia, http://dbpedia.org/resource/Georgetown,_Texas, http://dbpedia.org/resource/Sherman,_Texas, http://dbpedia.org/resource/Columbia,_South_Carolina, http://dbpedia.org/resource/Uxbridge,_Ontario, http://dbpedia.org/resource/Success,_Missouri, http://dbpedia.org/resource/Asheville,_North_Carolina, http://dbpedia.org/resource/Fort_Wayne,_Indiana, http://dbpedia.org/resource/Wisconsin, http://dbpedia.org/resource/Hayward,_Wisconsin, http://dbpedia.org/resource/Devon, http://dbpedia.org/resource/Sparta,_Illinois, http://dbpedia.org/resource/Woodside,_California, http://dbpedia.org/resource/Minas_Gerais, http://dbpedia.org/resource/Sete_Lagoas, http://dbpedia.org/resource/Baytown,_Texas, http://dbpedia.org/resource/Seneca,_Missouri, http://dbpedia.org/resource/Maple_Ridge,_British_Columbia, http://dbpedia.org/resource/Caddo,_Oklahoma, http://dbpedia.org/resource/Breckenridge,_Texas, http://dbpedia.org/resource/Hiroshima_Prefecture, http://dbpedia.org/resource/Japan, http://dbpedia.org/resource/Takehara,_Hiroshima, http://dbpedia.org/resource/Lake_Charles,_Louisiana, http://dbpedia.org/resource/Roblin,_Manitoba, http://dbpedia.org/resource/Binghamton,_New_York, http://dbpedia.org/resource/Midwest_City,_Oklahoma, http://dbpedia.org/resource/Duncan,_Arizona, http://dbpedia.org/resource/Karachi, http://dbpedia.org/resource/Pakistan, http://dbpedia.org/resource/Jordan, http://dbpedia.org/resource/State_of_Palestine, http://dbpedia.org/resource/Las_Vegas_Valley, http://dbpedia.org/resource/Nevada, http://dbpedia.org/resource/Little_Rock,_Arkansas, http://dbpedia.org/resource/Valletta, http://dbpedia.org/resource/Palma,_Majorca, http://dbpedia.org/resource/Balearic_Islands, http://dbpedia.org/resource/Santa_Barbara,_California, http://dbpedia.org/resource/Porto_Alegre, http://dbpedia.org/resource/Portland,_Oregon, http://dbpedia.org/resource/Oregon, http://dbpedia.org/resource/South_Africa, http://dbpedia.org/resource/KwaZulu-Natal, http://dbpedia.org/resource/Midlands_of_KwaZulu-Natal, http://dbpedia.org/resource/Bennington,_Vermont, http://dbpedia.org/resource/Orlando,_Florida, http://dbpedia.org/resource/Fries,_Virginia, http://dbpedia.org/resource/Wantagh,_New_York, http://dbpedia.org/resource/The_Bellmores,_New_York, http://dbpedia.org/resource/Queens, http://dbpedia.org/resource/Tulsa,_Oklahoma, http://dbpedia.org/resource/Hawaii, http://dbpedia.org/resource/Ames,_Iowa, http://dbpedia.org/resource/Bronaugh,_Missouri, http://dbpedia.org/resource/New_South_Wales, http://dbpedia.org/resource/London_Borough_of_Hackney, http://dbpedia.org/resource/Durban, http://dbpedia.org/resource/Tuusula, http://dbpedia.org/resource/Nepal, http://dbpedia.org/resource/Pittsburgh, http://dbpedia.org/resource/Glendale,_California, http://dbpedia.org/resource/Ivory_Coast, http://dbpedia.org/resource/Jamaica, http://dbpedia.org/resource/North_Carolina, http://dbpedia.org/resource/Research_Triangle, http://dbpedia.org/resource/Montevideo, http://dbpedia.org/resource/Uruguay, http://dbpedia.org/resource/Niagara_Falls,_New_York, http://dbpedia.org/resource/Switzerland, http://dbpedia.org/resource/Zürich, http://dbpedia.org/resource/Fort_Worth,_Texas, http://dbpedia.org/resource/Fresno,_California, http://dbpedia.org/resource/Vallejo,_California, http://dbpedia.org/resource/Essex, http://dbpedia.org/resource/St._Petersburg,_Florida, http://dbpedia.org/resource/New_Mexico, http://dbpedia.org/resource/South_Korea, http://dbpedia.org/resource/Brownsville,_Texas, http://dbpedia.org/resource/Kingston,_Ontario, http://dbpedia.org/resource/Castaic,_California, http://dbpedia.org/resource/Duluth,_Minnesota, http://dbpedia.org/resource/Republic_of_Ireland, http://dbpedia.org/resource/County_Wicklow, http://dbpedia.org/resource/Lisbon, http://dbpedia.org/resource/Sleepy_Hollow,_New_York, http://dbpedia.org/resource/Tampa,_Florida, http://dbpedia.org/resource/Kirkcaldy, http://dbpedia.org/resource/Ridgewood,_New_Jersey, http://dbpedia.org/resource/Silver_Lake,_Los_Angeles, http://dbpedia.org/resource/Fraser_Canyon, http://dbpedia.org/resource/London,_Ontario, http://dbpedia.org/resource/Osaka, http://dbpedia.org/resource/Kasai,_Hyōgo, http://dbpedia.org/resource/Gilroy,_California, http://dbpedia.org/resource/Somers_Point,_New_Jersey, http://dbpedia.org/resource/Bad_Windsheim, http://dbpedia.org/resource/Barnesboro,_Pennsylvania, http://dbpedia.org/resource/Carleton_Place, http://dbpedia.org/resource/Fremont,_California, http://dbpedia.org/resource/Cheshire, http://dbpedia.org/resource/Philippines, http://dbpedia.org/resource/Cebu_City, http://dbpedia.org/resource/Union_City,_Tennessee, http://dbpedia.org/resource/Brisbane, http://dbpedia.org/resource/Queensland, http://dbpedia.org/resource/Lebanon, http://dbpedia.org/resource/Tripoli,_Lebanon, http://dbpedia.org/resource/Washington_(state), http://dbpedia.org/resource/Sebastopol,_Texas, http://dbpedia.org/resource/Hyattsville,_Maryland, http://dbpedia.org/resource/Woodbridge,_Virginia, http://dbpedia.org/resource/Brixton, http://dbpedia.org/resource/Bremerton,_Washington, http://dbpedia.org/resource/Kosovo, http://dbpedia.org/resource/Wollongong, http://dbpedia.org/resource/Cheongju, http://dbpedia.org/resource/North_Chungcheong_Province, http://dbpedia.org/resource/Vietnam, http://dbpedia.org/resource/Córdoba,_Andalusia, http://dbpedia.org/resource/Novi_Sad, http://dbpedia.org/resource/Atlanta, http://dbpedia.org/resource/Georgia_(U.S._state), http://dbpedia.org/resource/Russian_Soviet_Federative_Socialist_Republic, http://dbpedia.org/resource/Yekaterinburg, http://dbpedia.org/resource/Bodø, http://dbpedia.org/resource/Milan, http://dbpedia.org/resource/Shetland, http://dbpedia.org/resource/Grand_Rapids,_Michigan, http://dbpedia.org/resource/Spokane,_Washington, http://dbpedia.org/resource/Franklin_Lakes,_New_Jersey, http://dbpedia.org/resource/Grycksbo, http://dbpedia.org/resource/Falun, http://dbpedia.org/resource/Virginia, http://dbpedia.org/resource/Catalonia, http://dbpedia.org/resource/Sabadell, http://dbpedia.org/resource/Tokyo, http://dbpedia.org/resource/Ventura,_California, http://dbpedia.org/resource/Maryland, http://dbpedia.org/resource/Linz, http://dbpedia.org/resource/Europe, http://dbpedia.org/resource/Iran, http://dbpedia.org/resource/Tehran, http://dbpedia.org/resource/Tasmania, http://dbpedia.org/resource/Franklin,_Tasmania, http://dbpedia.org/resource/Harlem, http://dbpedia.org/resource/Winston-Salem,_North_Carolina, http://dbpedia.org/resource/Nacogdoches,_Texas, http://dbpedia.org/resource/Finglas, http://dbpedia.org/resource/County_Dublin, http://dbpedia.org/resource/Louisiana, http://dbpedia.org/resource/New_Orleans, http://dbpedia.org/resource/Iceland, http://dbpedia.org/resource/Amsterdam, http://dbpedia.org/resource/Homer,_Alaska, http://dbpedia.org/resource/Waterloo,_Ontario, http://dbpedia.org/resource/Forest_Lake,_Minnesota, http://dbpedia.org/resource/Southlake,_Texas, http://dbpedia.org/resource/Alaska, http://dbpedia.org/resource/Anchorage,_Alaska, http://dbpedia.org/resource/San_Juan,_Puerto_Rico, http://dbpedia.org/resource/Halstead, http://dbpedia.org/resource/Bay_Village,_Ohio, http://dbpedia.org/resource/Chicopee,_Massachusetts, http://dbpedia.org/resource/Hemel_Hempstead, http://dbpedia.org/resource/Hertfordshire, http://dbpedia.org/resource/Aarhus, http://dbpedia.org/resource/Germans, http://dbpedia.org/resource/Huế, http://dbpedia.org/resource/Dayton,_Ohio, http://dbpedia.org/resource/Argentina, http://dbpedia.org/resource/Buenos_Aires, http://dbpedia.org/resource/Richmond,_Virginia, http://dbpedia.org/resource/South_Lyon,_Michigan, http://dbpedia.org/resource/Moscow, http://dbpedia.org/resource/Soviet_Union, http://dbpedia.org/resource/Rodeo,_California, http://dbpedia.org/resource/Lexington,_Kentucky, http://dbpedia.org/resource/Portsmouth,_Virginia, http://dbpedia.org/resource/Chesapeake,_Virginia, http://dbpedia.org/resource/Castle_Hill,_New_South_Wales, http://dbpedia.org/resource/Carpentras, http://dbpedia.org/resource/Keithville,_Louisiana, http://dbpedia.org/resource/Caddo_Parish,_Louisiana, http://dbpedia.org/resource/Hawthorne,_California, http://dbpedia.org/resource/Beech_Mountain,_North_Carolina, http://dbpedia.org/resource/Beijing, http://dbpedia.org/resource/China, http://dbpedia.org/resource/San_Fernando_Valley, http://dbpedia.org/resource/Rock_Island,_Illinois, http://dbpedia.org/resource/Mayfield,_Kentucky, http://dbpedia.org/resource/Walla_Walla,_Washington, http://dbpedia.org/resource/Nagasaki, http://dbpedia.org/resource/Detroit, http://dbpedia.org/resource/Jacksonville,_Florida, http://dbpedia.org/resource/Wimbledon,_London, http://dbpedia.org/resource/Ruston,_Louisiana, http://dbpedia.org/resource/Daisy,_Kentucky, http://dbpedia.org/resource/Russia, http://dbpedia.org/resource/Trenton,_New_Jersey, http://dbpedia.org/resource/Brest,_France, http://dbpedia.org/resource/Brittany, http://dbpedia.org/resource/Angeles,_Philippines, http://dbpedia.org/resource/Pampanga, http://dbpedia.org/resource/Perth, http://dbpedia.org/resource/Ann_Arbor,_Michigan, http://dbpedia.org/resource/Paris,_Illinois, http://dbpedia.org/resource/Marlton,_New_Jersey, http://dbpedia.org/resource/Caldwell_County,_North_Carolina, http://dbpedia.org/resource/Darby,_Pennsylvania, http://dbpedia.org/resource/Hamilton,_New_Zealand, http://dbpedia.org/resource/Dublin, http://dbpedia.org/resource/Glasgow, http://dbpedia.org/resource/Albury, http://dbpedia.org/resource/Poggio_Bustone, http://dbpedia.org/resource/Burbank,_California, http://dbpedia.org/resource/Hiroshima, http://dbpedia.org/resource/Leeds, http://dbpedia.org/resource/Oakville,_Ontario, http://dbpedia.org/resource/Newton,_Mississippi, http://dbpedia.org/resource/New_Brunswick, http://dbpedia.org/resource/Edmundston, http://dbpedia.org/resource/Oxford, http://dbpedia.org/resource/Elverum, http://dbpedia.org/resource/Hedmark, http://dbpedia.org/resource/Hollywood, http://dbpedia.org/resource/South_Orange,_New_Jersey, http://dbpedia.org/resource/Ottawa, http://dbpedia.org/resource/Berlin, http://dbpedia.org/resource/Chemnitz, http://dbpedia.org/resource/Saxony, http://dbpedia.org/resource/Kansas_City,_Kansas, http://dbpedia.org/resource/St_Kilda,_Victoria, http://dbpedia.org/resource/Stellenbosch, http://dbpedia.org/resource/Shreveport,_Louisiana, http://dbpedia.org/resource/Chapel_Hill,_North_Carolina, http://dbpedia.org/resource/West_Chester,_Pennsylvania, http://dbpedia.org/resource/Blenheim,_New_Zealand, http://dbpedia.org/resource/Belfast, http://dbpedia.org/resource/Northern_Ireland, http://dbpedia.org/resource/Kirghiz_Soviet_Socialist_Republic, http://dbpedia.org/resource/Poland, http://dbpedia.org/resource/Enville,_Oklahoma, http://dbpedia.org/resource/Aabyhøj, http://dbpedia.org/resource/South_Carolina, http://dbpedia.org/resource/Laurens,_South_Carolina, http://dbpedia.org/resource/Canterbury, http://dbpedia.org/resource/Pamplona, http://dbpedia.org/resource/Chechen-Ingush_Autonomous_Soviet_Socialist_Republic, http://dbpedia.org/resource/Grozny, http://dbpedia.org/resource/Humphreys_County,_Mississippi, http://dbpedia.org/resource/Winthrop,_Massachusetts, http://dbpedia.org/resource/Wuppertal, http://dbpedia.org/resource/Bowie,_Maryland, http://dbpedia.org/resource/Country_music, http://dbpedia.org/resource/Karlshamn, http://dbpedia.org/resource/Tomnavoulin, http://dbpedia.org/resource/Banffshire, http://dbpedia.org/resource/Geraardsbergen, http://dbpedia.org/resource/Kingston,_Jamaica, http://dbpedia.org/resource/Israel, http://dbpedia.org/resource/Jerusalem, http://dbpedia.org/resource/Gloucestershire, http://dbpedia.org/resource/Cheltenham, http://dbpedia.org/resource/Rugby,_Warwickshire, http://dbpedia.org/resource/Beersheba, http://dbpedia.org/resource/Farmerville,_Louisiana, http://dbpedia.org/resource/Cambridge,_Massachusetts, http://dbpedia.org/resource/Somerville,_Massachusetts, http://dbpedia.org/resource/Bastrop,_Louisiana, http://dbpedia.org/resource/New_Bedford,_Massachusetts, http://dbpedia.org/resource/Maine, http://dbpedia.org/resource/Portland,_Maine, http://dbpedia.org/resource/Knoxville,_Tennessee, http://dbpedia.org/resource/Lodi,_New_Jersey, http://dbpedia.org/resource/Booneville,_Mississippi, http://dbpedia.org/resource/Dumont,_New_Jersey, http://dbpedia.org/resource/San_Francisco_Bay_Area, http://dbpedia.org/resource/Derbyshire, http://dbpedia.org/resource/Buxton, http://dbpedia.org/resource/Senatobia,_Mississippi, http://dbpedia.org/resource/St._Ignatius,_Montana, http://dbpedia.org/resource/Ramsey,_New_Jersey, http://dbpedia.org/resource/Sussex_County,_New_Jersey, http://dbpedia.org/resource/Hastings-on-Hudson,_New_York, http://dbpedia.org/resource/Dix_Hills,_New_York, http://dbpedia.org/resource/Fayetteville,_North_Carolina, http://dbpedia.org/resource/Venice,_Los_Angeles, http://dbpedia.org/resource/Derby, http://dbpedia.org/resource/Maplewood,_New_Jersey, http://dbpedia.org/resource/Cohoes,_New_York, http://dbpedia.org/resource/Cairo, http://dbpedia.org/resource/Egypt, http://dbpedia.org/resource/South_London, http://dbpedia.org/resource/West_Mifflin,_Pennsylvania, http://dbpedia.org/resource/Cleveland, http://dbpedia.org/resource/São_Paulo, http://dbpedia.org/resource/Voorheesville,_New_York, http://dbpedia.org/resource/Aguascalientes, http://dbpedia.org/resource/Newmarket,_Ontario, http://dbpedia.org/resource/Mexico_City, http://dbpedia.org/resource/Portugal, http://dbpedia.org/resource/Loures, http://dbpedia.org/resource/Cartago_Valle, http://dbpedia.org/resource/Colombia, http://dbpedia.org/resource/São_Caetano_do_Sul, http://dbpedia.org/resource/Hazard,_Kentucky, http://dbpedia.org/resource/Wicklow, http://dbpedia.org/resource/Galway, http://dbpedia.org/resource/Windsor,_Ontario, http://dbpedia.org/resource/Pacific_Palisades,_Los_Angeles, http://dbpedia.org/resource/Grafton,_New_South_Wales, http://dbpedia.org/resource/Seoul, http://dbpedia.org/resource/Northeastern_United_States, http://dbpedia.org/resource/Redondo_Beach,_California, http://dbpedia.org/resource/Moldova, http://dbpedia.org/resource/Bălți, http://dbpedia.org/resource/Nakano,_Tokyo, http://dbpedia.org/resource/Worthing, http://dbpedia.org/resource/West_Pubnico,_Nova_Scotia, http://dbpedia.org/resource/Brighton, http://dbpedia.org/resource/Barbados, http://dbpedia.org/resource/Hare_Bay,_Newfoundland_and_Labrador, http://dbpedia.org/resource/Guelph, http://dbpedia.org/resource/Richardson,_Texas, http://dbpedia.org/resource/Jackson,_Mississippi, http://dbpedia.org/resource/Reseda,_Los_Angeles, http://dbpedia.org/resource/Valparaíso, http://dbpedia.org/resource/Yorkshire, http://dbpedia.org/resource/Kingston_upon_Hull, http://dbpedia.org/resource/The_Bronx, http://dbpedia.org/resource/Xinjiang, http://dbpedia.org/resource/Norfolk,_Virginia, http://dbpedia.org/resource/Alicante, http://dbpedia.org/resource/West_Yorkshire, http://dbpedia.org/resource/South_Haven,_Michigan, http://dbpedia.org/resource/West_Midlands_(region), http://dbpedia.org/resource/Wolverhampton, http://dbpedia.org/resource/Nevada,_Missouri, http://dbpedia.org/resource/Southern_California, http://dbpedia.org/resource/Terneuzen, http://dbpedia.org/resource/Taylorsville,_Kentucky, http://dbpedia.org/resource/St._Louis, http://dbpedia.org/resource/Dharan,_Nepal, http://dbpedia.org/resource/Suffolk, http://dbpedia.org/resource/Bildeston, http://dbpedia.org/resource/Tampere, http://dbpedia.org/resource/Metropolitan_Borough_of_Oldham, http://dbpedia.org/resource/Greater_Manchester, http://dbpedia.org/resource/São_Paulo_(state), http://dbpedia.org/resource/Ribeirão_Preto, http://dbpedia.org/resource/Newfoundland_and_Labrador, http://dbpedia.org/resource/St._John's,_Newfoundland_and_Labrador, http://dbpedia.org/resource/Peru, http://dbpedia.org/resource/Callao, http://dbpedia.org/resource/Sri_Lanka, http://dbpedia.org/resource/Greater_Sudbury, http://dbpedia.org/resource/Mount_Dennis, http://dbpedia.org/resource/Kagoshima, http://dbpedia.org/resource/Kagoshima_Prefecture, http://dbpedia.org/resource/Terre_Haute,_Indiana, http://dbpedia.org/resource/County_Cavan, http://dbpedia.org/resource/Shercock, http://dbpedia.org/resource/Whittlesey, http://dbpedia.org/resource/Tajikistan, http://dbpedia.org/resource/Khorugh, http://dbpedia.org/resource/Airdrie,_Alberta, http://dbpedia.org/resource/Tacoma,_Washington, http://dbpedia.org/resource/Colorado, http://dbpedia.org/resource/Morrinsville, http://dbpedia.org/resource/Montana, http://dbpedia.org/resource/Hong_Kong, http://dbpedia.org/resource/Venezuela, http://dbpedia.org/resource/Maracaibo, http://dbpedia.org/resource/Zulia, http://dbpedia.org/resource/Marseille, http://dbpedia.org/resource/Wall_Township,_New_Jersey, http://dbpedia.org/resource/Scarborough,_Toronto, http://dbpedia.org/resource/Kingston_upon_Thames, http://dbpedia.org/resource/Delhi, http://dbpedia.org/resource/India, http://dbpedia.org/resource/Haryana, http://dbpedia.org/resource/Bhiwani, http://dbpedia.org/resource/County_Kildare, http://dbpedia.org/resource/Celbridge, http://dbpedia.org/resource/Westchester_County,_New_York, http://dbpedia.org/resource/Malaysia, http://dbpedia.org/resource/Bukit_Jalil, http://dbpedia.org/resource/Carrara, http://dbpedia.org/resource/Tuscany, http://dbpedia.org/resource/Sullivan_County,_Pennsylvania, http://dbpedia.org/resource/Newport_Beach,_California, http://dbpedia.org/resource/Caloocan, http://dbpedia.org/resource/South_Holland, http://dbpedia.org/resource/Staten_Island, http://dbpedia.org/resource/Algeria, http://dbpedia.org/resource/Algiers, http://dbpedia.org/resource/Mohamed_Belouizdad, http://dbpedia.org/resource/Denver, http://dbpedia.org/resource/Erfurt, http://dbpedia.org/resource/Saint_Paul,_Minnesota, http://dbpedia.org/resource/Vantaa, http://dbpedia.org/resource/Beşiktaş, http://dbpedia.org/resource/Levent, http://dbpedia.org/resource/Valley_Stream,_New_York, http://dbpedia.org/resource/Campbell,_California, http://dbpedia.org/resource/Jacksonville,_Texas, http://dbpedia.org/resource/Canton,_Ohio, http://dbpedia.org/resource/Tamaulipas, http://dbpedia.org/resource/Ciudad_Madero, http://dbpedia.org/resource/Korumburra, http://dbpedia.org/resource/New_Haven,_Connecticut, http://dbpedia.org/resource/Irvine,_North_Ayrshire, http://dbpedia.org/resource/Croatia, http://dbpedia.org/resource/Medicine_Hat, http://dbpedia.org/resource/Woodland,_Alabama, http://dbpedia.org/resource/Paris, http://dbpedia.org/resource/Fort_Scott,_Kansas, http://dbpedia.org/resource/Ankara, http://dbpedia.org/resource/Hamilton,_Ohio, http://dbpedia.org/resource/Kitee, http://dbpedia.org/resource/Paterson,_New_Jersey, http://dbpedia.org/resource/Surbiton, http://dbpedia.org/resource/Corpus_Christi,_Texas, http://dbpedia.org/resource/Pottstown,_Pennsylvania, http://dbpedia.org/resource/Fargo,_North_Dakota, http://dbpedia.org/resource/Cassine, http://dbpedia.org/resource/Stockton,_California, http://dbpedia.org/resource/Swillington, http://dbpedia.org/resource/Jujuy_Province, http://dbpedia.org/resource/Kyoto, http://dbpedia.org/resource/Mulberry,_Indiana, http://dbpedia.org/resource/Moreno_Valley,_California, http://dbpedia.org/resource/Albany,_New_York, http://dbpedia.org/resource/Bolton, http://dbpedia.org/resource/Ponca_City,_Oklahoma, http://dbpedia.org/resource/Santa_Monica,_California, http://dbpedia.org/resource/Clarion,_Pennsylvania, http://dbpedia.org/resource/Oromia_Region, http://dbpedia.org/resource/Dembidolo, http://dbpedia.org/resource/Utica,_New_York, http://dbpedia.org/resource/Wakefield, http://dbpedia.org/resource/Worsley, http://dbpedia.org/resource/Holtville,_Alabama, http://dbpedia.org/resource/Rayne,_Louisiana, http://dbpedia.org/resource/Madison,_Wisconsin, http://dbpedia.org/resource/Batangas, http://dbpedia.org/resource/Rosario,_Batangas, http://dbpedia.org/resource/Três_Pontas, http://dbpedia.org/resource/Kiryat_Gat, http://dbpedia.org/resource/Moose_Lake,_Minnesota, http://dbpedia.org/resource/Manila, http://dbpedia.org/resource/Shepherd's_Bush, http://dbpedia.org/resource/Dushanbe, http://dbpedia.org/resource/Andover,_Massachusetts, http://dbpedia.org/resource/Tver, http://dbpedia.org/resource/Michigan, http://dbpedia.org/resource/Gifu_Prefecture, http://dbpedia.org/resource/Quantico,_Virginia, http://dbpedia.org/resource/New_Cross, http://dbpedia.org/resource/Morristown,_New_Jersey, http://dbpedia.org/resource/Neusiedl_am_See, http://dbpedia.org/resource/Santurce,_San_Juan,_Puerto_Rico, http://dbpedia.org/resource/Ardnacrusha, http://dbpedia.org/resource/Allegheny_County,_Pennsylvania, http://dbpedia.org/resource/Reykjavík, http://dbpedia.org/resource/Diamond_Bar,_California, http://dbpedia.org/resource/El_Dorado,_Arkansas, http://dbpedia.org/resource/Anaheim,_California, http://dbpedia.org/resource/Half_Moon_Bay,_California, http://dbpedia.org/resource/East_Bay_(San_Francisco_Bay_Area), http://dbpedia.org/resource/Northwest_Arkansas, http://dbpedia.org/resource/Ukiah,_California, http://dbpedia.org/resource/La_Habra,_California, http://dbpedia.org/resource/Rogers,_Arkansas, http://dbpedia.org/resource/Arizona, http://dbpedia.org/resource/Ørum_(Sønderlyng), http://dbpedia.org/resource/Yvelines, http://dbpedia.org/resource/Saint-Germain-en-Laye, http://dbpedia.org/resource/Île-de-France, http://dbpedia.org/resource/Widnes, http://dbpedia.org/resource/Nigeria, http://dbpedia.org/resource/Calabar, http://dbpedia.org/resource/Ghana, http://dbpedia.org/resource/Accra, http://dbpedia.org/resource/Harrisonburg,_Virginia, http://dbpedia.org/resource/Omaha,_Nebraska, http://dbpedia.org/resource/Nikšić, http://dbpedia.org/resource/Montenegro, http://dbpedia.org/resource/Missouri, http://dbpedia.org/resource/Western_Australia, http://dbpedia.org/resource/Albany,_Western_Australia, http://dbpedia.org/resource/Shepparton, http://dbpedia.org/resource/Amarillo,_Texas, http://dbpedia.org/resource/Eugene,_Oregon, http://dbpedia.org/resource/Acton,_Massachusetts, http://dbpedia.org/resource/County_Tipperary, http://dbpedia.org/resource/Carrick-on-Suir, http://dbpedia.org/resource/Belvidere,_Illinois, http://dbpedia.org/resource/Ayr,_Ontario, http://dbpedia.org/resource/Chișinău, http://dbpedia.org/resource/Hampshire, http://dbpedia.org/resource/Farnborough,_Hampshire, http://dbpedia.org/resource/North_Miami_Beach,_Florida, http://dbpedia.org/resource/Dieppe,_New_Brunswick, http://dbpedia.org/resource/Center_Moriches,_New_York, http://dbpedia.org/resource/Sindh, http://dbpedia.org/resource/Americans, http://dbpedia.org/resource/Lahore, http://dbpedia.org/resource/Punjab,_Pakistan, http://dbpedia.org/resource/Bærum, http://dbpedia.org/resource/Aomori_Prefecture, http://dbpedia.org/resource/Canberra, http://dbpedia.org/resource/Australian_Capital_Territory, http://dbpedia.org/resource/Oxford,_Mississippi, http://dbpedia.org/resource/Pasadena,_California, http://dbpedia.org/resource/Czechoslovakia, http://dbpedia.org/resource/Uherské_Hradiště, http://dbpedia.org/resource/Portrane, http://dbpedia.org/resource/Kamakura, http://dbpedia.org/resource/Kanagawa_Prefecture, http://dbpedia.org/resource/Arlington_County,_Virginia, http://dbpedia.org/resource/Wales, http://dbpedia.org/resource/Bongabong,_Oriental_Mindoro, http://dbpedia.org/resource/Oriental_Mindoro, http://dbpedia.org/resource/Ludvika, http://dbpedia.org/resource/Eastry, http://dbpedia.org/resource/Kent, http://dbpedia.org/resource/Aylesbury, http://dbpedia.org/resource/Buckinghamshire, http://dbpedia.org/resource/La_Habra_Heights,_California, http://dbpedia.org/resource/Marshall,_North_Carolina, http://dbpedia.org/resource/Revere,_North_Carolina, http://dbpedia.org/resource/Winnetka,_Illinois, http://dbpedia.org/resource/Orange_County,_California, http://dbpedia.org/resource/Paddington, http://dbpedia.org/resource/List_of_sub_regions_used_in_the_London_Plan, http://dbpedia.org/resource/Vermont, http://dbpedia.org/resource/Haiti, http://dbpedia.org/resource/Port-au-Prince, http://dbpedia.org/resource/Coppell,_Texas, http://dbpedia.org/resource/Alum_Creek,_West_Virginia, http://dbpedia.org/resource/Jansen,_Saskatchewan, http://dbpedia.org/resource/Knowlton_Township,_New_Jersey, http://dbpedia.org/resource/Halmstad, http://dbpedia.org/resource/Kristianstad, http://dbpedia.org/resource/Manhattan_Beach,_California, http://dbpedia.org/resource/St._Catharines, http://dbpedia.org/resource/Selangor, http://dbpedia.org/resource/Kajang, http://dbpedia.org/resource/Strängnäs, http://dbpedia.org/resource/Worcester,_Massachusetts, http://dbpedia.org/resource/Hamilton,_Ontario, http://dbpedia.org/resource/Cambridge, http://dbpedia.org/resource/Tamil_Nadu, http://dbpedia.org/resource/Georgia_(country), http://dbpedia.org/resource/Fuengirola, http://dbpedia.org/resource/Bowling_Green,_Kentucky, http://dbpedia.org/resource/Story_City,_Iowa, http://dbpedia.org/resource/Syrian_Americans, http://dbpedia.org/resource/Prague, http://dbpedia.org/resource/North_Vancouver_(city), http://dbpedia.org/resource/Katy,_Texas, http://dbpedia.org/resource/Westfield,_New_Jersey, http://dbpedia.org/resource/New_Ulm,_Minnesota, http://dbpedia.org/resource/Lagos, http://dbpedia.org/resource/Mount_Dora,_Florida, http://dbpedia.org/resource/Montpelier,_Vermont, http://dbpedia.org/resource/Totnes, http://dbpedia.org/resource/West_Hartford,_Connecticut, http://dbpedia.org/resource/Norton,_New_Brunswick, http://dbpedia.org/resource/Mount_Kisco,_New_York, http://dbpedia.org/resource/Shelby,_North_Carolina, http://dbpedia.org/resource/Indonesia, http://dbpedia.org/resource/Central_Java, http://dbpedia.org/resource/Franklin,_Tennessee, http://dbpedia.org/resource/Blackpool, http://dbpedia.org/resource/Manhattan, http://dbpedia.org/resource/Bradford, http://dbpedia.org/resource/Cwmaman, http://dbpedia.org/resource/Oran_Province, http://dbpedia.org/resource/Mission_Viejo,_California, http://dbpedia.org/resource/Nagasaki_Prefecture, http://dbpedia.org/resource/Des_Moines,_Iowa, http://dbpedia.org/resource/Iowa, http://dbpedia.org/resource/Crawley, http://dbpedia.org/resource/Tilburg, http://dbpedia.org/resource/Peterborough,_Ontario, http://dbpedia.org/resource/Mississauga, http://dbpedia.org/resource/DeKalb,_Illinois, http://dbpedia.org/resource/Wheaton,_Illinois, http://dbpedia.org/resource/Chickasaw_County,_Mississippi, http://dbpedia.org/resource/Kelowna, http://dbpedia.org/resource/Wikwemikong_First_Nation, http://dbpedia.org/resource/Jaffray,_British_Columbia, http://dbpedia.org/resource/Ware,_Massachusetts, http://dbpedia.org/resource/Linden,_Texas, http://dbpedia.org/resource/Sicily, http://dbpedia.org/resource/Riposto, http://dbpedia.org/resource/Waycross,_Georgia, http://dbpedia.org/resource/Stanthorpe,_Queensland, http://dbpedia.org/resource/Douglas,_Georgia, http://dbpedia.org/resource/Sabinal,_Texas, http://dbpedia.org/resource/Moorhead,_Mississippi, http://dbpedia.org/resource/Westland,_Michigan, http://dbpedia.org/resource/Cross_Lanes,_West_Virginia, http://dbpedia.org/resource/Sandy_Hook,_Kentucky, http://dbpedia.org/resource/Cairns, http://dbpedia.org/resource/Chatham-Kent, http://dbpedia.org/resource/Winnipeg, http://dbpedia.org/resource/Pikeville,_Kentucky, http://dbpedia.org/resource/Highland_Park,_Illinois, http://dbpedia.org/resource/Hawkesbury,_Ontario, http://dbpedia.org/resource/Hackleburg,_Alabama, http://dbpedia.org/resource/Frederiksberg, http://dbpedia.org/resource/Beauregard_Parish,_Louisiana, http://dbpedia.org/resource/San_Carlos,_Chile, http://dbpedia.org/resource/Morrisville,_Missouri, http://dbpedia.org/resource/Swarcliffe, http://dbpedia.org/resource/Streetsville,_Ontario, http://dbpedia.org/resource/Lyons,_New_York, http://dbpedia.org/resource/Henderson,_Nevada, http://dbpedia.org/resource/Lasker,_North_Carolina, http://dbpedia.org/resource/Grand_Saline,_Texas, http://dbpedia.org/resource/Springfield,_Virginia, http://dbpedia.org/resource/Murray,_Utah, http://dbpedia.org/resource/Hillsboro,_Oregon, http://dbpedia.org/resource/Meonstoke, http://dbpedia.org/resource/South_Shields, http://dbpedia.org/resource/Tyne_and_Wear, http://dbpedia.org/resource/Miyagi_Prefecture, http://dbpedia.org/resource/Montebello,_California, http://dbpedia.org/resource/West_Riding_of_Yorkshire, http://dbpedia.org/resource/Kirkstall, http://dbpedia.org/resource/Albuquerque,_New_Mexico, http://dbpedia.org/resource/Bryn_Mawr,_Pennsylvania, http://dbpedia.org/resource/Santa_Rosa,_California, http://dbpedia.org/resource/Englewood,_New_Jersey, http://dbpedia.org/resource/Barnsley, http://dbpedia.org/resource/Cawthorne, http://dbpedia.org/resource/Fukuoka, http://dbpedia.org/resource/Nova_Scotia, http://dbpedia.org/resource/Dartmouth,_Nova_Scotia, http://dbpedia.org/resource/Kazakh_Soviet_Socialist_Republic, http://dbpedia.org/resource/Dagenham, http://dbpedia.org/resource/Chūbu_region, http://dbpedia.org/resource/Rotterdam, http://dbpedia.org/resource/Erdington, http://dbpedia.org/resource/Essen, http://dbpedia.org/resource/Darien,_Connecticut, http://dbpedia.org/resource/Connecticut, http://dbpedia.org/resource/Sharon,_Massachusetts, http://dbpedia.org/resource/San_Jose,_California, http://dbpedia.org/resource/Kansas_City,_Missouri, http://dbpedia.org/resource/Estonia, http://dbpedia.org/resource/Tallinn, http://dbpedia.org/resource/Hokkaido, http://dbpedia.org/resource/Sapporo, http://dbpedia.org/resource/Rockville,_Maryland, http://dbpedia.org/resource/East_Brunswick,_New_Jersey, http://dbpedia.org/resource/Bothell,_Washington, http://dbpedia.org/resource/Akron,_Ohio, http://dbpedia.org/resource/Uppsala, http://dbpedia.org/resource/Italian_Americans, http://dbpedia.org/resource/Faroe_Islands, http://dbpedia.org/resource/Wildwood,_Florida, http://dbpedia.org/resource/Hanover_Park,_Illinois, http://dbpedia.org/resource/Offenburg, http://dbpedia.org/resource/Lippstadt, http://dbpedia.org/resource/Blackwood,_Caerphilly, http://dbpedia.org/resource/Dorset, http://dbpedia.org/resource/Little_Hanford, http://dbpedia.org/resource/Halifax,_West_Yorkshire, http://dbpedia.org/resource/Salta, http://dbpedia.org/resource/Bilbao, http://dbpedia.org/resource/Inglewood,_California, http://dbpedia.org/resource/Plasencia, http://dbpedia.org/resource/Sewell,_Chile, http://dbpedia.org/resource/La_Rioja_(Spain), http://dbpedia.org/resource/Logroño, http://dbpedia.org/resource/Stuttgart, http://dbpedia.org/resource/Lincoln,_Nebraska, http://dbpedia.org/resource/Nebraska, http://dbpedia.org/resource/Clane, http://dbpedia.org/resource/Carlisle,_Cumbria, http://dbpedia.org/resource/Ise,_Mie, http://dbpedia.org/resource/Baltimore, http://dbpedia.org/resource/Healdsburg,_California, http://dbpedia.org/resource/Bandar-e_Anzali, http://dbpedia.org/resource/Gilan_Province, http://dbpedia.org/resource/Macon,_Georgia, http://dbpedia.org/resource/Madera,_California, http://dbpedia.org/resource/Paternò, http://dbpedia.org/resource/Bryne, http://dbpedia.org/resource/Beni_Yenni, http://dbpedia.org/resource/Chiba,_Chiba, http://dbpedia.org/resource/Oak_Park,_Illinois, http://dbpedia.org/resource/Kerala, http://dbpedia.org/resource/Kochi, http://dbpedia.org/resource/Chelyabinsk, http://dbpedia.org/resource/Västerås, http://dbpedia.org/resource/Pétionville, http://dbpedia.org/resource/Tel_Aviv, http://dbpedia.org/resource/Newark,_New_Jersey, http://dbpedia.org/resource/Nihonmatsu,_Fukushima, http://dbpedia.org/resource/Fukushima_Prefecture, http://dbpedia.org/resource/European_Union, http://dbpedia.org/resource/Zagreb, http://dbpedia.org/resource/Panama, http://dbpedia.org/resource/Cebu, http://dbpedia.org/resource/Bosnia_and_Herzegovina, http://dbpedia.org/resource/Socialist_Federal_Republic_of_Yugoslavia, http://dbpedia.org/resource/Bihać, http://dbpedia.org/resource/Tórshavn, http://dbpedia.org/resource/Bulgaria, http://dbpedia.org/resource/Morocco, http://dbpedia.org/resource/Oujda, http://dbpedia.org/resource/Makati, http://dbpedia.org/resource/Burnham-on-Crouch, http://dbpedia.org/resource/Columbia,_Maryland, http://dbpedia.org/resource/Nagano_Prefecture, http://dbpedia.org/resource/Chikuma,_Nagano, http://dbpedia.org/resource/Turin, http://dbpedia.org/resource/Yamatokōriyama, http://dbpedia.org/resource/Watford, http://dbpedia.org/resource/Lachine,_Quebec, http://dbpedia.org/resource/East_Los_Angeles,_California, http://dbpedia.org/resource/Charlestown,_Boston, http://dbpedia.org/resource/Mjölby, http://dbpedia.org/resource/Seminole_County,_Oklahoma, http://dbpedia.org/resource/Somerset, http://dbpedia.org/resource/Taunton, http://dbpedia.org/resource/Mumbai, http://dbpedia.org/resource/Edmonton, http://dbpedia.org/resource/Oulu, http://dbpedia.org/resource/Clark_Air_Base, http://dbpedia.org/resource/Islington, http://dbpedia.org/resource/Broussard,_Louisiana, http://dbpedia.org/resource/Singapore, http://dbpedia.org/resource/Quebec_City, http://dbpedia.org/resource/Northern_California, http://dbpedia.org/resource/Metairie,_Louisiana, http://dbpedia.org/resource/Brzeg, http://dbpedia.org/resource/Yinnar,_Victoria, http://dbpedia.org/resource/Pleasantville,_New_York, http://dbpedia.org/resource/Spruce_Pine,_North_Carolina, http://dbpedia.org/resource/Hanover,_Ontario, http://dbpedia.org/resource/Victoria,_British_Columbia, http://dbpedia.org/resource/Leicestershire, http://dbpedia.org/resource/Sileby]... [INFO] P: 0.9901900674432863, R: 1.0, F: 0.9950708564386939 -[INFO] Cumulative metrics, sample: 774, P: 0.495, R: 0.548, macro F1: 0.502, macro F1*: 0.520 -[INFO] Current time: 7130 ms, average: 2882.19 ms/ques, entity detection: 477.66 ms/ques (linking tool: 384.99 ms/ques), relation detection: 935.72 ms/ques (similarity: 709.61 ms/ques), EDG generation: 46.27 ms/ques +[INFO] Cumulative metrics, sample: 774, P: 0.495, R: 0.548, macro F1: 0.520 +[INFO] Current question consumed time: 6229 ms, average: 2592.62 ms/ques, entity detection: 421.42 ms/ques (linking tool: 343.46 ms/ques), relation detection: 862.45 ms/ques (similarity: 701.78 ms/ques), EDG generation: 46.18 ms/ques [INFO] Entity Linking P: 0.7403100775193798, R: 0.6937984496124031, micro F1: 0.7086563307493549, macro F1: 0.7163000209511838 [INFO] Relation Linking P: 0.5967915590008612, R: 0.5426356589147286, micro F1: 0.5579672695951774, macro F1: 0.5684266195529322 [INFO] Type Linking P: 0.5503875968992248, R: 0.5503875968992248, micro F1: 0.5503875968992248, macro F1: 0.5503875968992248 @@ -10867,8 +10867,8 @@ Question 774: How many religions are followed amongst the people who've worked i [INFO] Golden Answer: [7] [INFO] Predicted Answer: [1] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 775, P: 0.494, R: 0.548, macro F1: 0.502, macro F1*: 0.520 -[INFO] Current time: 3607 ms, average: 2883.13 ms/ques, entity detection: 477.18 ms/ques (linking tool: 384.58 ms/ques), relation detection: 936.06 ms/ques (similarity: 710.33 ms/ques), EDG generation: 46.25 ms/ques +[INFO] Cumulative metrics, sample: 775, P: 0.494, R: 0.548, macro F1: 0.520 +[INFO] Current question consumed time: 3148 ms, average: 2593.34 ms/ques, entity detection: 421.00 ms/ques (linking tool: 343.10 ms/ques), relation detection: 862.72 ms/ques (similarity: 702.48 ms/ques), EDG generation: 46.21 ms/ques [INFO] Entity Linking P: 0.7393548387096774, R: 0.6929032258064516, micro F1: 0.7077419354838719, macro F1: 0.7153757628596338 [INFO] Relation Linking P: 0.5966666666666666, R: 0.5425806451612903, micro F1: 0.5578924731182804, macro F1: 0.56833978291647 [INFO] Type Linking P: 0.5509677419354839, R: 0.5509677419354839, micro F1: 0.5509677419354839, macro F1: 0.5509677419354839 @@ -10881,8 +10881,8 @@ Question 775: Was andy warhol a part of Pop Art movement? [INFO] Golden Answer: [true] [INFO] Predicted Answer: [true] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 776, P: 0.495, R: 0.548, macro F1: 0.502, macro F1*: 0.520 -[INFO] Current time: 1872 ms, average: 2881.82 ms/ques, entity detection: 477.03 ms/ques (linking tool: 384.32 ms/ques), relation detection: 935.52 ms/ques (similarity: 709.76 ms/ques), EDG generation: 46.21 ms/ques +[INFO] Cumulative metrics, sample: 776, P: 0.495, R: 0.548, macro F1: 0.520 +[INFO] Current question consumed time: 1538 ms, average: 2591.98 ms/ques, entity detection: 420.82 ms/ques (linking tool: 342.84 ms/ques), relation detection: 862.18 ms/ques (similarity: 701.91 ms/ques), EDG generation: 46.18 ms/ques [INFO] Entity Linking P: 0.7390463917525774, R: 0.6926546391752577, micro F1: 0.7074742268041246, macro F1: 0.7150988939100096 [INFO] Relation Linking P: 0.595897766323024, R: 0.541881443298969, micro F1: 0.5571735395189011, macro F1: 0.5676073862890002 [INFO] Type Linking P: 0.5515463917525774, R: 0.5515463917525774, micro F1: 0.5515463917525774, macro F1: 0.5515463917525774 @@ -10895,8 +10895,8 @@ Question 776: Count the number of musical work which were sold by Sony Music? [INFO] Golden Answer: [98] [INFO] Predicted Answer: [215] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 777, P: 0.494, R: 0.548, macro F1: 0.502, macro F1*: 0.520 -[INFO] Current time: 1881 ms, average: 2880.53 ms/ques, entity detection: 476.81 ms/ques (linking tool: 384.13 ms/ques), relation detection: 934.77 ms/ques (similarity: 709.30 ms/ques), EDG generation: 46.17 ms/ques +[INFO] Cumulative metrics, sample: 777, P: 0.494, R: 0.548, macro F1: 0.520 +[INFO] Current question consumed time: 1564 ms, average: 2590.66 ms/ques, entity detection: 420.59 ms/ques (linking tool: 342.65 ms/ques), relation detection: 861.47 ms/ques (similarity: 701.47 ms/ques), EDG generation: 46.14 ms/ques [INFO] Entity Linking P: 0.7380952380952381, R: 0.6917631917631918, micro F1: 0.7065637065637075, macro F1: 0.7141785607132142 [INFO] Relation Linking P: 0.5964178464178463, R: 0.5418275418275418, micro F1: 0.5573144573144582, macro F1: 0.5678136172812511 [INFO] Type Linking P: 0.5508365508365508, R: 0.5508365508365508, micro F1: 0.5508365508365508, macro F1: 0.5508365508365508 @@ -10909,8 +10909,8 @@ Question 777: Who is the person whose home town is Fairmount, Indiana? [INFO] Golden Answer: [http://dbpedia.org/resource/James_Dean] [INFO] Predicted Answer: [http://dbpedia.org/resource/Dottie_Rambo, http://dbpedia.org/resource/Jesse_Vest, http://dbpedia.org/resource/Johnny_Ross, http://dbpedia.org/resource/Tony_Fate, http://dbpedia.org/resource/Chris_Wallace_(musician), http://dbpedia.org/resource/Eli_Lilly_(industrialist), http://dbpedia.org/resource/Paul_Dresser, http://dbpedia.org/resource/Kevon_Edmonds, http://dbpedia.org/resource/Krista_Detor, http://dbpedia.org/resource/Indiana_Gregg, http://dbpedia.org/resource/Jaron_Marquis, http://dbpedia.org/resource/Kyle_Cook, http://dbpedia.org/resource/Sally_Anthony, http://dbpedia.org/resource/Shannon_Hurley, http://dbpedia.org/resource/Babyface_(musician), http://dbpedia.org/resource/Jim_Cooper_(musician), http://dbpedia.org/resource/Mark_Battles, http://dbpedia.org/resource/J._J._Johnson, http://dbpedia.org/resource/Rick_Florian, http://dbpedia.org/resource/Brendan_Bayliss, http://dbpedia.org/resource/Bumble_Bee_Slim, http://dbpedia.org/resource/Chris_Clavin, http://dbpedia.org/resource/Rayna_Gellert, http://dbpedia.org/resource/Andy_Farag, http://dbpedia.org/resource/Aaron_Deer, http://dbpedia.org/resource/Chris_Funk, http://dbpedia.org/resource/Beata_Moon, http://dbpedia.org/resource/Figure_(musician), http://dbpedia.org/resource/Josh_Kaufman, http://dbpedia.org/resource/David_Simkins, http://dbpedia.org/resource/Josiah_K._Lilly,_Sr., http://dbpedia.org/resource/Zohra_Sarwari, http://dbpedia.org/resource/Craig_Fox_(musician), http://dbpedia.org/resource/Chief_Menominee, http://dbpedia.org/resource/Mary_Flower, http://dbpedia.org/resource/Carl_Broemel, http://dbpedia.org/resource/Larry_Pierce_(singer)] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 778, P: 0.494, R: 0.547, macro F1: 0.501, macro F1*: 0.519 -[INFO] Current time: 6095 ms, average: 2884.67 ms/ques, entity detection: 476.94 ms/ques (linking tool: 384.22 ms/ques), relation detection: 936.47 ms/ques (similarity: 711.23 ms/ques), EDG generation: 46.14 ms/ques +[INFO] Cumulative metrics, sample: 778, P: 0.494, R: 0.547, macro F1: 0.519 +[INFO] Current question consumed time: 5202 ms, average: 2594.01 ms/ques, entity detection: 420.68 ms/ques (linking tool: 342.71 ms/ques), relation detection: 862.92 ms/ques (similarity: 703.38 ms/ques), EDG generation: 46.12 ms/ques [INFO] Entity Linking P: 0.737146529562982, R: 0.6908740359897172, micro F1: 0.7056555269922888, macro F1: 0.7132605934115263 [INFO] Relation Linking P: 0.5969365895458439, R: 0.5424164524421594, micro F1: 0.5578834618680385, macro F1: 0.5683720766118556 [INFO] Type Linking P: 0.5501285347043702, R: 0.5501285347043702, micro F1: 0.5501285347043702, macro F1: 0.5501285347043702 @@ -10923,8 +10923,8 @@ Question 778: Name the movies directed by Stanley Kubrick and edited by Anthony [INFO] Golden Answer: [http://dbpedia.org/resource/Lolita_(1962_film), http://dbpedia.org/resource/Dr._Strangelove] [INFO] Predicted Answer: [http://dbpedia.org/resource/Lolita_(1962_film), http://dbpedia.org/resource/Dr._Strangelove] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 779, P: 0.494, R: 0.548, macro F1: 0.502, macro F1*: 0.520 -[INFO] Current time: 1894 ms, average: 2883.39 ms/ques, entity detection: 476.86 ms/ques (linking tool: 384.06 ms/ques), relation detection: 936.03 ms/ques (similarity: 710.75 ms/ques), EDG generation: 46.11 ms/ques +[INFO] Cumulative metrics, sample: 779, P: 0.494, R: 0.548, macro F1: 0.520 +[INFO] Current question consumed time: 1565 ms, average: 2592.69 ms/ques, entity detection: 420.53 ms/ques (linking tool: 342.51 ms/ques), relation detection: 862.51 ms/ques (similarity: 702.92 ms/ques), EDG generation: 46.09 ms/ques [INFO] Entity Linking P: 0.7374839537869063, R: 0.6912708600770219, micro F1: 0.7060333761232358, macro F1: 0.713630025362532 [INFO] Relation Linking P: 0.597454000855798, R: 0.5430038510911425, micro F1: 0.558451005562688, macro F1: 0.5689290888929801 [INFO] Type Linking P: 0.5494223363286265, R: 0.5494223363286265, micro F1: 0.5494223363286265, macro F1: 0.5494223363286265 @@ -10937,8 +10937,8 @@ Question 779: List all binomial authority of the insects whose one of the binom [INFO] Golden Answer: [http://dbpedia.org/resource/Édouard_Ménétries, http://dbpedia.org/resource/Azerbaijan, http://dbpedia.org/resource/Kura_River, http://dbpedia.org/resource/Transcaucasia, http://dbpedia.org/resource/Sal'yana, http://dbpedia.org/resource/Julian_Simashko] [INFO] Predicted Answer: [http://dbpedia.org/resource/A.F.P._Hulsewé] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 780, P: 0.494, R: 0.547, macro F1: 0.501, macro F1*: 0.519 -[INFO] Current time: 4377 ms, average: 2885.31 ms/ques, entity detection: 476.35 ms/ques (linking tool: 383.62 ms/ques), relation detection: 938.24 ms/ques (similarity: 711.88 ms/ques), EDG generation: 46.13 ms/ques +[INFO] Cumulative metrics, sample: 780, P: 0.494, R: 0.547, macro F1: 0.519 +[INFO] Current question consumed time: 3876 ms, average: 2594.34 ms/ques, entity detection: 420.09 ms/ques (linking tool: 342.13 ms/ques), relation detection: 864.43 ms/ques (similarity: 704.06 ms/ques), EDG generation: 46.16 ms/ques [INFO] Entity Linking P: 0.7365384615384616, R: 0.6903846153846154, micro F1: 0.705128205128206, macro F1: 0.7127151150736057 [INFO] Relation Linking P: 0.5966880341880342, R: 0.5423076923076923, micro F1: 0.5577350427350436, macro F1: 0.5681996926251686 [INFO] Type Linking P: 0.55, R: 0.55, micro F1: 0.55, macro F1: 0.55 @@ -10951,8 +10951,8 @@ Question 780: Who is the military general buried in St Mary's Church, Battersea? [INFO] Golden Answer: [http://dbpedia.org/resource/Benedict_Arnold] [INFO] Predicted Answer: [http://dbpedia.org/resource/Albert_Bridge,_London, http://dbpedia.org/resource/Chelsea_Bridge] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 781, P: 0.493, R: 0.546, macro F1: 0.500, macro F1*: 0.518 -[INFO] Current time: 2845 ms, average: 2885.26 ms/ques, entity detection: 476.70 ms/ques (linking tool: 383.74 ms/ques), relation detection: 937.82 ms/ques (similarity: 711.81 ms/ques), EDG generation: 46.10 ms/ques +[INFO] Cumulative metrics, sample: 781, P: 0.493, R: 0.546, macro F1: 0.518 +[INFO] Current question consumed time: 2408 ms, average: 2594.10 ms/ques, entity detection: 420.27 ms/ques (linking tool: 342.17 ms/ques), relation detection: 864.02 ms/ques (similarity: 704.01 ms/ques), EDG generation: 46.14 ms/ques [INFO] Entity Linking P: 0.735595390524968, R: 0.6895006402048656, micro F1: 0.7042253521126769, macro F1: 0.7118025477047535 [INFO] Relation Linking P: 0.5959240290226205, R: 0.5416133162612036, micro F1: 0.5570209133589423, macro F1: 0.5674721642095154 [INFO] Type Linking P: 0.5492957746478874, R: 0.5492957746478874, micro F1: 0.5492957746478874, macro F1: 0.5492957746478874 @@ -10965,8 +10965,8 @@ Question 781: What is the military unit whose command structures are United Stat [INFO] Golden Answer: [http://dbpedia.org/resource/United_States_Navy] [INFO] Predicted Answer: [http://dbpedia.org/resource/United_States_Navy] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 782, P: 0.494, R: 0.547, macro F1: 0.501, macro F1*: 0.519 -[INFO] Current time: 6536 ms, average: 2889.93 ms/ques, entity detection: 478.61 ms/ques (linking tool: 385.68 ms/ques), relation detection: 937.76 ms/ques (similarity: 711.56 ms/ques), EDG generation: 47.29 ms/ques +[INFO] Cumulative metrics, sample: 782, P: 0.494, R: 0.547, macro F1: 0.519 +[INFO] Current question consumed time: 5818 ms, average: 2598.22 ms/ques, entity detection: 421.93 ms/ques (linking tool: 343.84 ms/ques), relation detection: 863.96 ms/ques (similarity: 703.80 ms/ques), EDG generation: 47.33 ms/ques [INFO] Entity Linking P: 0.7359335038363172, R: 0.6898976982097187, micro F1: 0.7046035805626607, macro F1: 0.7121724220981042 [INFO] Relation Linking P: 0.5964407502131287, R: 0.5421994884910486, micro F1: 0.5575873827791995, macro F1: 0.5680281772736356 [INFO] Type Linking P: 0.5485933503836317, R: 0.5485933503836317, micro F1: 0.5485933503836317, macro F1: 0.5485933503836317 @@ -10979,8 +10979,8 @@ Question 782: What are some other things developed by the company who made the r [INFO] Golden Answer: [http://dbpedia.org/resource/Microsoft_Office, http://dbpedia.org/resource/Windows_Live_Messenger, http://dbpedia.org/resource/Remote_Desktop_Protocol] [INFO] Predicted Answer: [http://dbpedia.org/resource/Desktop_virtualization, http://dbpedia.org/resource/Linux, http://dbpedia.org/resource/Server_(computing), http://dbpedia.org/resource/HTML5, http://dbpedia.org/resource/Remote_Desktop_Services, http://dbpedia.org/resource/Terminal_emulator, http://dbpedia.org/resource/Business_continuity, http://dbpedia.org/resource/Remote_Desktop_Protocol, http://dbpedia.org/resource/Microsoft_Office, http://dbpedia.org/resource/Windows_Live_Messenger] [INFO] P: 0.3, R: 1.0, F: 0.4615384615384615 -[INFO] Cumulative metrics, sample: 783, P: 0.493, R: 0.547, macro F1: 0.501, macro F1*: 0.519 -[INFO] Current time: 1279 ms, average: 2887.87 ms/ques, entity detection: 478.20 ms/ques (linking tool: 385.31 ms/ques), relation detection: 936.97 ms/ques (similarity: 710.94 ms/ques), EDG generation: 47.26 ms/ques +[INFO] Cumulative metrics, sample: 783, P: 0.493, R: 0.547, macro F1: 0.519 +[INFO] Current question consumed time: 1020 ms, average: 2596.21 ms/ques, entity detection: 421.53 ms/ques (linking tool: 343.49 ms/ques), relation detection: 863.21 ms/ques (similarity: 703.18 ms/ques), EDG generation: 47.30 ms/ques [INFO] Entity Linking P: 0.7362707535121328, R: 0.69029374201788, micro F1: 0.7049808429118782, macro F1: 0.7125413469531026 [INFO] Relation Linking P: 0.5969561515538526, R: 0.5421455938697318, micro F1: 0.5577266922094517, macro F1: 0.5682322033103477 [INFO] Type Linking P: 0.5478927203065134, R: 0.5478927203065134, micro F1: 0.5478927203065134, macro F1: 0.5478927203065134 @@ -10993,8 +10993,8 @@ Question 783: Where was Al-Muzaffar Umar buried ? [INFO] Golden Answer: [http://dbpedia.org/resource/Syria, http://dbpedia.org/resource/Hama] [INFO] Predicted Answer: [http://dbpedia.org/resource/Syria, http://dbpedia.org/resource/Hama] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 784, P: 0.494, R: 0.548, macro F1: 0.502, macro F1*: 0.520 -[INFO] Current time: 767 ms, average: 2885.16 ms/ques, entity detection: 477.77 ms/ques (linking tool: 384.92 ms/ques), relation detection: 936.05 ms/ques (similarity: 710.18 ms/ques), EDG generation: 47.21 ms/ques +[INFO] Cumulative metrics, sample: 784, P: 0.494, R: 0.548, macro F1: 0.520 +[INFO] Current question consumed time: 601 ms, average: 2593.66 ms/ques, entity detection: 421.12 ms/ques (linking tool: 343.13 ms/ques), relation detection: 862.35 ms/ques (similarity: 702.43 ms/ques), EDG generation: 47.26 ms/ques [INFO] Entity Linking P: 0.7366071428571429, R: 0.6906887755102041, micro F1: 0.7053571428571437, macro F1: 0.712909325928763 [INFO] Relation Linking P: 0.597470238095238, R: 0.5427295918367347, micro F1: 0.5582908163265314, macro F1: 0.5687858740960726 [INFO] Type Linking P: 0.548469387755102, R: 0.548469387755102, micro F1: 0.548469387755102, macro F1: 0.548469387755102 @@ -11007,8 +11007,8 @@ Question 784: List the guests of On Broadway ? [INFO] Golden Answer: [http://dbpedia.org/resource/Thorsten_Kaye, http://dbpedia.org/resource/Michael_Cristofer, http://dbpedia.org/resource/Jennifer_Hudson, http://dbpedia.org/resource/Brian_d'Arcy_James, as Jerry Rand@en, as Frank Houston@en, as Nick@en, as Veronica Moore@en] [INFO] Predicted Answer: [http://dbpedia.org/resource/Thorsten_Kaye, http://dbpedia.org/resource/Michael_Cristofer, http://dbpedia.org/resource/Jennifer_Hudson, http://dbpedia.org/resource/Brian_d'Arcy_James] [INFO] P: 1.0, R: 0.5, F: 0.6666666666666666 -[INFO] Cumulative metrics, sample: 785, P: 0.495, R: 0.548, macro F1: 0.502, macro F1*: 0.520 -[INFO] Current time: 727 ms, average: 2882.41 ms/ques, entity detection: 477.35 ms/ques (linking tool: 384.53 ms/ques), relation detection: 935.16 ms/ques (similarity: 709.46 ms/ques), EDG generation: 47.16 ms/ques +[INFO] Cumulative metrics, sample: 785, P: 0.495, R: 0.548, macro F1: 0.520 +[INFO] Current question consumed time: 617 ms, average: 2591.14 ms/ques, entity detection: 420.73 ms/ques (linking tool: 342.78 ms/ques), relation detection: 861.52 ms/ques (similarity: 701.71 ms/ques), EDG generation: 47.21 ms/ques [INFO] Entity Linking P: 0.7369426751592356, R: 0.6910828025477707, micro F1: 0.705732484076434, macro F1: 0.7132763626652726 [INFO] Relation Linking P: 0.5967091295116772, R: 0.5420382165605095, micro F1: 0.5575796178343958, macro F1: 0.5680613061035935 [INFO] Type Linking P: 0.5490445859872611, R: 0.5490445859872611, micro F1: 0.5490445859872611, macro F1: 0.5490445859872611 @@ -11021,8 +11021,8 @@ Question 785: How many universities are there whose country's capital is Oslo? [INFO] Golden Answer: [58] [INFO] Predicted Answer: [58, 52] [INFO] P: 0.5, R: 1.0, F: 0.6666666666666666 -[INFO] Cumulative metrics, sample: 786, P: 0.495, R: 0.548, macro F1: 0.502, macro F1*: 0.520 -[INFO] Current time: 12474 ms, average: 2894.62 ms/ques, entity detection: 477.80 ms/ques (linking tool: 384.98 ms/ques), relation detection: 941.49 ms/ques (similarity: 714.32 ms/ques), EDG generation: 47.13 ms/ques +[INFO] Cumulative metrics, sample: 786, P: 0.495, R: 0.548, macro F1: 0.520 +[INFO] Current question consumed time: 11173 ms, average: 2602.06 ms/ques, entity detection: 421.11 ms/ques (linking tool: 343.17 ms/ques), relation detection: 867.34 ms/ques (similarity: 706.62 ms/ques), EDG generation: 47.17 ms/ques [INFO] Entity Linking P: 0.7372773536895675, R: 0.6914758269720102, micro F1: 0.7061068702290085, macro F1: 0.7136424607841138 [INFO] Relation Linking P: 0.5972222222222222, R: 0.5426208651399491, micro F1: 0.5581424936386777, macro F1: 0.5686137723622608 [INFO] Type Linking P: 0.5483460559796438, R: 0.5483460559796438, micro F1: 0.5483460559796438, macro F1: 0.5483460559796438 @@ -11035,8 +11035,8 @@ Question 786: Did Bob Dylan write the lyrics of the times they are a changin'? [INFO] Golden Answer: [true] [INFO] Predicted Answer: [true] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 787, P: 0.495, R: 0.549, macro F1: 0.503, macro F1*: 0.521 -[INFO] Current time: 1432 ms, average: 2892.76 ms/ques, entity detection: 477.83 ms/ques (linking tool: 385.04 ms/ques), relation detection: 940.76 ms/ques (similarity: 713.72 ms/ques), EDG generation: 47.20 ms/ques +[INFO] Cumulative metrics, sample: 787, P: 0.495, R: 0.549, macro F1: 0.521 +[INFO] Current question consumed time: 1435 ms, average: 2600.58 ms/ques, entity detection: 421.13 ms/ques (linking tool: 343.20 ms/ques), relation detection: 866.74 ms/ques (similarity: 706.11 ms/ques), EDG generation: 47.37 ms/ques [INFO] Entity Linking P: 0.7376111817026684, R: 0.6912325285895807, micro F1: 0.7060567556120296, macro F1: 0.7136691557959122 [INFO] Relation Linking P: 0.5964633629817874, R: 0.5419313850063533, micro F1: 0.5574332909783999, macro F1: 0.5678912643922962 [INFO] Type Linking P: 0.5489199491740788, R: 0.5489199491740788, micro F1: 0.5489199491740788, macro F1: 0.5489199491740788 @@ -11049,8 +11049,8 @@ Question 787: What is the religion of the ones in the Kampuchean Union of Salvat [INFO] Golden Answer: [http://dbpedia.org/resource/Buddhism] [INFO] Predicted Answer: [http://dbpedia.org/resource/Officer_(armed_forces), http://dbpedia.org/resource/Nikita_Muravyov__1] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 788, P: 0.495, R: 0.548, macro F1: 0.502, macro F1*: 0.520 -[INFO] Current time: 1148 ms, average: 2890.54 ms/ques, entity detection: 477.42 ms/ques (linking tool: 384.73 ms/ques), relation detection: 939.86 ms/ques (similarity: 713.01 ms/ques), EDG generation: 47.26 ms/ques +[INFO] Cumulative metrics, sample: 788, P: 0.495, R: 0.548, macro F1: 0.520 +[INFO] Current question consumed time: 1085 ms, average: 2598.66 ms/ques, entity detection: 420.78 ms/ques (linking tool: 342.94 ms/ques), relation detection: 865.92 ms/ques (similarity: 705.41 ms/ques), EDG generation: 47.49 ms/ques [INFO] Entity Linking P: 0.7366751269035533, R: 0.6903553299492385, micro F1: 0.7051607445008469, macro F1: 0.7127634842784045 [INFO] Relation Linking P: 0.5957064297800337, R: 0.541243654822335, micro F1: 0.5567258883248739, macro F1: 0.5671705901989049 [INFO] Type Linking P: 0.549492385786802, R: 0.549492385786802, micro F1: 0.549492385786802, macro F1: 0.549492385786802 @@ -11063,8 +11063,8 @@ Question 788: Where did the members of judge advocate general corps study? [INFO] Golden Answer: [http://dbpedia.org/resource/Baker_High_School_(Louisiana), http://dbpedia.org/resource/Paul_M._Hebert_Law_Center, http://dbpedia.org/resource/Louisiana_State_University, http://dbpedia.org/resource/Tulane_University_Law_School, http://dbpedia.org/resource/Tulane_University, http://dbpedia.org/resource/Trinity_University_(Texas), http://dbpedia.org/resource/University_of_Texas_School_of_Law] [INFO] Predicted Answer: [] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 789, P: 0.494, R: 0.548, macro F1: 0.501, macro F1*: 0.519 -[INFO] Current time: 529 ms, average: 2887.55 ms/ques, entity detection: 476.81 ms/ques (linking tool: 384.25 ms/ques), relation detection: 938.84 ms/ques (similarity: 712.21 ms/ques), EDG generation: 47.30 ms/ques +[INFO] Cumulative metrics, sample: 789, P: 0.494, R: 0.548, macro F1: 0.519 +[INFO] Current question consumed time: 467 ms, average: 2595.95 ms/ques, entity detection: 420.24 ms/ques (linking tool: 342.50 ms/ques), relation detection: 864.97 ms/ques (similarity: 704.62 ms/ques), EDG generation: 47.53 ms/ques [INFO] Entity Linking P: 0.7357414448669202, R: 0.6894803548795945, micro F1: 0.7042670046472336, macro F1: 0.7118601085061887 [INFO] Relation Linking P: 0.5949514152936206, R: 0.5405576679340938, micro F1: 0.5560202788339679, macro F1: 0.5664517428095527 [INFO] Type Linking P: 0.550063371356147, R: 0.550063371356147, micro F1: 0.550063371356147, macro F1: 0.550063371356147 @@ -11077,8 +11077,8 @@ Question 789: Name the airlines which have a hub in airport in SF? [INFO] Golden Answer: [http://dbpedia.org/resource/Virgin_America, http://dbpedia.org/resource/Pacific_Air_Lines] [INFO] Predicted Answer: [] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 790, P: 0.494, R: 0.547, macro F1: 0.501, macro F1*: 0.519 -[INFO] Current time: 1221 ms, average: 2885.44 ms/ques, entity detection: 476.93 ms/ques (linking tool: 384.48 ms/ques), relation detection: 938.08 ms/ques (similarity: 711.58 ms/ques), EDG generation: 47.27 ms/ques +[INFO] Cumulative metrics, sample: 790, P: 0.494, R: 0.547, macro F1: 0.519 +[INFO] Current question consumed time: 1011 ms, average: 2593.95 ms/ques, entity detection: 420.29 ms/ques (linking tool: 342.65 ms/ques), relation detection: 864.25 ms/ques (similarity: 703.99 ms/ques), EDG generation: 47.50 ms/ques [INFO] Entity Linking P: 0.7348101265822785, R: 0.6886075949367089, micro F1: 0.7033755274261612, macro F1: 0.7109590197612441 [INFO] Relation Linking P: 0.5941983122362868, R: 0.539873417721519, micro F1: 0.5553164556962034, macro F1: 0.5657347152870089 [INFO] Type Linking P: 0.549367088607595, R: 0.549367088607595, micro F1: 0.549367088607595, macro F1: 0.549367088607595 @@ -11091,8 +11091,8 @@ Question 790: Which executive producer of the ultimate merger is also an importa [INFO] Golden Answer: [http://dbpedia.org/resource/Donald_Trump] [INFO] Predicted Answer: [http://dbpedia.org/resource/Donald_Trump] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 791, P: 0.494, R: 0.547, macro F1: 0.501, macro F1*: 0.519 -[INFO] Current time: 1493 ms, average: 2883.68 ms/ques, entity detection: 476.69 ms/ques (linking tool: 384.21 ms/ques), relation detection: 937.36 ms/ques (similarity: 710.96 ms/ques), EDG generation: 47.25 ms/ques +[INFO] Cumulative metrics, sample: 791, P: 0.494, R: 0.547, macro F1: 0.519 +[INFO] Current question consumed time: 1195 ms, average: 2592.18 ms/ques, entity detection: 420.06 ms/ques (linking tool: 342.39 ms/ques), relation detection: 863.57 ms/ques (similarity: 703.38 ms/ques), EDG generation: 47.48 ms/ques [INFO] Entity Linking P: 0.7351453855878635, R: 0.6890012642225032, micro F1: 0.7037505267593771, macro F1: 0.7113257614653982 [INFO] Relation Linking P: 0.5947113358617783, R: 0.5404551201011378, micro F1: 0.5558786346396974, macro F1: 0.5662866177208209 [INFO] Type Linking P: 0.549936788874842, R: 0.549936788874842, micro F1: 0.549936788874842, macro F1: 0.549936788874842 @@ -11105,8 +11105,8 @@ Question 791: What is the official name of Colorado wine ? [INFO] Golden Answer: [http://dbpedia.org/resource/Colorado] [INFO] Predicted Answer: [http://dbpedia.org/resource/Colorado] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 792, P: 0.495, R: 0.548, macro F1: 0.502, macro F1*: 0.520 -[INFO] Current time: 1529 ms, average: 2881.97 ms/ques, entity detection: 477.01 ms/ques (linking tool: 384.55 ms/ques), relation detection: 936.35 ms/ques (similarity: 710.18 ms/ques), EDG generation: 47.21 ms/ques +[INFO] Cumulative metrics, sample: 792, P: 0.495, R: 0.548, macro F1: 0.520 +[INFO] Current question consumed time: 1370 ms, average: 2590.64 ms/ques, entity detection: 420.35 ms/ques (linking tool: 342.70 ms/ques), relation detection: 862.65 ms/ques (similarity: 702.61 ms/ques), EDG generation: 47.44 ms/ques [INFO] Entity Linking P: 0.735479797979798, R: 0.6893939393939394, micro F1: 0.7041245791245799, macro F1: 0.7116915723473102 [INFO] Relation Linking P: 0.5952230639730639, R: 0.5410353535353535, micro F1: 0.5564393939393948, macro F1: 0.5668371136122778 [INFO] Type Linking P: 0.5505050505050505, R: 0.5505050505050505, micro F1: 0.5505050505050505, macro F1: 0.5505050505050505 @@ -11119,8 +11119,8 @@ Question 792: In how many states can one find Burr Tuss kind of bridges? [INFO] Golden Answer: [2] [INFO] Predicted Answer: [1] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 793, P: 0.494, R: 0.547, macro F1: 0.501, macro F1*: 0.519 -[INFO] Current time: 1392 ms, average: 2880.09 ms/ques, entity detection: 476.60 ms/ques (linking tool: 384.21 ms/ques), relation detection: 935.68 ms/ques (similarity: 709.88 ms/ques), EDG generation: 47.17 ms/ques +[INFO] Cumulative metrics, sample: 793, P: 0.494, R: 0.547, macro F1: 0.519 +[INFO] Current question consumed time: 1280 ms, average: 2588.98 ms/ques, entity detection: 420.00 ms/ques (linking tool: 342.40 ms/ques), relation detection: 862.05 ms/ques (similarity: 702.31 ms/ques), EDG generation: 47.39 ms/ques [INFO] Entity Linking P: 0.7345523329129886, R: 0.6885245901639344, micro F1: 0.703236654056327, macro F1: 0.7107941050429629 [INFO] Relation Linking P: 0.5944724674232871, R: 0.5403530895334174, micro F1: 0.5557377049180336, macro F1: 0.5661223127123886 [INFO] Type Linking P: 0.5498108448928121, R: 0.5498108448928121, micro F1: 0.5498108448928121, macro F1: 0.5498108448928121 @@ -11133,8 +11133,8 @@ Question 793: who developed google videos? [INFO] Golden Answer: [http://dbpedia.org/resource/Google] [INFO] Predicted Answer: [http://dbpedia.org/resource/Google] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 794, P: 0.495, R: 0.548, macro F1: 0.502, macro F1*: 0.520 -[INFO] Current time: 768 ms, average: 2877.43 ms/ques, entity detection: 476.22 ms/ques (linking tool: 383.88 ms/ques), relation detection: 934.75 ms/ques (similarity: 709.13 ms/ques), EDG generation: 47.11 ms/ques +[INFO] Cumulative metrics, sample: 794, P: 0.495, R: 0.548, macro F1: 0.520 +[INFO] Current question consumed time: 681 ms, average: 2586.58 ms/ques, entity detection: 419.67 ms/ques (linking tool: 342.12 ms/ques), relation detection: 861.17 ms/ques (similarity: 701.56 ms/ques), EDG generation: 47.34 ms/ques [INFO] Entity Linking P: 0.7348866498740554, R: 0.6889168765743073, micro F1: 0.7036104114189765, macro F1: 0.7111596594093027 [INFO] Relation Linking P: 0.5949832073887489, R: 0.5409319899244333, micro F1: 0.5562972292191444, macro F1: 0.5666716161658713 [INFO] Type Linking P: 0.5503778337531486, R: 0.5503778337531486, micro F1: 0.5503778337531486, macro F1: 0.5503778337531486 @@ -11147,8 +11147,8 @@ Question 794: Who is the child of John Tyler, Sr.? [INFO] Golden Answer: [http://dbpedia.org/resource/John_Tyler] [INFO] Predicted Answer: [http://dbpedia.org/resource/John_Tyler,_Sr., http://dbpedia.org/resource/Lyon_Gardiner_Tyler, http://dbpedia.org/resource/David_Gardiner_Tyler, http://dbpedia.org/resource/John_Alexander_Tyler] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 795, P: 0.494, R: 0.547, macro F1: 0.501, macro F1*: 0.519 -[INFO] Current time: 1816 ms, average: 2876.10 ms/ques, entity detection: 475.90 ms/ques (linking tool: 383.58 ms/ques), relation detection: 934.67 ms/ques (similarity: 708.89 ms/ques), EDG generation: 47.08 ms/ques +[INFO] Cumulative metrics, sample: 795, P: 0.494, R: 0.547, macro F1: 0.519 +[INFO] Current question consumed time: 1625 ms, average: 2585.37 ms/ques, entity detection: 419.38 ms/ques (linking tool: 341.86 ms/ques), relation detection: 861.11 ms/ques (similarity: 701.34 ms/ques), EDG generation: 47.30 ms/ques [INFO] Entity Linking P: 0.7339622641509433, R: 0.6880503144654088, micro F1: 0.702725366876311, macro F1: 0.7102651189572153 [INFO] Relation Linking P: 0.5954926624737945, R: 0.5415094339622641, micro F1: 0.5568553459119505, macro F1: 0.5672195250925829 [INFO] Type Linking P: 0.5509433962264151, R: 0.5509433962264151, micro F1: 0.5509433962264151, macro F1: 0.5509433962264151 @@ -11161,8 +11161,8 @@ Question 795: Is Brazilian jiu-jitsu the trainer of Nick Diaz? [INFO] Golden Answer: [true] [INFO] Predicted Answer: [true] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 796, P: 0.495, R: 0.548, macro F1: 0.502, macro F1*: 0.520 -[INFO] Current time: 1500 ms, average: 2874.37 ms/ques, entity detection: 475.89 ms/ques (linking tool: 383.39 ms/ques), relation detection: 933.88 ms/ques (similarity: 708.23 ms/ques), EDG generation: 47.12 ms/ques +[INFO] Cumulative metrics, sample: 796, P: 0.495, R: 0.548, macro F1: 0.520 +[INFO] Current question consumed time: 1337 ms, average: 2583.80 ms/ques, entity detection: 419.34 ms/ques (linking tool: 341.68 ms/ques), relation detection: 860.38 ms/ques (similarity: 700.69 ms/ques), EDG generation: 47.41 ms/ques [INFO] Entity Linking P: 0.7342964824120602, R: 0.6884422110552764, micro F1: 0.7030988274706875, macro F1: 0.7106304147669916 [INFO] Relation Linking P: 0.596000837520938, R: 0.5420854271356784, micro F1: 0.5574120603015084, macro F1: 0.5677660448318309 [INFO] Type Linking P: 0.5515075376884422, R: 0.5515075376884422, micro F1: 0.5515075376884422, macro F1: 0.5515075376884422 @@ -11175,8 +11175,8 @@ Question 796: Who is the owner of Latvian Wikipedia and Wiktionary ? [INFO] Golden Answer: [http://dbpedia.org/resource/Wikimedia_Foundation] [INFO] Predicted Answer: [http://dbpedia.org/resource/Wikimedia_Foundation] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 797, P: 0.495, R: 0.548, macro F1: 0.503, macro F1*: 0.521 -[INFO] Current time: 1089 ms, average: 2872.13 ms/ques, entity detection: 475.56 ms/ques (linking tool: 383.08 ms/ques), relation detection: 933.10 ms/ques (similarity: 707.58 ms/ques), EDG generation: 47.10 ms/ques +[INFO] Cumulative metrics, sample: 797, P: 0.495, R: 0.548, macro F1: 0.521 +[INFO] Current question consumed time: 1049 ms, average: 2581.88 ms/ques, entity detection: 419.08 ms/ques (linking tool: 341.42 ms/ques), relation detection: 859.75 ms/ques (similarity: 700.13 ms/ques), EDG generation: 47.39 ms/ques [INFO] Entity Linking P: 0.7346298619824341, R: 0.6888331242158093, micro F1: 0.7034713508992062, macro F1: 0.7109947892963532 [INFO] Relation Linking P: 0.5965077373483898, R: 0.5426599749058971, micro F1: 0.5579673776662493, macro F1: 0.5683111806953911 [INFO] Type Linking P: 0.5520702634880803, R: 0.5520702634880803, micro F1: 0.5520702634880803, macro F1: 0.5520702634880803 @@ -11189,8 +11189,8 @@ Question 797: Name all those buried in a place governed by a city council. [INFO] Golden Answer: [http://dbpedia.org/resource/Shirley_Temple, http://dbpedia.org/resource/Ray_Lyman_Wilbur, http://dbpedia.org/resource/Paul_Shoup, http://dbpedia.org/resource/Ronald_Reagan] [INFO] Predicted Answer: [http://dbpedia.org/resource/Louis_H._Narcisse] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 798, P: 0.495, R: 0.548, macro F1: 0.502, macro F1*: 0.520 -[INFO] Current time: 8671 ms, average: 2879.40 ms/ques, entity detection: 475.14 ms/ques (linking tool: 382.69 ms/ques), relation detection: 938.52 ms/ques (similarity: 710.68 ms/ques), EDG generation: 47.07 ms/ques +[INFO] Cumulative metrics, sample: 798, P: 0.495, R: 0.548, macro F1: 0.520 +[INFO] Current question consumed time: 7626 ms, average: 2588.20 ms/ques, entity detection: 418.71 ms/ques (linking tool: 341.08 ms/ques), relation detection: 864.75 ms/ques (similarity: 703.23 ms/ques), EDG generation: 47.36 ms/ques [INFO] Entity Linking P: 0.7349624060150376, R: 0.6892230576441103, micro F1: 0.7038429406850467, macro F1: 0.7113582460330323 [INFO] Relation Linking P: 0.5963868003341687, R: 0.5426065162907269, micro F1: 0.5578947368421061, macro F1: 0.5682269761687615 [INFO] Type Linking P: 0.5526315789473685, R: 0.5526315789473685, micro F1: 0.5526315789473685, macro F1: 0.5526315789473685 @@ -11203,8 +11203,8 @@ Question 798: What royalties's sepulchre is Mausoleum of Augustus? [INFO] Golden Answer: [http://dbpedia.org/resource/Augustus, http://dbpedia.org/resource/Caligula, http://dbpedia.org/resource/Claudius, http://dbpedia.org/resource/Tiberius, http://dbpedia.org/resource/Agrippina_the_Elder, http://dbpedia.org/resource/Britannicus, http://dbpedia.org/resource/Drusus_Caesar, http://dbpedia.org/resource/Drusus_Julius_Caesar, http://dbpedia.org/resource/Gaius_Caesar, http://dbpedia.org/resource/Germanicus, http://dbpedia.org/resource/Julia_Livilla, http://dbpedia.org/resource/Livia, http://dbpedia.org/resource/Nero_Claudius_Drusus, http://dbpedia.org/resource/Nero_Julius_Caesar, http://dbpedia.org/resource/Octavia_the_Younger, http://dbpedia.org/resource/Poppaea_Sabina, http://dbpedia.org/resource/Early_life_of_Augustus] [INFO] Predicted Answer: [http://dbpedia.org/resource/Augustus, http://dbpedia.org/resource/Caligula, http://dbpedia.org/resource/Claudius, http://dbpedia.org/resource/Tiberius, http://dbpedia.org/resource/Agrippina_the_Elder, http://dbpedia.org/resource/Britannicus, http://dbpedia.org/resource/Drusus_Caesar, http://dbpedia.org/resource/Drusus_Julius_Caesar, http://dbpedia.org/resource/Gaius_Caesar, http://dbpedia.org/resource/Germanicus, http://dbpedia.org/resource/Julia_Livilla, http://dbpedia.org/resource/Livia, http://dbpedia.org/resource/Nero_Claudius_Drusus, http://dbpedia.org/resource/Nero_Julius_Caesar, http://dbpedia.org/resource/Octavia_the_Younger, http://dbpedia.org/resource/Poppaea_Sabina, http://dbpedia.org/resource/Early_life_of_Augustus] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 799, P: 0.496, R: 0.548, macro F1: 0.503, macro F1*: 0.521 -[INFO] Current time: 1276 ms, average: 2877.39 ms/ques, entity detection: 474.77 ms/ques (linking tool: 382.41 ms/ques), relation detection: 937.78 ms/ques (similarity: 710.17 ms/ques), EDG generation: 47.03 ms/ques +[INFO] Cumulative metrics, sample: 799, P: 0.496, R: 0.548, macro F1: 0.521 +[INFO] Current question consumed time: 1072 ms, average: 2586.30 ms/ques, entity detection: 418.36 ms/ques (linking tool: 340.80 ms/ques), relation detection: 864.06 ms/ques (similarity: 702.73 ms/ques), EDG generation: 47.31 ms/ques [INFO] Entity Linking P: 0.7352941176470589, R: 0.6896120150187734, micro F1: 0.7042136003337514, macro F1: 0.7117207884471312 [INFO] Relation Linking P: 0.5968919482686691, R: 0.5431789737171464, micro F1: 0.5584480600750947, macro F1: 0.5687701521513547 [INFO] Type Linking P: 0.5519399249061326, R: 0.5519399249061326, micro F1: 0.5519399249061326, macro F1: 0.5519399249061326 @@ -11217,8 +11217,8 @@ Question 799: Name the common university of the Cardinal and Jim Wylie? [INFO] Golden Answer: [http://dbpedia.org/resource/Stanford_University] [INFO] Predicted Answer: [http://dbpedia.org/resource/Stanford_University, http://dbpedia.org/resource/University_of_Auckland] [INFO] P: 0.5, R: 1.0, F: 0.6666666666666666 -[INFO] Cumulative metrics, sample: 800, P: 0.496, R: 0.549, macro F1: 0.503, macro F1*: 0.521 -[INFO] Current time: 2544 ms, average: 2876.97 ms/ques, entity detection: 474.72 ms/ques (linking tool: 382.28 ms/ques), relation detection: 937.97 ms/ques (similarity: 710.04 ms/ques), EDG generation: 47.01 ms/ques +[INFO] Cumulative metrics, sample: 800, P: 0.496, R: 0.549, macro F1: 0.521 +[INFO] Current question consumed time: 2098 ms, average: 2585.69 ms/ques, entity detection: 418.24 ms/ques (linking tool: 340.65 ms/ques), relation detection: 864.16 ms/ques (similarity: 702.61 ms/ques), EDG generation: 47.30 ms/ques [INFO] Entity Linking P: 0.735625, R: 0.689375, micro F1: 0.7041666666666674, macro F1: 0.711749451754386 [INFO] Relation Linking P: 0.5973958333333332, R: 0.54375, micro F1: 0.5590000000000008, macro F1: 0.569311958010041 [INFO] Type Linking P: 0.5525, R: 0.5525, micro F1: 0.5525, macro F1: 0.5525 @@ -11231,8 +11231,8 @@ Question 800: What is the automobile platform of Chiva bus ? [INFO] Golden Answer: [http://dbpedia.org/resource/Ford_Motor_Company, http://dbpedia.org/resource/Dodge, http://dbpedia.org/resource/Blue_Bird_Corporation] [INFO] Predicted Answer: [http://dbpedia.org/resource/Ford_Motor_Company, http://dbpedia.org/resource/Dodge, http://dbpedia.org/resource/Blue_Bird_Corporation] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 801, P: 0.496, R: 0.549, macro F1: 0.504, macro F1*: 0.521 -[INFO] Current time: 800 ms, average: 2874.38 ms/ques, entity detection: 474.36 ms/ques (linking tool: 381.95 ms/ques), relation detection: 936.96 ms/ques (similarity: 709.24 ms/ques), EDG generation: 46.97 ms/ques +[INFO] Cumulative metrics, sample: 801, P: 0.496, R: 0.549, macro F1: 0.521 +[INFO] Current question consumed time: 657 ms, average: 2583.28 ms/ques, entity detection: 417.91 ms/ques (linking tool: 340.34 ms/ques), relation detection: 863.23 ms/ques (similarity: 701.82 ms/ques), EDG generation: 47.25 ms/ques [INFO] Entity Linking P: 0.7359550561797753, R: 0.6897627965043696, micro F1: 0.7045359966708289, macro F1: 0.7121106279147563 [INFO] Relation Linking P: 0.5978984602580107, R: 0.5443196004993758, micro F1: 0.5595505617977536, macro F1: 0.5698523989561732 [INFO] Type Linking P: 0.5530586766541823, R: 0.5530586766541823, micro F1: 0.5530586766541823, macro F1: 0.5530586766541823 @@ -11245,8 +11245,8 @@ Question 801: Who are the animator of The Skeleton Dance? [INFO] Golden Answer: [http://dbpedia.org/resource/Les_Clark, http://dbpedia.org/resource/Wilfred_Jackson, http://dbpedia.org/resource/Roy_O._Disney, http://dbpedia.org/resource/Walt_Disney, http://dbpedia.org/resource/Ub_Iwerks] [INFO] Predicted Answer: [http://dbpedia.org/resource/Les_Clark, http://dbpedia.org/resource/Wilfred_Jackson, http://dbpedia.org/resource/Roy_O._Disney, http://dbpedia.org/resource/Walt_Disney, http://dbpedia.org/resource/Ub_Iwerks, Wilfred Jackson@en, Les Clark ,@en, Roy O. Disney ,@en, Walt Disney ,@en] [INFO] P: 0.625, R: 1.0, F: 0.7692307692307693 -[INFO] Cumulative metrics, sample: 802, P: 0.496, R: 0.550, macro F1: 0.504, macro F1*: 0.522 -[INFO] Current time: 714 ms, average: 2871.69 ms/ques, entity detection: 474.00 ms/ques (linking tool: 381.64 ms/ques), relation detection: 935.97 ms/ques (similarity: 708.47 ms/ques), EDG generation: 46.92 ms/ques +[INFO] Cumulative metrics, sample: 802, P: 0.496, R: 0.550, macro F1: 0.522 +[INFO] Current question consumed time: 586 ms, average: 2580.79 ms/ques, entity detection: 417.57 ms/ques (linking tool: 340.05 ms/ques), relation detection: 862.32 ms/ques (similarity: 701.06 ms/ques), EDG generation: 47.21 ms/ques [INFO] Entity Linking P: 0.736284289276808, R: 0.690149625935162, micro F1: 0.7049044056525361, macro F1: 0.7124708988019461 [INFO] Relation Linking P: 0.5983998337489609, R: 0.5448877805486284, micro F1: 0.5600997506234422, macro F1: 0.5703914801744163 [INFO] Type Linking P: 0.5536159600997507, R: 0.5536159600997507, micro F1: 0.5536159600997507, macro F1: 0.5536159600997507 @@ -11259,8 +11259,8 @@ Question 802: How many movies have a screenplay located in the States? [INFO] Golden Answer: [812] [INFO] Predicted Answer: [1] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 803, P: 0.496, R: 0.549, macro F1: 0.503, macro F1*: 0.521 -[INFO] Current time: 33057 ms, average: 2909.28 ms/ques, entity detection: 500.20 ms/ques (linking tool: 407.81 ms/ques), relation detection: 935.30 ms/ques (similarity: 708.08 ms/ques), EDG generation: 46.88 ms/ques +[INFO] Cumulative metrics, sample: 803, P: 0.496, R: 0.549, macro F1: 0.521 +[INFO] Current question consumed time: 28697 ms, average: 2613.32 ms/ques, entity detection: 440.15 ms/ques (linking tool: 362.63 ms/ques), relation detection: 861.69 ms/ques (similarity: 700.68 ms/ques), EDG generation: 47.17 ms/ques [INFO] Entity Linking P: 0.7353673723536738, R: 0.6892901618929016, micro F1: 0.7040265670402664, macro F1: 0.7115836374086686 [INFO] Relation Linking P: 0.5976546284765463, R: 0.5442092154420921, micro F1: 0.5594022415940233, macro F1: 0.5696811545453074 [INFO] Type Linking P: 0.5529265255292652, R: 0.5529265255292652, micro F1: 0.5529265255292652, macro F1: 0.5529265255292652 @@ -11273,8 +11273,8 @@ Question 803: Which ceremonial County of the East Tilbury is also the local auth [INFO] Golden Answer: [http://dbpedia.org/resource/Essex] [INFO] Predicted Answer: [http://dbpedia.org/resource/Shawbrook_Bank, http://dbpedia.org/resource/Equity_Insurance_Group] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 804, P: 0.495, R: 0.549, macro F1: 0.503, macro F1*: 0.520 -[INFO] Current time: 5114 ms, average: 2912.02 ms/ques, entity detection: 500.34 ms/ques (linking tool: 407.68 ms/ques), relation detection: 936.92 ms/ques (similarity: 708.81 ms/ques), EDG generation: 46.98 ms/ques +[INFO] Cumulative metrics, sample: 804, P: 0.495, R: 0.549, macro F1: 0.520 +[INFO] Current question consumed time: 4441 ms, average: 2615.59 ms/ques, entity detection: 440.24 ms/ques (linking tool: 362.50 ms/ques), relation detection: 863.14 ms/ques (similarity: 701.40 ms/ques), EDG generation: 47.28 ms/ques [INFO] Entity Linking P: 0.7344527363184079, R: 0.6884328358208955, micro F1: 0.7031509121061368, macro F1: 0.7106985831332847 [INFO] Relation Linking P: 0.5969112769485904, R: 0.5435323383084577, micro F1: 0.5587064676616924, macro F1: 0.5689725958953754 [INFO] Type Linking P: 0.5534825870646766, R: 0.5534825870646766, micro F1: 0.5534825870646766, macro F1: 0.5534825870646766 @@ -11287,8 +11287,8 @@ Question 804: What system is both a platform where Nord runs and also distribute [INFO] Golden Answer: [http://dbpedia.org/resource/Facebook] [INFO] Predicted Answer: [http://dbpedia.org/resource/Facebook, http://dbpedia.org/resource/YouTube] [INFO] P: 0.5, R: 1.0, F: 0.6666666666666666 -[INFO] Cumulative metrics, sample: 805, P: 0.495, R: 0.549, macro F1: 0.503, macro F1*: 0.521 -[INFO] Current time: 3439 ms, average: 2912.67 ms/ques, entity detection: 500.13 ms/ques (linking tool: 407.45 ms/ques), relation detection: 937.14 ms/ques (similarity: 709.46 ms/ques), EDG generation: 46.99 ms/ques +[INFO] Cumulative metrics, sample: 805, P: 0.495, R: 0.549, macro F1: 0.521 +[INFO] Current question consumed time: 3023 ms, average: 2616.10 ms/ques, entity detection: 439.99 ms/ques (linking tool: 362.25 ms/ques), relation detection: 863.32 ms/ques (similarity: 702.06 ms/ques), EDG generation: 47.29 ms/ques [INFO] Entity Linking P: 0.7347826086956522, R: 0.6881987577639752, micro F1: 0.7031055900621125, macro F1: 0.7107281802136907 [INFO] Relation Linking P: 0.5974120082815735, R: 0.5434782608695652, micro F1: 0.5588405797101458, macro F1: 0.5691703191141023 [INFO] Type Linking P: 0.5540372670807453, R: 0.5540372670807453, micro F1: 0.5540372670807453, macro F1: 0.5540372670807453 @@ -11301,8 +11301,8 @@ Question 805: Name the country whose leader's deputy is Piotr Glinski? [INFO] Golden Answer: [http://dbpedia.org/resource/Poland] [INFO] Predicted Answer: [http://dbpedia.org/resource/Poland] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 806, P: 0.496, R: 0.550, macro F1: 0.503, macro F1*: 0.521 -[INFO] Current time: 1570 ms, average: 2911.01 ms/ques, entity detection: 499.73 ms/ques (linking tool: 407.06 ms/ques), relation detection: 936.75 ms/ques (similarity: 709.25 ms/ques), EDG generation: 46.96 ms/ques +[INFO] Cumulative metrics, sample: 806, P: 0.496, R: 0.550, macro F1: 0.521 +[INFO] Current question consumed time: 1406 ms, average: 2614.59 ms/ques, entity detection: 439.63 ms/ques (linking tool: 361.89 ms/ques), relation detection: 862.95 ms/ques (similarity: 701.86 ms/ques), EDG generation: 47.30 ms/ques [INFO] Entity Linking P: 0.7351116625310173, R: 0.6885856079404467, micro F1: 0.70347394540943, macro F1: 0.7110884055855592 [INFO] Relation Linking P: 0.5972911497105045, R: 0.543424317617866, micro F1: 0.5587675765095128, macro F1: 0.5690858846874671 [INFO] Type Linking P: 0.5533498759305211, R: 0.5533498759305211, micro F1: 0.5533498759305211, macro F1: 0.5533498759305211 @@ -11315,8 +11315,8 @@ Question 806: Is Cologny a neighbouring city of Geneva? [INFO] Golden Answer: [true] [INFO] Predicted Answer: [true] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 807, P: 0.496, R: 0.550, macro F1: 0.504, macro F1*: 0.522 -[INFO] Current time: 4327 ms, average: 2912.76 ms/ques, entity detection: 499.70 ms/ques (linking tool: 406.99 ms/ques), relation detection: 939.69 ms/ques (similarity: 710.76 ms/ques), EDG generation: 46.91 ms/ques +[INFO] Cumulative metrics, sample: 807, P: 0.496, R: 0.550, macro F1: 0.522 +[INFO] Current question consumed time: 3908 ms, average: 2616.20 ms/ques, entity detection: 439.58 ms/ques (linking tool: 361.82 ms/ques), relation detection: 865.62 ms/ques (similarity: 703.38 ms/ques), EDG generation: 47.26 ms/ques [INFO] Entity Linking P: 0.7354399008674102, R: 0.6889714993804213, micro F1: 0.7038413878562585, macro F1: 0.7114477335924839 [INFO] Relation Linking P: 0.5965510119785212, R: 0.5427509293680297, micro F1: 0.5580751755472954, macro F1: 0.5683806977175939 [INFO] Type Linking P: 0.5539033457249071, R: 0.5539033457249071, micro F1: 0.5539033457249071, macro F1: 0.5539033457249071 @@ -11329,8 +11329,8 @@ Question 807: Whose creators are Jerry Seinfeld and Larry David? [INFO] Golden Answer: [http://dbpedia.org/resource/Cosmo_Kramer, http://dbpedia.org/resource/Elaine_Benes, http://dbpedia.org/resource/Jackie_Chiles, http://dbpedia.org/resource/Newman_(Seinfeld), http://dbpedia.org/resource/Uncle_Leo, http://dbpedia.org/resource/Seinfeld, http://dbpedia.org/resource/George_Costanza, http://dbpedia.org/resource/Jerry_Seinfeld_(character)] [INFO] Predicted Answer: [http://dbpedia.org/resource/Cosmo_Kramer, http://dbpedia.org/resource/Seinfeld, http://dbpedia.org/resource/George_Costanza, http://dbpedia.org/resource/Jerry_Seinfeld_(character), http://dbpedia.org/resource/Elaine_Benes, http://dbpedia.org/resource/Jackie_Chiles, http://dbpedia.org/resource/Newman_(Seinfeld), http://dbpedia.org/resource/Uncle_Leo] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 808, P: 0.497, R: 0.551, macro F1: 0.505, macro F1*: 0.522 -[INFO] Current time: 3323 ms, average: 2913.27 ms/ques, entity detection: 499.73 ms/ques (linking tool: 406.92 ms/ques), relation detection: 939.56 ms/ques (similarity: 711.01 ms/ques), EDG generation: 46.88 ms/ques +[INFO] Cumulative metrics, sample: 808, P: 0.497, R: 0.551, macro F1: 0.522 +[INFO] Current question consumed time: 3001 ms, average: 2616.67 ms/ques, entity detection: 439.60 ms/ques (linking tool: 361.76 ms/ques), relation detection: 865.52 ms/ques (similarity: 703.65 ms/ques), EDG generation: 47.22 ms/ques [INFO] Entity Linking P: 0.7357673267326733, R: 0.6893564356435643, micro F1: 0.7042079207920799, macro F1: 0.7118061675902718 [INFO] Relation Linking P: 0.5970503300330032, R: 0.5433168316831684, micro F1: 0.5586221122112219, macro F1: 0.5689176338272337 [INFO] Type Linking P: 0.5544554455445545, R: 0.5544554455445545, micro F1: 0.5544554455445545, macro F1: 0.5544554455445545 @@ -11343,8 +11343,8 @@ Question 808: Which TV show is related to The Sarah Jane Adventures and theme mu [INFO] Golden Answer: [http://dbpedia.org/resource/Doctor_Who] [INFO] Predicted Answer: [http://dbpedia.org/resource/Doctor_Who] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 809, P: 0.498, R: 0.551, macro F1: 0.505, macro F1*: 0.523 -[INFO] Current time: 3090 ms, average: 2913.49 ms/ques, entity detection: 499.54 ms/ques (linking tool: 406.67 ms/ques), relation detection: 939.62 ms/ques (similarity: 711.47 ms/ques), EDG generation: 46.88 ms/ques +[INFO] Cumulative metrics, sample: 809, P: 0.498, R: 0.551, macro F1: 0.523 +[INFO] Current question consumed time: 2726 ms, average: 2616.81 ms/ques, entity detection: 439.41 ms/ques (linking tool: 361.52 ms/ques), relation detection: 865.55 ms/ques (similarity: 704.11 ms/ques), EDG generation: 47.23 ms/ques [INFO] Entity Linking P: 0.7360939431396786, R: 0.6897404202719407, micro F1: 0.7045735475896175, macro F1: 0.7121637109179726 [INFO] Relation Linking P: 0.5975484136794396, R: 0.5438813349814586, micro F1: 0.5591676967449535, macro F1: 0.5694532306159085 [INFO] Type Linking P: 0.553770086526576, R: 0.553770086526576, micro F1: 0.553770086526576, macro F1: 0.553770086526576 @@ -11357,8 +11357,8 @@ Question 809: Name the parent company of Ford Air Transport Service ? [INFO] Golden Answer: [http://dbpedia.org/resource/Ford_Motor_Company] [INFO] Predicted Answer: [http://dbpedia.org/resource/Ford_Motor_Company] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 810, P: 0.498, R: 0.552, macro F1: 0.506, macro F1*: 0.524 -[INFO] Current time: 646 ms, average: 2910.69 ms/ques, entity detection: 499.13 ms/ques (linking tool: 406.29 ms/ques), relation detection: 938.60 ms/ques (similarity: 710.69 ms/ques), EDG generation: 46.91 ms/ques +[INFO] Cumulative metrics, sample: 810, P: 0.498, R: 0.552, macro F1: 0.524 +[INFO] Current question consumed time: 519 ms, average: 2614.22 ms/ques, entity detection: 439.05 ms/ques (linking tool: 361.19 ms/ques), relation detection: 864.62 ms/ques (similarity: 703.33 ms/ques), EDG generation: 47.19 ms/ques [INFO] Entity Linking P: 0.7364197530864197, R: 0.6901234567901234, micro F1: 0.7049382716049389, macro F1: 0.7125203668979811 [INFO] Relation Linking P: 0.5980452674897119, R: 0.5444444444444444, micro F1: 0.5597119341563794, macro F1: 0.5699874931212165 [INFO] Type Linking P: 0.554320987654321, R: 0.554320987654321, micro F1: 0.554320987654321, macro F1: 0.554320987654321 @@ -11371,8 +11371,8 @@ Question 810: Who has these major shrines : Emmitsburg, Maryland and Shrine of [INFO] Golden Answer: [http://dbpedia.org/resource/Elizabeth_Ann_Seton] [INFO] Predicted Answer: [http://dbpedia.org/resource/Elizabeth_Ann_Seton] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 811, P: 0.499, R: 0.552, macro F1: 0.506, macro F1*: 0.524 -[INFO] Current time: 5715 ms, average: 2914.15 ms/ques, entity detection: 499.30 ms/ques (linking tool: 406.39 ms/ques), relation detection: 940.75 ms/ques (similarity: 711.55 ms/ques), EDG generation: 46.91 ms/ques +[INFO] Cumulative metrics, sample: 811, P: 0.499, R: 0.552, macro F1: 0.524 +[INFO] Current question consumed time: 4973 ms, average: 2617.13 ms/ques, entity detection: 439.18 ms/ques (linking tool: 361.26 ms/ques), relation detection: 866.59 ms/ques (similarity: 704.21 ms/ques), EDG generation: 47.20 ms/ques [INFO] Entity Linking P: 0.7367447595561035, R: 0.6898890258939581, micro F1: 0.7048910809699965, macro F1: 0.7125474381532236 [INFO] Relation Linking P: 0.5985408960131524, R: 0.5450061652281134, micro F1: 0.5602548294286898, macro F1: 0.570520426355245 [INFO] Type Linking P: 0.5548705302096177, R: 0.5548705302096177, micro F1: 0.5548705302096177, macro F1: 0.5548705302096177 @@ -11385,8 +11385,8 @@ Question 811: Which region of Bannock is the origin od Spaghetti squash ? [INFO] Golden Answer: [http://dbpedia.org/resource/North_America] [INFO] Predicted Answer: [http://dbpedia.org/resource/North_America] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 812, P: 0.499, R: 0.553, macro F1: 0.507, macro F1*: 0.525 -[INFO] Current time: 2331 ms, average: 2913.43 ms/ques, entity detection: 499.12 ms/ques (linking tool: 406.15 ms/ques), relation detection: 940.77 ms/ques (similarity: 711.32 ms/ques), EDG generation: 46.88 ms/ques +[INFO] Cumulative metrics, sample: 812, P: 0.499, R: 0.553, macro F1: 0.525 +[INFO] Current question consumed time: 1909 ms, average: 2616.26 ms/ques, entity detection: 438.97 ms/ques (linking tool: 361.02 ms/ques), relation detection: 866.54 ms/ques (similarity: 703.99 ms/ques), EDG generation: 47.18 ms/ques [INFO] Entity Linking P: 0.7370689655172413, R: 0.6902709359605911, micro F1: 0.7052545155993438, macro F1: 0.7129027699265121 [INFO] Relation Linking P: 0.599035303776683, R: 0.5455665024630542, micro F1: 0.5607963875205263, macro F1: 0.5710520353047336 [INFO] Type Linking P: 0.5554187192118226, R: 0.5554187192118226, micro F1: 0.5554187192118226, macro F1: 0.5554187192118226 @@ -11399,8 +11399,8 @@ Question 812: Where in the us military did pierre r graham and edwin f hunter wo [INFO] Golden Answer: [http://dbpedia.org/resource/United_States_Navy] [INFO] Predicted Answer: [http://dbpedia.org/resource/United_States_Navy] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 813, P: 0.500, R: 0.554, macro F1: 0.508, macro F1*: 0.525 -[INFO] Current time: 2379 ms, average: 2912.77 ms/ques, entity detection: 498.73 ms/ques (linking tool: 405.75 ms/ques), relation detection: 940.80 ms/ques (similarity: 711.13 ms/ques), EDG generation: 46.89 ms/ques +[INFO] Cumulative metrics, sample: 813, P: 0.500, R: 0.554, macro F1: 0.525 +[INFO] Current question consumed time: 1942 ms, average: 2615.43 ms/ques, entity detection: 438.61 ms/ques (linking tool: 360.66 ms/ques), relation detection: 866.52 ms/ques (similarity: 703.80 ms/ques), EDG generation: 47.19 ms/ques [INFO] Entity Linking P: 0.7373923739237392, R: 0.6900369003690037, micro F1: 0.7052070520705213, macro F1: 0.7129291198125253 [INFO] Relation Linking P: 0.5982984829848298, R: 0.5448954489544895, micro F1: 0.5601066010660115, macro F1: 0.5703496342772986 [INFO] Type Linking P: 0.5559655596555966, R: 0.5559655596555966, micro F1: 0.5559655596555966, macro F1: 0.5559655596555966 @@ -11413,8 +11413,8 @@ Question 813: Count everyone who lives in a place where Indian English is an off [INFO] Golden Answer: [1199] [INFO] Predicted Answer: [2] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 814, P: 0.499, R: 0.553, macro F1: 0.507, macro F1*: 0.525 -[INFO] Current time: 12845 ms, average: 2924.97 ms/ques, entity detection: 498.35 ms/ques (linking tool: 405.40 ms/ques), relation detection: 947.20 ms/ques (similarity: 715.47 ms/ques), EDG generation: 46.87 ms/ques +[INFO] Cumulative metrics, sample: 814, P: 0.499, R: 0.553, macro F1: 0.525 +[INFO] Current question consumed time: 11696 ms, average: 2626.58 ms/ques, entity detection: 438.25 ms/ques (linking tool: 360.33 ms/ques), relation detection: 872.36 ms/ques (similarity: 708.14 ms/ques), EDG generation: 47.17 ms/ques [INFO] Entity Linking P: 0.7377149877149877, R: 0.6904176904176904, micro F1: 0.7055692055692062, macro F1: 0.7132831365089429 [INFO] Relation Linking P: 0.5981777231777231, R: 0.5448402948402948, micro F1: 0.5600327600327609, macro F1: 0.5702645486344546 [INFO] Type Linking P: 0.5565110565110565, R: 0.5565110565110565, micro F1: 0.5565110565110565, macro F1: 0.5565110565110565 @@ -11427,8 +11427,8 @@ Question 814: Did Raymond Picard take birth in Paris? [INFO] Golden Answer: [true] [INFO] Predicted Answer: [true] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 815, P: 0.500, R: 0.553, macro F1: 0.508, macro F1*: 0.525 -[INFO] Current time: 7706 ms, average: 2930.84 ms/ques, entity detection: 500.22 ms/ques (linking tool: 407.14 ms/ques), relation detection: 951.42 ms/ques (similarity: 717.14 ms/ques), EDG generation: 46.82 ms/ques +[INFO] Cumulative metrics, sample: 815, P: 0.500, R: 0.553, macro F1: 0.525 +[INFO] Current question consumed time: 6726 ms, average: 2631.61 ms/ques, entity detection: 439.90 ms/ques (linking tool: 361.90 ms/ques), relation detection: 875.98 ms/ques (similarity: 709.83 ms/ques), EDG generation: 47.12 ms/ques [INFO] Entity Linking P: 0.7380368098159509, R: 0.69079754601227, micro F1: 0.7059304703476489, macro F1: 0.7136362798220358 [INFO] Relation Linking P: 0.597443762781186, R: 0.5441717791411043, micro F1: 0.5593456032719845, macro F1: 0.5695648375318355 [INFO] Type Linking P: 0.5570552147239264, R: 0.5570552147239264, micro F1: 0.5570552147239264, macro F1: 0.5570552147239264 @@ -11441,8 +11441,8 @@ Question 815: How many religions are practiced by diplomats? [INFO] Golden Answer: [9] [INFO] Predicted Answer: [] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 816, P: 0.499, R: 0.553, macro F1: 0.507, macro F1*: 0.525 -[INFO] Current time: 376 ms, average: 2927.71 ms/ques, entity detection: 499.61 ms/ques (linking tool: 406.64 ms/ques), relation detection: 950.57 ms/ques (similarity: 716.47 ms/ques), EDG generation: 46.78 ms/ques +[INFO] Cumulative metrics, sample: 816, P: 0.499, R: 0.553, macro F1: 0.525 +[INFO] Current question consumed time: 333 ms, average: 2628.80 ms/ques, entity detection: 439.36 ms/ques (linking tool: 361.45 ms/ques), relation detection: 875.19 ms/ques (similarity: 709.17 ms/ques), EDG generation: 47.08 ms/ques [INFO] Entity Linking P: 0.7371323529411765, R: 0.6899509803921569, micro F1: 0.7050653594771248, macro F1: 0.712761725557548 [INFO] Relation Linking P: 0.5967116013071895, R: 0.5435049019607843, micro F1: 0.558660130718955, macro F1: 0.5688668414074094 [INFO] Type Linking P: 0.5575980392156863, R: 0.5575980392156863, micro F1: 0.5575980392156863, macro F1: 0.5575980392156863 @@ -11455,8 +11455,8 @@ Question 816: Was Elton John a chairman of a Watford FC Season? [INFO] Golden Answer: [true] [INFO] Predicted Answer: [true] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 817, P: 0.500, R: 0.553, macro F1: 0.508, macro F1*: 0.525 -[INFO] Current time: 3469 ms, average: 2928.37 ms/ques, entity detection: 500.03 ms/ques (linking tool: 406.98 ms/ques), relation detection: 951.57 ms/ques (similarity: 716.78 ms/ques), EDG generation: 46.74 ms/ques +[INFO] Cumulative metrics, sample: 817, P: 0.500, R: 0.553, macro F1: 0.525 +[INFO] Current question consumed time: 3000 ms, average: 2629.25 ms/ques, entity detection: 439.70 ms/ques (linking tool: 361.72 ms/ques), relation detection: 876.05 ms/ques (similarity: 709.48 ms/ques), EDG generation: 47.08 ms/ques [INFO] Entity Linking P: 0.7368421052631579, R: 0.6897184822521419, micro F1: 0.7048143614851087, macro F1: 0.7125019756598703 [INFO] Relation Linking P: 0.5959812321501428, R: 0.5428396572827417, micro F1: 0.5579763361893113, macro F1: 0.5681705539638261 [INFO] Type Linking P: 0.5581395348837209, R: 0.5581395348837209, micro F1: 0.5581395348837209, macro F1: 0.5581395348837209 @@ -11469,8 +11469,8 @@ Question 817: Which publisher of Lucifer's hammer is also known for Jo Garcia ? [INFO] Golden Answer: [http://dbpedia.org/resource/Playboy] [INFO] Predicted Answer: [http://dbpedia.org/resource/Playboy_Cyber_Club, http://dbpedia.org/resource/Playboy, 2007-06-18, appearance@en, ''@en] [INFO] P: 0.2, R: 1.0, F: 0.33333333333333337 -[INFO] Cumulative metrics, sample: 818, P: 0.500, R: 0.554, macro F1: 0.507, macro F1*: 0.525 -[INFO] Current time: 5618 ms, average: 2931.66 ms/ques, entity detection: 500.26 ms/ques (linking tool: 406.96 ms/ques), relation detection: 952.56 ms/ques (similarity: 717.07 ms/ques), EDG generation: 46.73 ms/ques +[INFO] Cumulative metrics, sample: 818, P: 0.500, R: 0.554, macro F1: 0.525 +[INFO] Current question consumed time: 4617 ms, average: 2631.68 ms/ques, entity detection: 439.82 ms/ques (linking tool: 361.68 ms/ques), relation detection: 876.88 ms/ques (similarity: 709.77 ms/ques), EDG generation: 47.07 ms/ques [INFO] Entity Linking P: 0.7371638141809291, R: 0.6894865525672371, micro F1: 0.7047677261613697, macro F1: 0.7125285196196127 [INFO] Relation Linking P: 0.5964751426242868, R: 0.5427872860635696, micro F1: 0.5581092094539536, macro F1: 0.5683661915232406 [INFO] Type Linking P: 0.558679706601467, R: 0.558679706601467, micro F1: 0.558679706601467, macro F1: 0.558679706601467 @@ -11483,8 +11483,8 @@ Question 818: To whom was Tessa Dahl born to? [INFO] Golden Answer: [http://dbpedia.org/resource/Patricia_Neal, http://dbpedia.org/resource/Roald_Dahl] [INFO] Predicted Answer: [Oxford@en] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 819, P: 0.499, R: 0.553, macro F1: 0.507, macro F1*: 0.525 -[INFO] Current time: 776 ms, average: 2929.03 ms/ques, entity detection: 499.89 ms/ques (linking tool: 406.59 ms/ques), relation detection: 951.58 ms/ques (similarity: 716.30 ms/ques), EDG generation: 46.68 ms/ques +[INFO] Cumulative metrics, sample: 819, P: 0.499, R: 0.553, macro F1: 0.525 +[INFO] Current question consumed time: 633 ms, average: 2629.24 ms/ques, entity detection: 439.48 ms/ques (linking tool: 361.33 ms/ques), relation detection: 875.98 ms/ques (similarity: 709.01 ms/ques), EDG generation: 47.02 ms/ques [INFO] Entity Linking P: 0.7374847374847375, R: 0.6898656898656899, micro F1: 0.7051282051282057, macro F1: 0.7128808839672827 [INFO] Relation Linking P: 0.5957468457468457, R: 0.5421245421245421, micro F1: 0.5574277574277583, macro F1: 0.567672215709415 [INFO] Type Linking P: 0.5592185592185592, R: 0.5592185592185592, micro F1: 0.5592185592185592, macro F1: 0.5592185592185592 @@ -11497,8 +11497,8 @@ Question 819: Which sport at Polytechnic University of the Philippines Taguig is [INFO] Golden Answer: [http://dbpedia.org/resource/Chess] [INFO] Predicted Answer: [http://dbpedia.org/resource/Chess] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 820, P: 0.500, R: 0.554, macro F1: 0.507, macro F1*: 0.525 -[INFO] Current time: 3356 ms, average: 2929.55 ms/ques, entity detection: 500.71 ms/ques (linking tool: 407.34 ms/ques), relation detection: 950.86 ms/ques (similarity: 715.70 ms/ques), EDG generation: 46.65 ms/ques +[INFO] Cumulative metrics, sample: 820, P: 0.500, R: 0.554, macro F1: 0.525 +[INFO] Current question consumed time: 2802 ms, average: 2629.45 ms/ques, entity detection: 440.15 ms/ques (linking tool: 361.96 ms/ques), relation detection: 875.31 ms/ques (similarity: 708.41 ms/ques), EDG generation: 46.99 ms/ques [INFO] Entity Linking P: 0.7378048780487805, R: 0.6902439024390243, micro F1: 0.7054878048780493, macro F1: 0.713232384245277 [INFO] Relation Linking P: 0.5962398373983739, R: 0.5426829268292683, micro F1: 0.5579674796747977, macro F1: 0.568202147177178 [INFO] Type Linking P: 0.5597560975609757, R: 0.5597560975609757, micro F1: 0.5597560975609757, macro F1: 0.5597560975609757 @@ -11511,8 +11511,8 @@ Question 820: Count the number of things people who know linguistics are known f [INFO] Golden Answer: [35] [INFO] Predicted Answer: [] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 821, P: 0.499, R: 0.553, macro F1: 0.507, macro F1*: 0.525 -[INFO] Current time: 652 ms, average: 2926.78 ms/ques, entity detection: 500.29 ms/ques (linking tool: 406.98 ms/ques), relation detection: 949.70 ms/ques (similarity: 714.83 ms/ques), EDG generation: 46.62 ms/ques +[INFO] Cumulative metrics, sample: 821, P: 0.499, R: 0.553, macro F1: 0.525 +[INFO] Current question consumed time: 575 ms, average: 2626.95 ms/ques, entity detection: 439.79 ms/ques (linking tool: 361.64 ms/ques), relation detection: 874.25 ms/ques (similarity: 707.55 ms/ques), EDG generation: 46.96 ms/ques [INFO] Entity Linking P: 0.7369062119366626, R: 0.6894031668696712, micro F1: 0.7046285018270407, macro F1: 0.712363648089071 [INFO] Relation Linking P: 0.5955136012992286, R: 0.5420219244823387, micro F1: 0.5572878603329282, macro F1: 0.5675100617360365 [INFO] Type Linking P: 0.5602923264311814, R: 0.5602923264311814, micro F1: 0.5602923264311814, macro F1: 0.5602923264311814 @@ -11525,8 +11525,8 @@ Question 821: Where were sverre krogh sundbo and havard vad petersson born? [INFO] Golden Answer: [http://dbpedia.org/resource/Oslo] [INFO] Predicted Answer: [http://dbpedia.org/resource/Oslo] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 822, P: 0.500, R: 0.554, macro F1: 0.507, macro F1*: 0.525 -[INFO] Current time: 1928 ms, average: 2925.56 ms/ques, entity detection: 499.97 ms/ques (linking tool: 406.71 ms/ques), relation detection: 949.25 ms/ques (similarity: 714.36 ms/ques), EDG generation: 46.58 ms/ques +[INFO] Cumulative metrics, sample: 822, P: 0.500, R: 0.554, macro F1: 0.525 +[INFO] Current question consumed time: 1705 ms, average: 2625.83 ms/ques, entity detection: 439.50 ms/ques (linking tool: 361.40 ms/ques), relation detection: 873.83 ms/ques (similarity: 707.09 ms/ques), EDG generation: 46.92 ms/ques [INFO] Entity Linking P: 0.7372262773722628, R: 0.6897810218978102, micro F1: 0.7049878345498789, macro F1: 0.7127149177665355 [INFO] Relation Linking P: 0.5960056772100567, R: 0.541970802919708, micro F1: 0.5574209245742101, macro F1: 0.5677053630939801 [INFO] Type Linking P: 0.5608272506082725, R: 0.5608272506082725, micro F1: 0.5608272506082725, macro F1: 0.5608272506082725 @@ -11539,8 +11539,8 @@ Question 822: Was Cliff Burton in Metallica? [INFO] Golden Answer: [true] [INFO] Predicted Answer: [false] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 823, P: 0.499, R: 0.553, macro F1: 0.507, macro F1*: 0.525 -[INFO] Current time: 1004 ms, average: 2923.23 ms/ques, entity detection: 499.89 ms/ques (linking tool: 406.58 ms/ques), relation detection: 948.09 ms/ques (similarity: 713.49 ms/ques), EDG generation: 46.53 ms/ques +[INFO] Cumulative metrics, sample: 823, P: 0.499, R: 0.553, macro F1: 0.525 +[INFO] Current question consumed time: 859 ms, average: 2623.68 ms/ques, entity detection: 439.43 ms/ques (linking tool: 361.28 ms/ques), relation detection: 872.77 ms/ques (similarity: 706.23 ms/ques), EDG generation: 46.87 ms/ques [INFO] Entity Linking P: 0.7363304981773997, R: 0.6889428918590522, micro F1: 0.7041312272174975, macro F1: 0.711848921511655 [INFO] Relation Linking P: 0.5952814904819764, R: 0.5413122721749696, micro F1: 0.5567436208991503, macro F1: 0.5670155631388233 [INFO] Type Linking P: 0.5613608748481167, R: 0.5613608748481167, micro F1: 0.5613608748481167, macro F1: 0.5613608748481167 @@ -11553,8 +11553,8 @@ Question 823: How many religions are followed by organizations which are interna [INFO] Golden Answer: [3] [INFO] Predicted Answer: [3, 2] [INFO] P: 0.5, R: 1.0, F: 0.6666666666666666 -[INFO] Cumulative metrics, sample: 824, P: 0.499, R: 0.553, macro F1: 0.507, macro F1*: 0.525 -[INFO] Current time: 5580 ms, average: 2926.45 ms/ques, entity detection: 499.64 ms/ques (linking tool: 406.37 ms/ques), relation detection: 949.17 ms/ques (similarity: 714.90 ms/ques), EDG generation: 46.52 ms/ques +[INFO] Cumulative metrics, sample: 824, P: 0.499, R: 0.553, macro F1: 0.525 +[INFO] Current question consumed time: 4850 ms, average: 2626.38 ms/ques, entity detection: 439.21 ms/ques (linking tool: 361.09 ms/ques), relation detection: 873.72 ms/ques (similarity: 707.65 ms/ques), EDG generation: 46.93 ms/ques [INFO] Entity Linking P: 0.7366504854368932, R: 0.6893203883495146, micro F1: 0.7044902912621365, macro F1: 0.7121999586862219 [INFO] Relation Linking P: 0.5957726537216828, R: 0.5418689320388349, micro F1: 0.5572815533980592, macro F1: 0.5675437600925907 [INFO] Type Linking P: 0.5606796116504854, R: 0.5606796116504854, micro F1: 0.5606796116504854, macro F1: 0.5606796116504854 @@ -11567,8 +11567,8 @@ Question 824: What are some people born in NY known for? [INFO] Golden Answer: [http://dbpedia.org/resource/Startup_company, http://dbpedia.org/resource/It's_a_Mad,_Mad,_Mad,_Mad_World, http://dbpedia.org/resource/Top_Cat, http://dbpedia.org/resource/Texaco_Star_Theatre, http://dbpedia.org/resource/MonsterTalk, http://dbpedia.org/resource/Artificial_neural_network, http://dbpedia.org/resource/Competition_model, http://dbpedia.org/resource/CHILDES, http://dbpedia.org/resource/Monkey_Business_(1931_film), http://dbpedia.org/resource/Duck_Soup_(1933_film), http://dbpedia.org/resource/Good_Times, http://dbpedia.org/resource/Frozen_food, http://dbpedia.org/resource/Enzyme, http://dbpedia.org/resource/Law_&_Order_(franchise), http://dbpedia.org/resource/Chicago_(franchise), http://dbpedia.org/resource/1950s_quiz_show_scandals, http://dbpedia.org/resource/Electrospray_ionization, http://dbpedia.org/resource/Catecholamine, http://dbpedia.org/resource/Field_mark, http://dbpedia.org/resource/What's_My_Name%3F_(radio_program), http://dbpedia.org/resource/Dastardly_and_Muttley_in_Their_Flying_Machines, http://dbpedia.org/resource/The_Smurfs_(TV_series), http://dbpedia.org/resource/Wacky_Races, http://dbpedia.org/resource/The_Paul_Winchell_Show, http://dbpedia.org/resource/Winnie_the_Pooh_(franchise), http://dbpedia.org/resource/Evolutionary_biology, http://dbpedia.org/resource/Population_genetics, http://dbpedia.org/resource/Pundit, http://dbpedia.org/resource/Small-world_experiment, http://dbpedia.org/resource/Milgram_experiment, http://dbpedia.org/resource/Familiar_stranger, http://dbpedia.org/resource/Crohn's_disease, http://dbpedia.org/resource/Columbia_University, http://dbpedia.org/resource/Students_for_a_Democratic_Society, http://dbpedia.org/resource/Weather_Underground, http://dbpedia.org/resource/Baywatch, http://dbpedia.org/resource/Internet_protocol_suite, http://dbpedia.org/resource/Scientometrics, http://dbpedia.org/resource/Institute_for_Scientific_Information, http://dbpedia.org/resource/Science_Citation_Index, http://dbpedia.org/resource/Stone–Čech_compactification, http://dbpedia.org/resource/Stone–Weierstrass_theorem, http://dbpedia.org/resource/Stone–von_Neumann_theorem, http://dbpedia.org/resource/Information_economy, http://dbpedia.org/resource/New_York_University, http://dbpedia.org/resource/Chief_technology_officer, http://dbpedia.org/resource/Entrepreneurship, http://dbpedia.org/resource/Radioimmunoassay, http://dbpedia.org/resource/Quantum_optics, http://dbpedia.org/resource/Virtual_reality, http://dbpedia.org/resource/The_Facts_of_Life_(TV_series), http://dbpedia.org/resource/The_Passing_of_the_Great_Race, http://dbpedia.org/resource/Eugenics, http://dbpedia.org/resource/Scientific_racism, http://dbpedia.org/resource/Old_Chinese, http://dbpedia.org/resource/Neurology, http://dbpedia.org/resource/Quine–McCluskey_algorithm, http://dbpedia.org/resource/Space_advocacy, http://dbpedia.org/resource/2001:_A_Space_Odyssey_(film), http://dbpedia.org/resource/Germany_Must_Perish!, http://dbpedia.org/resource/Central_Park, http://dbpedia.org/resource/Jupiter's_Wife_(film), http://dbpedia.org/resource/Erotic_photography, http://dbpedia.org/resource/Lehman_Brothers, http://dbpedia.org/resource/Lexicography, http://dbpedia.org/resource/Hemiptera, http://dbpedia.org/resource/Entomology, http://dbpedia.org/resource/Gender_studies, http://dbpedia.org/resource/Masculinity, http://dbpedia.org/resource/Men's_studies, http://dbpedia.org/resource/Men_and_feminism, http://dbpedia.org/resource/ADM_formalism, http://dbpedia.org/resource/Supergravity, http://dbpedia.org/resource/Leaves_of_Grass_(film), http://dbpedia.org/resource/Deadbeat_(TV_series), http://dbpedia.org/resource/Calvin_Klein, http://dbpedia.org/resource/Landing_Vehicle_Tracked, http://dbpedia.org/resource/Diary, http://dbpedia.org/resource/Pulitzer_Prize, http://dbpedia.org/resource/Apollo_Lunar_Module, http://dbpedia.org/resource/Grumman_F4F_Wildcat, http://dbpedia.org/resource/Grumman_F6F_Hellcat, http://dbpedia.org/resource/Belcourt_of_Newport, http://dbpedia.org/resource/Léon:_The_Professional, http://dbpedia.org/resource/Decision_tree_learning, http://dbpedia.org/resource/Bootstrap_aggregating, http://dbpedia.org/resource/Random_forest, http://dbpedia.org/resource/Universal_Studios, http://dbpedia.org/resource/Decca_Records, http://dbpedia.org/resource/Mantel_test, http://dbpedia.org/resource/Napoleon_III, http://dbpedia.org/resource/Hospice, http://dbpedia.org/resource/Pacific_Southwest_Airlines, http://dbpedia.org/resource/Hellman_&_Friedman, http://dbpedia.org/resource/Matrix_Partners, http://dbpedia.org/resource/The_New_Republic, http://dbpedia.org/resource/Paul_Anka, http://dbpedia.org/resource/Jason_Bateman, http://dbpedia.org/resource/Chester_A._Arthur, http://dbpedia.org/resource/Maynard_Institute, http://dbpedia.org/resource/Temperature_gradient_gel_electrophoresis, http://dbpedia.org/resource/Target_Corporation, http://dbpedia.org/resource/Film_criticism, http://dbpedia.org/resource/Shapiro–Francia_test, http://dbpedia.org/resource/Shapiro–Wilk_test, http://dbpedia.org/resource/McMurry_reaction, http://dbpedia.org/resource/Warnaco_Group, http://dbpedia.org/resource/Taxon, http://dbpedia.org/resource/National_Collegiate_Athletic_Association, http://dbpedia.org/resource/Southwest_Conference, http://dbpedia.org/resource/Southern_Methodist_University_football_scandal, http://dbpedia.org/resource/New_York_Eye_and_Ear_Infirmary, http://dbpedia.org/resource/Printmaking, http://dbpedia.org/resource/Sculpture, http://dbpedia.org/resource/Drawing, http://dbpedia.org/resource/Bioethics, http://dbpedia.org/resource/Multiple_sclerosis_research, http://dbpedia.org/resource/Software_engineering, http://dbpedia.org/resource/The_American_Hebrew, http://dbpedia.org/resource/Cryptography, http://dbpedia.org/resource/Security, http://dbpedia.org/resource/Muon, http://dbpedia.org/resource/Positron, http://dbpedia.org/resource/1971_Los_Angeles_Rams_season, http://dbpedia.org/resource/Post-industrial_society, http://dbpedia.org/resource/Retina, http://dbpedia.org/resource/Apple_Inc., http://dbpedia.org/resource/National_Semiconductor, http://dbpedia.org/resource/Celebrity, http://dbpedia.org/resource/Runway_(fashion), http://dbpedia.org/resource/Born–Oppenheimer_approximation, http://dbpedia.org/resource/Nuclear_weapon, http://dbpedia.org/resource/Oppenheimer–Phillips_process, http://dbpedia.org/resource/Tolman–Oppenheimer–Volkoff_limit, http://dbpedia.org/resource/Taser, http://dbpedia.org/resource/Human–computer_interaction, http://dbpedia.org/resource/The_Basketball_Diaries, http://dbpedia.org/resource/Polio_vaccine, http://dbpedia.org/resource/Favrile_glass, http://dbpedia.org/resource/Tiffany_lamp, http://dbpedia.org/resource/Hilbert's_tenth_problem, http://dbpedia.org/resource/Davis–Putnam_algorithm, http://dbpedia.org/resource/DPLL_algorithm, http://dbpedia.org/resource/Perceptrons_(book), http://dbpedia.org/resource/Triadex_Muse, http://dbpedia.org/resource/Artificial_intelligence, http://dbpedia.org/resource/The_Emotion_Machine, http://dbpedia.org/resource/Transhumanism, http://dbpedia.org/resource/Society_of_Mind, http://dbpedia.org/resource/Confocal_microscopy, http://dbpedia.org/resource/Useless_machine, http://dbpedia.org/resource/Frame_(artificial_intelligence), http://dbpedia.org/resource/The_Man_Who_Came_to_Dinner, http://dbpedia.org/resource/Warhol_superstars, http://dbpedia.org/resource/Arrestin, http://dbpedia.org/resource/G_protein–coupled_receptor, http://dbpedia.org/resource/Julius_and_Ethel_Rosenberg, http://dbpedia.org/resource/Joseph_McCarthy, http://dbpedia.org/resource/DDB_Worldwide, http://dbpedia.org/resource/Underboss, http://dbpedia.org/resource/Bonanno_crime_family, http://dbpedia.org/resource/Association_football, http://dbpedia.org/resource/Informant, http://dbpedia.org/resource/High_School_Musical, http://dbpedia.org/resource/Outliner, http://dbpedia.org/resource/RSS, http://dbpedia.org/resource/Blog, http://dbpedia.org/resource/Podcast, http://dbpedia.org/resource/Coherent_diffraction_imaging, http://dbpedia.org/resource/Fortran, http://dbpedia.org/resource/X-ray_microscope, http://dbpedia.org/resource/Sayre_equation, http://dbpedia.org/resource/Vanderbilt_family, http://dbpedia.org/resource/Journal_of_the_American_Chemical_Society, http://dbpedia.org/resource/Saccharin, http://dbpedia.org/resource/Exponential_polynomial, http://dbpedia.org/resource/Jewish-American_organized_crime, http://dbpedia.org/resource/Assassination_of_Martin_Luther_King,_Jr., http://dbpedia.org/resource/O._J._Simpson_murder_case, http://dbpedia.org/resource/Heathenry_(new_religious_movement), http://dbpedia.org/resource/Gay_liberation, http://dbpedia.org/resource/Transgender, http://dbpedia.org/resource/J/psi_meson, http://dbpedia.org/resource/Survivor:_Cook_Islands, http://dbpedia.org/resource/Survivor:_Philippines, http://dbpedia.org/resource/Survivor:_Micronesia, http://dbpedia.org/resource/Chi-squared_target_models, http://dbpedia.org/resource/Gambino_crime_family, http://dbpedia.org/resource/Paramount_on_Parade, http://dbpedia.org/resource/Kid_Boots_(film), http://dbpedia.org/resource/This_Gun_for_Hire, http://dbpedia.org/resource/The_Glass_Key_(1935_film), http://dbpedia.org/resource/The_Untamed_Lady, http://dbpedia.org/resource/Roman_Scandals, http://dbpedia.org/resource/I_Stole_a_Million, http://dbpedia.org/resource/College_Holiday, http://dbpedia.org/resource/Quantum_cryptography, http://dbpedia.org/resource/Flynn's_taxonomy, http://dbpedia.org/resource/New_York_City_Police_Department, http://dbpedia.org/resource/National_Lawyers_Guild, http://dbpedia.org/resource/Primary_aldosteronism, http://dbpedia.org/resource/Ware_Group, http://dbpedia.org/resource/General_Theological_Seminary, http://dbpedia.org/resource/Arc_lamp, http://dbpedia.org/resource/Mercury-arc_valve, http://dbpedia.org/resource/New_York_City_Fire_Department, http://dbpedia.org/resource/Dark_Shadows, http://dbpedia.org/resource/Emmy_Award, http://dbpedia.org/resource/Pulitzer_Prize_for_Public_Service, http://dbpedia.org/resource/New_York_City_Police_Commissioner, http://dbpedia.org/resource/Strong's_Concordance, http://dbpedia.org/resource/Advocacy_planning, http://dbpedia.org/resource/Economic_methodology, http://dbpedia.org/resource/History_of_economic_thought, http://dbpedia.org/resource/Chess, http://dbpedia.org/resource/Linguistics, http://dbpedia.org/resource/Metropolitan_Museum_of_Art, http://dbpedia.org/resource/Philanthropy, http://dbpedia.org/resource/Private_collection, http://dbpedia.org/resource/Henry_E._Steinway, http://dbpedia.org/resource/Steinway_&_Sons, http://dbpedia.org/resource/Communism, http://dbpedia.org/resource/The_Barclays, http://dbpedia.org/resource/New_York_Rangers, http://dbpedia.org/resource/Screenwriter, http://dbpedia.org/resource/Ku_Klux_Klan, http://dbpedia.org/resource/American_Sugar_Refining_Company, http://dbpedia.org/resource/Knight_Capital_Group, http://dbpedia.org/resource/Parental_alienation_syndrome, http://dbpedia.org/resource/Professor_of_Jurisprudence_(University_of_Oxford), http://dbpedia.org/resource/Pulitzer_Prize_for_Commentary, http://dbpedia.org/resource/Dora_the_Explorer, http://dbpedia.org/resource/Skidmore_College, http://dbpedia.org/resource/NCIS:_Los_Angeles, http://dbpedia.org/resource/List_of_NCIS:_Los_Angeles_characters, http://dbpedia.org/resource/Chauffeur, http://dbpedia.org/resource/Joseph_C._Amberg, http://dbpedia.org/resource/Malcolm_in_the_Middle, http://dbpedia.org/resource/Poor_Butterfly, http://dbpedia.org/resource/University_Challenge, http://dbpedia.org/resource/Emmanuel_College,_Cambridge, http://dbpedia.org/resource/Floyd–Warshall_algorithm, http://dbpedia.org/resource/Special_values_of_L-functions, http://dbpedia.org/resource/Wicked_(musical), http://dbpedia.org/resource/My_So-Called_Life, http://dbpedia.org/resource/Chronic_disease, http://dbpedia.org/resource/Chronic_condition, http://dbpedia.org/resource/MOO, http://dbpedia.org/resource/MOOSE_Crossing, http://dbpedia.org/resource/Fomalhaut, http://dbpedia.org/resource/Exoplanet, http://dbpedia.org/resource/Fomalhaut_b, http://dbpedia.org/resource/Facebook, http://dbpedia.org/resource/Vice_president, http://dbpedia.org/resource/Atomic_spies, http://dbpedia.org/resource/Grey_Gardens, http://dbpedia.org/resource/Humpty_Jackson, http://dbpedia.org/resource/Industrial_relations, http://dbpedia.org/resource/Group_dynamics, http://dbpedia.org/resource/T-groups, http://dbpedia.org/resource/She-She-She_Camps, http://dbpedia.org/resource/Bryn_Mawr_Summer_School_for_Women_Workers_in_Industry, http://dbpedia.org/resource/Worker_Education, http://dbpedia.org/resource/Prism_Skylabs, http://dbpedia.org/resource/Minor_Ventures, http://dbpedia.org/resource/University_of_Florida, http://dbpedia.org/resource/Myxobacteria, http://dbpedia.org/resource/Hologenome_theory_of_evolution, http://dbpedia.org/resource/Global_warming, http://dbpedia.org/resource/Marquess, http://dbpedia.org/resource/Luxemborg_Museum, http://dbpedia.org/resource/Heart_transplantation, http://dbpedia.org/resource/Ventricular_assist_device, http://dbpedia.org/resource/Amaryllidaceae, http://dbpedia.org/resource/Baseball, http://dbpedia.org/resource/Born_Free, http://dbpedia.org/resource/Star_Trek:_The_Next_Generation, http://dbpedia.org/resource/L.A._Law, http://dbpedia.org/resource/McCloud_(TV_series), http://dbpedia.org/resource/60_Minutes, http://dbpedia.org/resource/FM_broadcasting, http://dbpedia.org/resource/McCarthyism, http://dbpedia.org/resource/Exploratorium, http://dbpedia.org/resource/Enriched_uranium, http://dbpedia.org/resource/MARC_standards, http://dbpedia.org/resource/CBGB, http://dbpedia.org/resource/Asymptotic_freedom, http://dbpedia.org/resource/Quantum_chromodynamics, http://dbpedia.org/resource/World-System_Theory, http://dbpedia.org/resource/The_Jeffersons, http://dbpedia.org/resource/Louise_Jefferson, http://dbpedia.org/resource/Robert_De_Niro, http://dbpedia.org/resource/Goodfellas, http://dbpedia.org/resource/Lufthansa_heist, http://dbpedia.org/resource/Instructional_scaffolding, http://dbpedia.org/resource/Cognitive_psychology, http://dbpedia.org/resource/Educational_psychology, http://dbpedia.org/resource/The_Complete_Book_of_Running, http://dbpedia.org/resource/Neutrino, http://dbpedia.org/resource/GNU_Compiler_Collection, http://dbpedia.org/resource/GNU, http://dbpedia.org/resource/Free_software_movement, http://dbpedia.org/resource/Emacs, http://dbpedia.org/resource/Radiation_therapy, http://dbpedia.org/resource/Clash_of_Civilizations, http://dbpedia.org/resource/Belly_dance, http://dbpedia.org/resource/United_States, http://dbpedia.org/resource/Apollo_program, http://dbpedia.org/resource/C++, http://dbpedia.org/resource/Microkernel, http://dbpedia.org/resource/Electoral-vote.com, http://dbpedia.org/resource/MINIX, http://dbpedia.org/resource/Information_visualization, http://dbpedia.org/resource/Touchscreen, http://dbpedia.org/resource/Hyperlink, http://dbpedia.org/resource/Direct_manipulation_interface, http://dbpedia.org/resource/Treemapping, http://dbpedia.org/resource/Nassi–Shneiderman_diagram, http://dbpedia.org/resource/Big_History, http://dbpedia.org/resource/Differential_geometry, http://dbpedia.org/resource/Calculus_of_variations, http://dbpedia.org/resource/Schwinger's_quantum_action_principle, http://dbpedia.org/resource/Schwinger_model, http://dbpedia.org/resource/Schwinger–Dyson_equation, http://dbpedia.org/resource/Quantum_electrodynamics, http://dbpedia.org/resource/Lippmann–Schwinger_equation, http://dbpedia.org/resource/Schwinger_function, http://dbpedia.org/resource/Schwinger_parametrization, http://dbpedia.org/resource/Schwinger_variational_principle, http://dbpedia.org/resource/Spin–statistics_theorem, http://dbpedia.org/resource/Schwinger_limit, http://dbpedia.org/resource/Rarita–Schwinger_equation, http://dbpedia.org/resource/Internet, http://dbpedia.org/resource/Paxos_(computer_science), http://dbpedia.org/resource/LaTeX, http://dbpedia.org/resource/Byzantine_fault_tolerance, http://dbpedia.org/resource/Lamport's_bakery_algorithm, http://dbpedia.org/resource/Sequential_consistency, http://dbpedia.org/resource/Lamport_signature, http://dbpedia.org/resource/The_New_Yorker, http://dbpedia.org/resource/Space_tourism, http://dbpedia.org/resource/Abiogenesis, http://dbpedia.org/resource/Greibach's_theorem, http://dbpedia.org/resource/Greibach_normal_form, http://dbpedia.org/resource/Microsoft, http://dbpedia.org/resource/Abu_Ghraib, http://dbpedia.org/resource/Stanford_prison_experiment, http://dbpedia.org/resource/Black–Scholes_model, http://dbpedia.org/resource/Long-Term_Capital_Management, http://dbpedia.org/resource/Intertemporal_CAPM, http://dbpedia.org/resource/Jarrow–Turnbull_model, http://dbpedia.org/resource/Merton's_portfolio_problem, http://dbpedia.org/resource/Merton_model, http://dbpedia.org/resource/Oral_rehydration_therapy, http://dbpedia.org/resource/Typhus, http://dbpedia.org/resource/The_Moral_Obligation_to_Be_Intelligent, http://dbpedia.org/resource/Great_books, http://dbpedia.org/resource/Bellman–Ford_algorithm, http://dbpedia.org/resource/Bellman_equation, http://dbpedia.org/resource/Dynamic_programming, http://dbpedia.org/resource/Curse_of_dimensionality, http://dbpedia.org/resource/Hamilton–Jacobi–Bellman_equation, http://dbpedia.org/resource/Change_of_Mind, http://dbpedia.org/resource/Alfred_Hitchcock_Presents, http://dbpedia.org/resource/Suspense_(U.S._TV_series), http://dbpedia.org/resource/Mission:_Impossible, http://dbpedia.org/resource/Alien_abduction, http://dbpedia.org/resource/Room_Service_(play), http://dbpedia.org/resource/Judge_Judy, http://dbpedia.org/resource/Portland_Trail_Blazers, http://dbpedia.org/resource/Shapiro_delay, http://dbpedia.org/resource/Surgery_theory, http://dbpedia.org/resource/The_House_I_Live_In_(1945_film), http://dbpedia.org/resource/Strange_Fruit, http://dbpedia.org/resource/Minnesota_North_Stars, http://dbpedia.org/resource/Psychiatry, http://dbpedia.org/resource/World_War_I, http://dbpedia.org/resource/Psycho-Cybernetics, http://dbpedia.org/resource/Diffusion_process, http://dbpedia.org/resource/Malliavin_calculus, http://dbpedia.org/resource/Kickstarter, http://dbpedia.org/resource/Miguel,_Duke_of_Braganza, http://dbpedia.org/resource/Isentropic_analysis, http://dbpedia.org/resource/Television_program, http://dbpedia.org/resource/Evans_Solar_Facility, http://dbpedia.org/resource/Days_of_Our_Lives, http://dbpedia.org/resource/New_York_City, http://dbpedia.org/resource/Frank_Buck_(animal_collector), http://dbpedia.org/resource/Pepperidge_Farm, http://dbpedia.org/resource/Dutch_Mob, http://dbpedia.org/resource/American_Fine_Arts_Society, http://dbpedia.org/resource/Destination_Truth, http://dbpedia.org/resource/Jim_Rome_Is_Burning, http://dbpedia.org/resource/Last_Stop_for_Paul, http://dbpedia.org/resource/Barry_Bonds, http://dbpedia.org/resource/Barry_Sanders, http://dbpedia.org/resource/Wikipedia, http://dbpedia.org/resource/Atlantic_Rim_(film), http://dbpedia.org/resource/Buddy_Hutchins, http://dbpedia.org/resource/Born_Bad, http://dbpedia.org/resource/Clinical_research, http://dbpedia.org/resource/Writing, http://dbpedia.org/resource/Agent_Orange, http://dbpedia.org/resource/Schoenflies_problem, http://dbpedia.org/resource/Diophantine_geometry, http://dbpedia.org/resource/Eilenberg–Mazur_swindle, http://dbpedia.org/resource/Torsion_conjecture, http://dbpedia.org/resource/Ames_test, http://dbpedia.org/resource/Trent_Affair, http://dbpedia.org/resource/United_States_Exploring_Expedition, http://dbpedia.org/resource/Information_Please, http://dbpedia.org/resource/Roc-A-Fella_Records, http://dbpedia.org/resource/Hofstadter's_butterfly, http://dbpedia.org/resource/I_Am_a_Strange_Loop, http://dbpedia.org/resource/Gödel,_Escher,_Bach, http://dbpedia.org/resource/Belles_on_Their_Toes, http://dbpedia.org/resource/Cheaper_by_the_Dozen, http://dbpedia.org/resource/Radiation, http://dbpedia.org/resource/Dinosaur_renaissance, http://dbpedia.org/resource/Valachi_hearings, http://dbpedia.org/resource/Committee_for_Skeptical_Inquiry, http://dbpedia.org/resource/String_theory, http://dbpedia.org/resource/Color_confinement, http://dbpedia.org/resource/String_theory_landscape, http://dbpedia.org/resource/Hamiltonian_lattice_gauge_theory, http://dbpedia.org/resource/Holographic_principle, http://dbpedia.org/resource/Antibody, http://dbpedia.org/resource/World_War_II, http://dbpedia.org/resource/Baltimore_classification, http://dbpedia.org/resource/Reverse_transcriptase, http://dbpedia.org/resource/Algebraic_topology, http://dbpedia.org/resource/Differential_topology, http://dbpedia.org/resource/Graph_theory, http://dbpedia.org/resource/Singularity_theory, http://dbpedia.org/resource/Geometric_measure_theory, http://dbpedia.org/resource/Place_cell, http://dbpedia.org/resource/Phase_transition, http://dbpedia.org/resource/Climatic_Change_(journal), http://dbpedia.org/resource/Occidental_Petroleum, http://dbpedia.org/resource/The_Wallace_and_Ladmo_Show, http://dbpedia.org/resource/Kach_and_Kahane_Chai, http://dbpedia.org/resource/Yale_University_Press, http://dbpedia.org/resource/Life_with_Father, http://dbpedia.org/resource/Life_zone, http://dbpedia.org/resource/Kasner_metric, http://dbpedia.org/resource/Googol, http://dbpedia.org/resource/Midpoint_polygon, http://dbpedia.org/resource/Piano_concerto, http://dbpedia.org/resource/The_Good,_the_Bad_and_the_Ugly, http://dbpedia.org/resource/Baby_Doll, http://dbpedia.org/resource/Don_Altobello, http://dbpedia.org/resource/The_Two_Jakes, http://dbpedia.org/resource/Mr._Freeze, http://dbpedia.org/resource/The_Magnificent_Seven, http://dbpedia.org/resource/The_Holiday, http://dbpedia.org/resource/Mattachine_Society, http://dbpedia.org/resource/Gay, http://dbpedia.org/resource/Economic_growth, http://dbpedia.org/resource/Political_economy, http://dbpedia.org/resource/International_trade, http://dbpedia.org/resource/Cancer, http://dbpedia.org/resource/Chromosomal_translocation, http://dbpedia.org/resource/Columbia_Records, http://dbpedia.org/resource/Mental_health, http://dbpedia.org/resource/United_States_constitutional_law, http://dbpedia.org/resource/Medical_jurisprudence, http://dbpedia.org/resource/Woodstock, http://dbpedia.org/resource/American_Hardcore, http://dbpedia.org/resource/Four_Dogs_Playing_Poker, http://dbpedia.org/resource/Cycle_detection, http://dbpedia.org/resource/Floyd–Steinberg_dithering, http://dbpedia.org/resource/Georgi–Glashow_model, http://dbpedia.org/resource/Electroweak_interaction, http://dbpedia.org/resource/Superstring_theory, http://dbpedia.org/resource/Fiction, http://dbpedia.org/resource/Parents_(magazine), http://dbpedia.org/resource/Better_Homes_and_Gardens_(magazine), http://dbpedia.org/resource/McCall's, http://dbpedia.org/resource/Vogue_(magazine), http://dbpedia.org/resource/Good_Housekeeping, http://dbpedia.org/resource/Bottom_quark, http://dbpedia.org/resource/Belair_Mansion_(Bowie,_Maryland), http://dbpedia.org/resource/Belair_Stud, http://dbpedia.org/resource/Journey_of_Reconciliation, http://dbpedia.org/resource/Committee_for_Nonviolent_Revolution, http://dbpedia.org/resource/War_Resisters_League, http://dbpedia.org/resource/OPLS, http://dbpedia.org/resource/The_Wrong_Way_Home, http://dbpedia.org/resource/Clinical_study_design, http://dbpedia.org/resource/Zelen's_design, http://dbpedia.org/resource/Collier_Trophy, http://dbpedia.org/resource/Author, http://dbpedia.org/resource/Penicillin, http://dbpedia.org/resource/Spectral_method, http://dbpedia.org/resource/Chaos_theory, http://dbpedia.org/resource/Functional_analysis, http://dbpedia.org/resource/General_relativity, http://dbpedia.org/resource/Group_theory, http://dbpedia.org/resource/Particle_physics, http://dbpedia.org/resource/Quantum_chaos, http://dbpedia.org/resource/Statistical_mechanics, http://dbpedia.org/resource/Mathematical_physics, http://dbpedia.org/resource/Quantum_field_theory, http://dbpedia.org/resource/Relativistic_quantum_mechanics, http://dbpedia.org/resource/Accessible_surface_area, http://dbpedia.org/resource/Bovine_pancreatic_ribonuclease, http://dbpedia.org/resource/Social_penetration_theory, http://dbpedia.org/resource/Politician, http://dbpedia.org/resource/Journalist, http://dbpedia.org/resource/Hybrid_vehicle, http://dbpedia.org/resource/Electric_vehicle, http://dbpedia.org/resource/Karnaugh_map, http://dbpedia.org/resource/Ordered_weighted_averaging_aggregation_operator, http://dbpedia.org/resource/Fuzzy_set, http://dbpedia.org/resource/Shoulder_surgery, http://dbpedia.org/resource/DNA_repair, http://dbpedia.org/resource/Blue's_Clues, http://dbpedia.org/resource/Factor_XII, http://dbpedia.org/resource/Coagulation, http://dbpedia.org/resource/Mr._Terrific_(TV_series), http://dbpedia.org/resource/Miami_Herald, http://dbpedia.org/resource/Playboy_Cyber_Club, http://dbpedia.org/resource/Playboy, http://dbpedia.org/resource/Bluebird_Records, http://dbpedia.org/resource/Abhyankar's_conjecture, http://dbpedia.org/resource/Organic_horticulture, http://dbpedia.org/resource/Period_(gene), http://dbpedia.org/resource/Franklin_Simon_&_Co., http://dbpedia.org/resource/Cobb_Angle, http://dbpedia.org/resource/UNICEF, http://dbpedia.org/resource/RapidSMS, http://dbpedia.org/resource/Publish–subscribe_pattern, http://dbpedia.org/resource/Business_process_discovery, http://dbpedia.org/resource/Content_centric_networking, http://dbpedia.org/resource/Software_architecture, http://dbpedia.org/resource/Software_deployment, http://dbpedia.org/resource/Forks_Over_Knives, http://dbpedia.org/resource/Alcoholism, http://dbpedia.org/resource/Shoe_(comic_strip), http://dbpedia.org/resource/Shor's_algorithm, http://dbpedia.org/resource/Blum–Goldwasser_cryptosystem, http://dbpedia.org/resource/Goldwasser–Micali_cryptosystem, http://dbpedia.org/resource/Albinism, http://dbpedia.org/resource/Holocaust_denial, http://dbpedia.org/resource/Hilbert_curve, http://dbpedia.org/resource/Happy_Days, http://dbpedia.org/resource/Galactic_center, http://dbpedia.org/resource/Galactic_Center, http://dbpedia.org/resource/Adaptive_optics, http://dbpedia.org/resource/Dot-com_bubble, http://dbpedia.org/resource/The_Wilderness_Society_(United_States), http://dbpedia.org/resource/Clarence_Royce, http://dbpedia.org/resource/Cooley_High, http://dbpedia.org/resource/A_Different_World, http://dbpedia.org/resource/Folger_Shakespeare_Library, http://dbpedia.org/resource/Gentleman's_Agreement_(novel), http://dbpedia.org/resource/Tau_(particle), http://dbpedia.org/resource/John_Lennon, http://dbpedia.org/resource/Whale, http://dbpedia.org/resource/The_Twilight_Saga:_New_Moon, http://dbpedia.org/resource/American_Pie_(film), http://dbpedia.org/resource/List_of_Disney's_Cinderella_characters, http://dbpedia.org/resource/Maleficent, http://dbpedia.org/resource/Van_de_Graaff_generator, http://dbpedia.org/resource/Electron_beam_processing, http://dbpedia.org/resource/Agent-based_model, http://dbpedia.org/resource/Cognitive_dissonance, http://dbpedia.org/resource/Social_comparison_theory, http://dbpedia.org/resource/Publishing, http://dbpedia.org/resource/Apax_Partners, http://dbpedia.org/resource/Baupost_Group, http://dbpedia.org/resource/United_States_Under_Secretary_of_the_Air_Force, http://dbpedia.org/resource/Applied_Physics_Letters, http://dbpedia.org/resource/Cosmic_ray, http://dbpedia.org/resource/Mössbauer_effect, http://dbpedia.org/resource/Journal_of_Applied_Physics, http://dbpedia.org/resource/Parenting_styles, http://dbpedia.org/resource/Canadian_Football_League, http://dbpedia.org/resource/CBC_Sports, http://dbpedia.org/resource/Chairman, http://dbpedia.org/resource/Drexel_Burnham_Lambert, http://dbpedia.org/resource/Global_Crossing, http://dbpedia.org/resource/Prince_of_Persia, http://dbpedia.org/resource/Jewellery, http://dbpedia.org/resource/Franklin_graph, http://dbpedia.org/resource/Meta-analysis, http://dbpedia.org/resource/Randomized_controlled_trial, http://dbpedia.org/resource/Indiana_Pacers, http://dbpedia.org/resource/IMac, http://dbpedia.org/resource/IPod, http://dbpedia.org/resource/WebOS, http://dbpedia.org/resource/Divorce_Busting, http://dbpedia.org/resource/Solution_focused_brief_therapy, http://dbpedia.org/resource/Bioenergetic_analysis, http://dbpedia.org/resource/Film, http://dbpedia.org/resource/Videography, http://dbpedia.org/resource/Positive_Adult_Development, http://dbpedia.org/resource/Surgical_pathology, http://dbpedia.org/resource/Paramount_Pictures, http://dbpedia.org/resource/Columbia_Pictures, http://dbpedia.org/resource/United_Artists, http://dbpedia.org/resource/Lorimar_Television, http://dbpedia.org/resource/Council_of_Economic_Advisers, http://dbpedia.org/resource/Congressional_Budget_Office, http://dbpedia.org/resource/United_States_Congress_Joint_Economic_Committee, http://dbpedia.org/resource/Tax_Policy_Center, http://dbpedia.org/resource/Mutation, http://dbpedia.org/resource/Molecular_biology, http://dbpedia.org/resource/Berkshire_Hathaway, http://dbpedia.org/resource/The_Tombs, http://dbpedia.org/resource/Theron_S._Copeland, http://dbpedia.org/resource/Aide-de-camp, http://dbpedia.org/resource/Mercenary, http://dbpedia.org/resource/Adventure, http://dbpedia.org/resource/New_York_City_draft_riots, http://dbpedia.org/resource/Bloomingdale's, http://dbpedia.org/resource/Blood_fractionation, http://dbpedia.org/resource/Heterogeneous_ribonucleoprotein_particle, http://dbpedia.org/resource/Spinal_muscular_atrophy, http://dbpedia.org/resource/Survival_of_motor_neuron, http://dbpedia.org/resource/Sea_urchin, http://dbpedia.org/resource/Gene_regulatory_network, http://dbpedia.org/resource/Cagney_&_Lacey, http://dbpedia.org/resource/Count, http://dbpedia.org/resource/Umberto_Agnelli, http://dbpedia.org/resource/Gianni_Agnelli, http://dbpedia.org/resource/University_of_Michigan, http://dbpedia.org/resource/Episcopal_Diocese_of_Michigan, http://dbpedia.org/resource/Psychotherapy, http://dbpedia.org/resource/Design, http://dbpedia.org/resource/Photography, http://dbpedia.org/resource/Television_advertisement, http://dbpedia.org/resource/Advertising, http://dbpedia.org/resource/Youth_culture, http://dbpedia.org/resource/Brand_management, http://dbpedia.org/resource/Dell_Publishing, http://dbpedia.org/resource/Dell_Comics, http://dbpedia.org/resource/Molecular_electronics, http://dbpedia.org/resource/Nanotechnology, http://dbpedia.org/resource/The_Apprentice_(UK_TV_series), http://dbpedia.org/resource/Over_There_(TV_series), http://dbpedia.org/resource/Chemical_reaction_network_theory, http://dbpedia.org/resource/British_Phonographic_Industry, http://dbpedia.org/resource/CBS_Records_International, http://dbpedia.org/resource/Architect, http://dbpedia.org/resource/Plymouth_Rock_Assurance, http://dbpedia.org/resource/W_Hotels, http://dbpedia.org/resource/Starwood_Capital_Group, http://dbpedia.org/resource/Starwood_Hotels_and_Resorts_Worldwide, http://dbpedia.org/resource/Linear_programming, http://dbpedia.org/resource/Chomp, http://dbpedia.org/resource/Stable_marriage_problem, http://dbpedia.org/resource/Convex_analysis, http://dbpedia.org/resource/Gale_Transform, http://dbpedia.org/resource/Hex_(board_game), http://dbpedia.org/resource/Ramsey_problem, http://dbpedia.org/resource/Alfred_Pennyworth, http://dbpedia.org/resource/DC_animated_universe, http://dbpedia.org/resource/Maverick_(TV_series), http://dbpedia.org/resource/Dark_energy, http://dbpedia.org/resource/Superconductivity, http://dbpedia.org/resource/Cooper_pair, http://dbpedia.org/resource/Gell-Mann_matrices, http://dbpedia.org/resource/Gell-Mann–Okubo_mass_formula, http://dbpedia.org/resource/Elementary_particle, http://dbpedia.org/resource/Quark, http://dbpedia.org/resource/Effective_complexity, http://dbpedia.org/resource/Gell-Mann_and_Low_theorem, http://dbpedia.org/resource/Renormalization_group, http://dbpedia.org/resource/Gell-Mann–Nishijima_formula, http://dbpedia.org/resource/Strangeness, http://dbpedia.org/resource/Cosmology, http://dbpedia.org/resource/Star_Trek, http://dbpedia.org/resource/Too_Close_for_Comfort, http://dbpedia.org/resource/The_Honeymooners, http://dbpedia.org/resource/The_Fabric_of_the_Cosmos, http://dbpedia.org/resource/The_Elegant_Universe, http://dbpedia.org/resource/The_Hidden_Reality, http://dbpedia.org/resource/Randall–Sundrum_model, http://dbpedia.org/resource/Warped_Passages, http://dbpedia.org/resource/Matching_law, http://dbpedia.org/resource/The_Bell_Curve, http://dbpedia.org/resource/Hofstadter_Committee, http://dbpedia.org/resource/Chandigarh, http://dbpedia.org/resource/Differentially_closed_field, http://dbpedia.org/resource/Lie–Kolchin_theorem, http://dbpedia.org/resource/The_Real_Housewives_of_New_York_City, http://dbpedia.org/resource/The_Brady_Bunch, http://dbpedia.org/resource/Men's_Wearhouse, http://dbpedia.org/resource/A_Course_in_Miracles, http://dbpedia.org/resource/Flow_cytometry, http://dbpedia.org/resource/Star_Trek:_Deep_Space_Nine, http://dbpedia.org/resource/David_Reimer, http://dbpedia.org/resource/San_Diego_Chargers, http://dbpedia.org/resource/Seattle_SuperSonics, http://dbpedia.org/resource/1001_Ways_to_Beat_the_Draft, http://dbpedia.org/resource/The_Fugs, http://dbpedia.org/resource/Robert_Russa_Moton_Museum, http://dbpedia.org/resource/Davis_v._County_School_Board_of_Prince_Edward_County, http://dbpedia.org/resource/Woody_Allen, http://dbpedia.org/resource/Midnight_Cowboy, http://dbpedia.org/resource/Coming_Home_(1978_film), http://dbpedia.org/resource/Hacker_(computer_security), http://dbpedia.org/resource/Phreaking, http://dbpedia.org/resource/Information_security, http://dbpedia.org/resource/Paine_Webber, http://dbpedia.org/resource/Data_Resources_Inc., http://dbpedia.org/resource/Lightyear_Capital, http://dbpedia.org/resource/Friendly_Fire_(1979_film), http://dbpedia.org/resource/So_Much_Unfairness_of_Things, http://dbpedia.org/resource/Dwight_D._Eisenhower, http://dbpedia.org/resource/Video_blog, http://dbpedia.org/resource/Interview, http://dbpedia.org/resource/Big_wall_climbing, http://dbpedia.org/resource/Biopsychosocial_model, http://dbpedia.org/resource/Public_relations, http://dbpedia.org/resource/Constructed_language, http://dbpedia.org/resource/Hair_transplantation, http://dbpedia.org/resource/Dermatology, http://dbpedia.org/resource/Allen_&_Ginter, http://dbpedia.org/resource/Ginter_Park, http://dbpedia.org/resource/Jefferson_Hotel_(Richmond,_Virginia), http://dbpedia.org/resource/Lewis_Ginter_Botanical_Garden, http://dbpedia.org/resource/Berlusconi_IV_Cabinet, http://dbpedia.org/resource/United_States_support_for_Iraq_during_the_Iran–Iraq_war, http://dbpedia.org/resource/Medco_Health_Solutions, http://dbpedia.org/resource/Movieline, http://dbpedia.org/resource/Variety_(magazine), http://dbpedia.org/resource/Women's_Wear_Daily, http://dbpedia.org/resource/BGR.com, http://dbpedia.org/resource/India.com, http://dbpedia.org/resource/Penske_Media_Corporation, http://dbpedia.org/resource/Deadline.com, http://dbpedia.org/resource/Fairchild_Fashion_Media, http://dbpedia.org/resource/Dragon_Racing, http://dbpedia.org/resource/TVLine, http://dbpedia.org/resource/Set_theory, http://dbpedia.org/resource/Topology, http://dbpedia.org/resource/Boolean_algebra_(structure), http://dbpedia.org/resource/Mathematics_education, http://dbpedia.org/resource/Harry_Lazarus, http://dbpedia.org/resource/Hollywood_Squares, http://dbpedia.org/resource/Catch_21, http://dbpedia.org/resource/Gambit_(game_show), http://dbpedia.org/resource/Women's_music, http://dbpedia.org/resource/DJB_Foundation, http://dbpedia.org/resource/Foreign_policy_analysis, http://dbpedia.org/resource/Leptin, http://dbpedia.org/resource/Infertility, http://dbpedia.org/resource/Painting, http://dbpedia.org/resource/Honeycomb_(geometry), http://dbpedia.org/resource/Entropic_force, http://dbpedia.org/resource/Self-assembly, http://dbpedia.org/resource/Dynamical_heterogeneity, http://dbpedia.org/resource/Patchy_particles, http://dbpedia.org/resource/Bowery, http://dbpedia.org/resource/Tammany_Hall, http://dbpedia.org/resource/Emma_Cunningham, http://dbpedia.org/resource/Alexander_Turney_Stewart, http://dbpedia.org/resource/George_Leonidas_Leslie, http://dbpedia.org/resource/Tommy_Hadden, http://dbpedia.org/resource/Patsy_Conroy_Gang, http://dbpedia.org/resource/MacArthur_Fellows_Program, http://dbpedia.org/resource/Guggenheim_Fellowship, http://dbpedia.org/resource/International_Center_of_Photography, http://dbpedia.org/resource/Dorzolamide, http://dbpedia.org/resource/Scotchgard, http://dbpedia.org/resource/ATP-hydrolysis, http://dbpedia.org/resource/Protease, http://dbpedia.org/resource/Bacterial_small_RNA, http://dbpedia.org/resource/Gaîté_Parisienne, http://dbpedia.org/resource/History_of_the_Brooklyn_Dodgers, http://dbpedia.org/resource/University_of_California,_San_Diego, http://dbpedia.org/resource/Transplantation_tolerance, http://dbpedia.org/resource/MHC_class_II, http://dbpedia.org/resource/4chan, http://dbpedia.org/resource/Bright_Star_Catalogue, http://dbpedia.org/resource/Rockcityclub, http://dbpedia.org/resource/Trump_Hotel_Las_Vegas, http://dbpedia.org/resource/Lucchese_crime_family, http://dbpedia.org/resource/Infomercial, http://dbpedia.org/resource/Ronco, http://dbpedia.org/resource/Fermi–Pasta–Ulam_problem, http://dbpedia.org/resource/Adler–Bardeen_theorem, http://dbpedia.org/resource/Personality_psychology, http://dbpedia.org/resource/Lawsuit, http://dbpedia.org/resource/Cyrano_de_Bergerac_(play), http://dbpedia.org/resource/The_Vagabond_King, http://dbpedia.org/resource/Forbidden_Planet, http://dbpedia.org/resource/Flying_Down_to_Rio, http://dbpedia.org/resource/The_Great_Gatsby_(1949_film), http://dbpedia.org/resource/Tarzan_the_Ape_Man_(1932_film), http://dbpedia.org/resource/Tokyo_Joe_(1949_film), http://dbpedia.org/resource/Saffir–Simpson_hurricane_wind_scale, http://dbpedia.org/resource/The_Goldbergs_(broadcast_series), http://dbpedia.org/resource/Tiffany_glass, http://dbpedia.org/resource/Jean-Antoine_Watteau, http://dbpedia.org/resource/Quorum_sensing, http://dbpedia.org/resource/The_Tonight_Show, http://dbpedia.org/resource/The_Hollywood_Palace, http://dbpedia.org/resource/Seventh_Generation_Inc., http://dbpedia.org/resource/Recombinant_DNA, http://dbpedia.org/resource/Atomic_physics, http://dbpedia.org/resource/Television, http://dbpedia.org/resource/YouTube, http://dbpedia.org/resource/United_Federation_of_Teachers, http://dbpedia.org/resource/American_Federation_of_Teachers, http://dbpedia.org/resource/Chromosome, http://dbpedia.org/resource/History_of_France, http://dbpedia.org/resource/Transduction_(genetics), http://dbpedia.org/resource/Virology, http://dbpedia.org/resource/Biological_illustration, http://dbpedia.org/resource/Polyomaviridae, http://dbpedia.org/resource/Mortgage-backed_security, http://dbpedia.org/resource/Securitization, http://dbpedia.org/resource/Artist's_book, http://dbpedia.org/resource/Bookbinding, http://dbpedia.org/resource/Art_therapy, http://dbpedia.org/resource/Psychologist, http://dbpedia.org/resource/The_Nixon_Interviews, http://dbpedia.org/resource/Latin_America, http://dbpedia.org/resource/Al-Qaeda, http://dbpedia.org/resource/Fly_Away_(film), http://dbpedia.org/resource/Operant_conditioning, http://dbpedia.org/resource/Behaviorism, http://dbpedia.org/resource/Sociology_of_scientific_knowledge, http://dbpedia.org/resource/Levi's_reduction_process, http://www.Pzena.com, http://dbpedia.org/resource/The_New_York_Times, http://dbpedia.org/resource/Pentagon_Papers, http://dbpedia.org/resource/Prospect_(magazine), http://dbpedia.org/resource/Epistemic_Frame_Games, http://dbpedia.org/resource/Epistemic_Frame_Theory, http://dbpedia.org/resource/Epistemic_Network_Analysis, http://dbpedia.org/resource/Living_Lohan, http://dbpedia.org/resource/Game_theory, http://dbpedia.org/resource/Discrete_mathematics, http://dbpedia.org/resource/Pimsleur_Language_Aptitude_Battery, http://dbpedia.org/resource/Pimsleur_language_learning_system, http://dbpedia.org/resource/Action_for_Children's_Television, http://dbpedia.org/resource/Rock_'n_Roll_Fantasy_Camp, http://dbpedia.org/resource/Tibetology, http://dbpedia.org/resource/Detective_quantum_efficiency, http://dbpedia.org/resource/Rose_criterion, http://dbpedia.org/resource/Wide_area_information_server, http://dbpedia.org/resource/Internet_Archive, http://dbpedia.org/resource/Alexa_Internet, http://dbpedia.org/resource/American_Psychological_Association, http://dbpedia.org/resource/Soliton, http://dbpedia.org/resource/Computational_neuroscience, http://dbpedia.org/resource/History, http://dbpedia.org/resource/The_Daily_Show, http://dbpedia.org/resource/Sirius_XM_Holdings, http://dbpedia.org/resource/Atomic_bombings_of_Hiroshima_and_Nagasaki, http://dbpedia.org/resource/Physics, http://dbpedia.org/resource/Bluto, http://dbpedia.org/resource/Editing, http://dbpedia.org/resource/Hydrothermal_vent, http://dbpedia.org/resource/Tom_Leykis, http://dbpedia.org/resource/Staten_Island_(film), http://dbpedia.org/resource/The_Purge, http://dbpedia.org/resource/The_Purge:_Anarchy, http://dbpedia.org/resource/Intersex, http://dbpedia.org/resource/MTV, http://dbpedia.org/resource/Viacom, http://dbpedia.org/resource/Photo_League, http://dbpedia.org/resource/1921_Indianapolis_500, http://dbpedia.org/resource/Public_opinion, http://dbpedia.org/resource/Sociological_theory, http://dbpedia.org/resource/Sociology_of_culture, http://dbpedia.org/resource/Political_sociology, http://dbpedia.org/resource/Cognitive_sociology, http://dbpedia.org/resource/Social_network_analysis, http://dbpedia.org/resource/Organizational_sociology, http://dbpedia.org/resource/General_Circulation_Model, http://dbpedia.org/resource/Mic_(media_company), http://dbpedia.org/resource/The_FBomb, http://dbpedia.org/resource/Conflict_resolution, http://dbpedia.org/resource/Distributive_justice, http://dbpedia.org/resource/Costco, http://dbpedia.org/resource/FedMart, http://dbpedia.org/resource/PriceSmart, http://dbpedia.org/resource/Price_Club, http://dbpedia.org/resource/Herbert_Barrett_Management, http://dbpedia.org/resource/Behavioral_neurology, http://dbpedia.org/resource/Substance_use_disorder, http://dbpedia.org/resource/Addictive_behavior, http://dbpedia.org/resource/Ethology, http://dbpedia.org/resource/Psychoanalysis, http://dbpedia.org/resource/Clinical_psychology, http://dbpedia.org/resource/Conflict_of_laws, http://dbpedia.org/resource/Helium, http://dbpedia.org/resource/Myanmar, http://dbpedia.org/resource/Malmedy_massacre_trial, http://dbpedia.org/resource/International_recognition_of_the_National_Transitional_Council, http://dbpedia.org/resource/Southern_California_Library_for_Social_Studies_and_Research, http://dbpedia.org/resource/San_Francisco_Fire_Department, http://dbpedia.org/resource/Peg_+_Cat, http://dbpedia.org/resource/MMR_vaccine_controversy] [INFO] Predicted Answer: [] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 825, P: 0.498, R: 0.553, macro F1: 0.506, macro F1*: 0.524 -[INFO] Current time: 1867 ms, average: 2925.17 ms/ques, entity detection: 500.85 ms/ques (linking tool: 407.69 ms/ques), relation detection: 948.02 ms/ques (similarity: 714.03 ms/ques), EDG generation: 46.48 ms/ques +[INFO] Cumulative metrics, sample: 825, P: 0.498, R: 0.553, macro F1: 0.524 +[INFO] Current question consumed time: 1592 ms, average: 2625.13 ms/ques, entity detection: 440.25 ms/ques (linking tool: 362.23 ms/ques), relation detection: 872.66 ms/ques (similarity: 706.79 ms/ques), EDG generation: 46.89 ms/ques [INFO] Entity Linking P: 0.7357575757575757, R: 0.6884848484848485, micro F1: 0.7036363636363642, macro F1: 0.7113366860090263 [INFO] Relation Linking P: 0.595050505050505, R: 0.5412121212121213, micro F1: 0.5566060606060614, macro F1: 0.5668558282621755 [INFO] Type Linking P: 0.5612121212121212, R: 0.5612121212121212, micro F1: 0.5612121212121212, macro F1: 0.5612121212121212 @@ -11581,8 +11581,8 @@ Question 825: Who is the developer of the software which is produced by Claris ? [INFO] Golden Answer: [http://dbpedia.org/resource/FileMaker_Inc., http://dbpedia.org/resource/Apple_Inc., http://dbpedia.org/resource/Claris] [INFO] Predicted Answer: [http://dbpedia.org/resource/FileMaker_Inc., http://dbpedia.org/resource/Apple_Inc., http://dbpedia.org/resource/Claris, an Apple Subsidiary@en, Claris Corporation@en, Solosoft, Claris@en, Bob Hearn and Scott Haldaway@en] [INFO] P: 0.42857142857142855, R: 1.0, F: 0.6 -[INFO] Cumulative metrics, sample: 826, P: 0.498, R: 0.553, macro F1: 0.506, macro F1*: 0.524 -[INFO] Current time: 1927 ms, average: 2923.96 ms/ques, entity detection: 500.51 ms/ques (linking tool: 407.36 ms/ques), relation detection: 947.79 ms/ques (similarity: 713.94 ms/ques), EDG generation: 46.46 ms/ques +[INFO] Cumulative metrics, sample: 826, P: 0.498, R: 0.553, macro F1: 0.524 +[INFO] Current question consumed time: 1657 ms, average: 2623.96 ms/ques, entity detection: 439.96 ms/ques (linking tool: 361.96 ms/ques), relation detection: 872.42 ms/ques (similarity: 706.70 ms/ques), EDG generation: 46.87 ms/ques [INFO] Entity Linking P: 0.7360774818401937, R: 0.6888619854721549, micro F1: 0.7039951573849884, macro F1: 0.7116874888140531 [INFO] Relation Linking P: 0.5949354317998385, R: 0.5411622276029056, micro F1: 0.5565375302663447, macro F1: 0.5667762465454816 [INFO] Type Linking P: 0.5605326876513317, R: 0.5605326876513317, micro F1: 0.5605326876513317, macro F1: 0.5605326876513317 @@ -11595,8 +11595,8 @@ Question 826: What is the river whose source is Cowombat Flat and has it's mout [INFO] Golden Answer: [http://dbpedia.org/resource/Murray_River] [INFO] Predicted Answer: [http://dbpedia.org/resource/Murray_River, http://dbpedia.org/resource/Bremer_River_(South_Australia)] [INFO] P: 0.5, R: 1.0, F: 0.6666666666666666 -[INFO] Cumulative metrics, sample: 827, P: 0.498, R: 0.554, macro F1: 0.507, macro F1*: 0.525 -[INFO] Current time: 2258 ms, average: 2923.15 ms/ques, entity detection: 500.42 ms/ques (linking tool: 407.22 ms/ques), relation detection: 947.39 ms/ques (similarity: 713.52 ms/ques), EDG generation: 46.58 ms/ques +[INFO] Cumulative metrics, sample: 827, P: 0.498, R: 0.554, macro F1: 0.525 +[INFO] Current question consumed time: 1834 ms, average: 2623.00 ms/ques, entity detection: 439.83 ms/ques (linking tool: 361.80 ms/ques), relation detection: 872.03 ms/ques (similarity: 706.29 ms/ques), EDG generation: 46.90 ms/ques [INFO] Entity Linking P: 0.7363966142684402, R: 0.68863361547763, micro F1: 0.7039500201531645, macro F1: 0.7117146742908387 [INFO] Relation Linking P: 0.5954252317613865, R: 0.5411124546553809, micro F1: 0.5566706972994769, macro F1: 0.5669710957635686 [INFO] Type Linking P: 0.5598548972188634, R: 0.5598548972188634, micro F1: 0.5598548972188634, macro F1: 0.5598548972188634 @@ -11609,8 +11609,8 @@ Question 827: Where does the river flow into which begins in Lebrsnik? [INFO] Golden Answer: [http://dbpedia.org/resource/Adriatic_Sea] [INFO] Predicted Answer: [http://dbpedia.org/resource/Jablanica_lake, http://dbpedia.org/resource/Adriatic_Sea] [INFO] P: 0.5, R: 1.0, F: 0.6666666666666666 -[INFO] Cumulative metrics, sample: 828, P: 0.498, R: 0.554, macro F1: 0.507, macro F1*: 0.525 -[INFO] Current time: 1369 ms, average: 2921.28 ms/ques, entity detection: 499.89 ms/ques (linking tool: 406.78 ms/ques), relation detection: 946.99 ms/ques (similarity: 713.33 ms/ques), EDG generation: 46.55 ms/ques +[INFO] Cumulative metrics, sample: 828, P: 0.498, R: 0.554, macro F1: 0.525 +[INFO] Current question consumed time: 1154 ms, average: 2621.23 ms/ques, entity detection: 439.35 ms/ques (linking tool: 361.39 ms/ques), relation detection: 871.63 ms/ques (similarity: 706.11 ms/ques), EDG generation: 46.86 ms/ques [INFO] Entity Linking P: 0.7367149758454107, R: 0.6890096618357487, micro F1: 0.7043075684380037, macro F1: 0.7120641994405874 [INFO] Relation Linking P: 0.5947061191626408, R: 0.5404589371980676, micro F1: 0.5559983896940428, macro F1: 0.5662863480633711 [INFO] Type Linking P: 0.5591787439613527, R: 0.5591787439613527, micro F1: 0.5591787439613527, macro F1: 0.5591787439613527 @@ -11623,8 +11623,8 @@ Question 828: What is the government type of Ancient Rome ? [INFO] Golden Answer: [http://dbpedia.org/resource/Roman_Empire, http://dbpedia.org/resource/Roman_Kingdom, http://dbpedia.org/resource/Roman_Republic] [INFO] Predicted Answer: [http://dbpedia.org/resource/Roman_Empire, http://dbpedia.org/resource/Roman_Kingdom, http://dbpedia.org/resource/Roman_Republic] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 829, P: 0.499, R: 0.555, macro F1: 0.507, macro F1*: 0.525 -[INFO] Current time: 1959 ms, average: 2920.11 ms/ques, entity detection: 499.79 ms/ques (linking tool: 406.71 ms/ques), relation detection: 946.77 ms/ques (similarity: 712.90 ms/ques), EDG generation: 46.51 ms/ques +[INFO] Cumulative metrics, sample: 829, P: 0.499, R: 0.555, macro F1: 0.525 +[INFO] Current question consumed time: 1579 ms, average: 2619.97 ms/ques, entity detection: 439.19 ms/ques (linking tool: 361.28 ms/ques), relation detection: 871.34 ms/ques (similarity: 705.68 ms/ques), EDG generation: 46.82 ms/ques [INFO] Entity Linking P: 0.7370325693606755, R: 0.6893848009650181, micro F1: 0.704664254121432, macro F1: 0.7124128767689237 [INFO] Relation Linking P: 0.5951950140731805, R: 0.5410132689987938, micro F1: 0.5565339766787303, macro F1: 0.5668122738639049 [INFO] Type Linking P: 0.5597104945717732, R: 0.5597104945717732, micro F1: 0.5597104945717732, macro F1: 0.5597104945717732 @@ -11637,8 +11637,8 @@ Question 829: Which director of On deadly Ground also released Mojo Priest ? [INFO] Golden Answer: [http://dbpedia.org/resource/Steven_Seagal] [INFO] Predicted Answer: [http://dbpedia.org/resource/Steven_Seagal] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 830, P: 0.500, R: 0.555, macro F1: 0.508, macro F1*: 0.526 -[INFO] Current time: 1527 ms, average: 2918.44 ms/ques, entity detection: 499.62 ms/ques (linking tool: 406.49 ms/ques), relation detection: 946.03 ms/ques (similarity: 712.28 ms/ques), EDG generation: 46.47 ms/ques +[INFO] Cumulative metrics, sample: 830, P: 0.500, R: 0.555, macro F1: 0.526 +[INFO] Current question consumed time: 1217 ms, average: 2618.28 ms/ques, entity detection: 439.02 ms/ques (linking tool: 361.05 ms/ques), relation detection: 870.65 ms/ques (similarity: 705.07 ms/ques), EDG generation: 46.78 ms/ques [INFO] Entity Linking P: 0.7373493975903614, R: 0.6897590361445783, micro F1: 0.7050200803212856, macro F1: 0.712760709363414 [INFO] Relation Linking P: 0.5950803212851405, R: 0.5409638554216868, micro F1: 0.5564658634538162, macro F1: 0.5667331455738995 [INFO] Type Linking P: 0.5602409638554217, R: 0.5602409638554217, micro F1: 0.5602409638554217, macro F1: 0.5602409638554217 @@ -11651,8 +11651,8 @@ Question 830: What is the television show whose channel's parent organisation is [INFO] Golden Answer: [http://dbpedia.org/resource/The_Dan_Patrick_Show] [INFO] Predicted Answer: [http://dbpedia.org/resource/Movies_OK] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 831, P: 0.499, R: 0.555, macro F1: 0.507, macro F1*: 0.525 -[INFO] Current time: 2027 ms, average: 2917.36 ms/ques, entity detection: 499.30 ms/ques (linking tool: 406.16 ms/ques), relation detection: 945.73 ms/ques (similarity: 712.17 ms/ques), EDG generation: 46.44 ms/ques +[INFO] Cumulative metrics, sample: 831, P: 0.499, R: 0.555, macro F1: 0.525 +[INFO] Current question consumed time: 1802 ms, average: 2617.30 ms/ques, entity detection: 438.71 ms/ques (linking tool: 360.75 ms/ques), relation detection: 870.37 ms/ques (similarity: 704.98 ms/ques), EDG generation: 46.76 ms/ques [INFO] Entity Linking P: 0.7364620938628159, R: 0.6889290012033694, micro F1: 0.7041716807059772, macro F1: 0.711902994911713 [INFO] Relation Linking P: 0.5943642198154833, R: 0.5403128760529483, micro F1: 0.5557962294424398, macro F1: 0.5660511562290454 [INFO] Type Linking P: 0.5595667870036101, R: 0.5595667870036101, micro F1: 0.5595667870036101, macro F1: 0.5595667870036101 @@ -11665,8 +11665,8 @@ Question 831: Who commanded the invasion of Buwat and made Fatima bint Sa'd famo [INFO] Golden Answer: [http://dbpedia.org/resource/Muhammad] [INFO] Predicted Answer: [http://dbpedia.org/resource/Muhammad, http://dbpedia.org/resource/Umayyah_ibn_Khalaf] [INFO] P: 0.5, R: 1.0, F: 0.6666666666666666 -[INFO] Cumulative metrics, sample: 832, P: 0.499, R: 0.555, macro F1: 0.508, macro F1*: 0.526 -[INFO] Current time: 976 ms, average: 2915.03 ms/ques, entity detection: 498.90 ms/ques (linking tool: 405.80 ms/ques), relation detection: 944.80 ms/ques (similarity: 711.51 ms/ques), EDG generation: 46.40 ms/ques +[INFO] Cumulative metrics, sample: 832, P: 0.499, R: 0.555, macro F1: 0.526 +[INFO] Current question consumed time: 819 ms, average: 2615.14 ms/ques, entity detection: 438.35 ms/ques (linking tool: 360.42 ms/ques), relation detection: 869.52 ms/ques (similarity: 704.33 ms/ques), EDG generation: 46.72 ms/ques [INFO] Entity Linking P: 0.7367788461538461, R: 0.6887019230769231, micro F1: 0.704126602564103, macro F1: 0.7119296439226878 [INFO] Relation Linking P: 0.5948517628205128, R: 0.5402644230769231, micro F1: 0.5559294871794881, macro F1: 0.5662455499256766 [INFO] Type Linking P: 0.5600961538461539, R: 0.5600961538461539, micro F1: 0.5600961538461539, macro F1: 0.5600961538461539 @@ -11679,8 +11679,8 @@ Question 832: What is the home ground of the clubs managed by Ciao Zanardi? [INFO] Golden Answer: [http://dbpedia.org/resource/Florida, http://dbpedia.org/resource/Fort_Lauderdale,_Florida, http://dbpedia.org/resource/Lockhart_Stadium] [INFO] Predicted Answer: [http://dbpedia.org/resource/Guelphs_and_Ghibellines] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 833, P: 0.498, R: 0.555, macro F1: 0.507, macro F1*: 0.525 -[INFO] Current time: 2843 ms, average: 2914.94 ms/ques, entity detection: 498.48 ms/ques (linking tool: 405.42 ms/ques), relation detection: 945.25 ms/ques (similarity: 711.71 ms/ques), EDG generation: 46.47 ms/ques +[INFO] Cumulative metrics, sample: 833, P: 0.498, R: 0.555, macro F1: 0.525 +[INFO] Current question consumed time: 2540 ms, average: 2615.05 ms/ques, entity detection: 437.97 ms/ques (linking tool: 360.08 ms/ques), relation detection: 869.97 ms/ques (similarity: 704.53 ms/ques), EDG generation: 46.79 ms/ques [INFO] Entity Linking P: 0.7358943577430972, R: 0.687875150060024, micro F1: 0.7032813125250105, macro F1: 0.7110749864870063 [INFO] Relation Linking P: 0.5941376550620248, R: 0.5396158463385354, micro F1: 0.5552621048419377, macro F1: 0.5655657833591391 [INFO] Type Linking P: 0.5594237695078031, R: 0.5594237695078031, micro F1: 0.5594237695078031, macro F1: 0.5594237695078031 @@ -11693,8 +11693,8 @@ Question 833: What is the country with native language as Languages of Ethiopia? [INFO] Golden Answer: [http://dbpedia.org/resource/Ethiopia] [INFO] Predicted Answer: [Ethiopia@en] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 834, P: 0.499, R: 0.555, macro F1: 0.508, macro F1*: 0.526 -[INFO] Current time: 1035 ms, average: 2912.69 ms/ques, entity detection: 498.46 ms/ques (linking tool: 405.45 ms/ques), relation detection: 944.23 ms/ques (similarity: 710.93 ms/ques), EDG generation: 46.44 ms/ques +[INFO] Cumulative metrics, sample: 834, P: 0.499, R: 0.555, macro F1: 0.526 +[INFO] Current question consumed time: 904 ms, average: 2613.00 ms/ques, entity detection: 437.94 ms/ques (linking tool: 360.09 ms/ques), relation detection: 869.02 ms/ques (similarity: 703.76 ms/ques), EDG generation: 46.77 ms/ques [INFO] Entity Linking P: 0.7362110311750599, R: 0.6882494004796164, micro F1: 0.7036370903277382, macro F1: 0.7114227809671455 [INFO] Relation Linking P: 0.5934252597921662, R: 0.5389688249400479, micro F1: 0.5545963229416476, macro F1: 0.5648876469282528 [INFO] Type Linking P: 0.5587529976019184, R: 0.5587529976019184, micro F1: 0.5587529976019184, macro F1: 0.5587529976019184 @@ -11707,8 +11707,8 @@ Question 834: What is the band with previous members Kevin Jonas and Joe Jonas? [INFO] Golden Answer: [http://dbpedia.org/resource/Jonas_Brothers] [INFO] Predicted Answer: [http://dbpedia.org/resource/Jonas_Brothers] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 835, P: 0.500, R: 0.556, macro F1: 0.508, macro F1*: 0.526 -[INFO] Current time: 2181 ms, average: 2911.81 ms/ques, entity detection: 498.23 ms/ques (linking tool: 405.30 ms/ques), relation detection: 943.81 ms/ques (similarity: 710.87 ms/ques), EDG generation: 46.44 ms/ques +[INFO] Cumulative metrics, sample: 835, P: 0.500, R: 0.556, macro F1: 0.526 +[INFO] Current question consumed time: 1890 ms, average: 2612.13 ms/ques, entity detection: 437.72 ms/ques (linking tool: 359.94 ms/ques), relation detection: 868.65 ms/ques (similarity: 703.71 ms/ques), EDG generation: 46.77 ms/ques [INFO] Entity Linking P: 0.7365269461077845, R: 0.688622754491018, micro F1: 0.7039920159680644, macro F1: 0.7117697378352539 [INFO] Relation Linking P: 0.5939121756487026, R: 0.5395209580838324, micro F1: 0.555129740518963, macro F1: 0.5654115033119462 [INFO] Type Linking P: 0.5580838323353293, R: 0.5580838323353293, micro F1: 0.5580838323353293, macro F1: 0.5580838323353293 @@ -11721,8 +11721,8 @@ Question 835: List the people with Scientology as their religious belief? [INFO] Golden Answer: [http://dbpedia.org/resource/Kirstie_Alley, http://dbpedia.org/resource/Nancy_Cartwright, http://dbpedia.org/resource/Tim_Bowles, http://dbpedia.org/resource/Death_of_Elli_Perkins, http://dbpedia.org/resource/Bob_Adams_(American_football), http://dbpedia.org/resource/David_Miscavige, http://dbpedia.org/resource/Karen_Black, http://dbpedia.org/resource/L._Ron_Hubbard, http://dbpedia.org/resource/Greta_Van_Susteren, http://dbpedia.org/resource/Anne_Archer, http://dbpedia.org/resource/Catherine_Bell_(actress), http://dbpedia.org/resource/Elisabeth_Moss, http://dbpedia.org/resource/Keith_Code, http://dbpedia.org/resource/Jason_Lee_(actor), http://dbpedia.org/resource/Tom_Cruise, http://dbpedia.org/resource/Jenna_Elfman, http://dbpedia.org/resource/Michele_Miscavige, http://dbpedia.org/resource/Sonny_Bono, http://dbpedia.org/resource/Christopher_Masterson, http://dbpedia.org/resource/Milton_Katselas, http://dbpedia.org/resource/Jordan_Masterson, http://dbpedia.org/resource/Pat_Broeker, http://dbpedia.org/resource/John_Travolta, http://dbpedia.org/resource/Ethan_Suplee, http://dbpedia.org/resource/Michael_Doven, http://dbpedia.org/resource/Grant_Cardone, http://dbpedia.org/resource/Hans_Hunt, http://dbpedia.org/resource/John_Carmichael_(Scientology)] [INFO] Predicted Answer: [http://dbpedia.org/resource/Album] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 836, P: 0.499, R: 0.555, macro F1: 0.508, macro F1*: 0.526 -[INFO] Current time: 2092 ms, average: 2910.83 ms/ques, entity detection: 497.98 ms/ques (linking tool: 405.01 ms/ques), relation detection: 943.19 ms/ques (similarity: 710.51 ms/ques), EDG generation: 46.40 ms/ques +[INFO] Cumulative metrics, sample: 836, P: 0.499, R: 0.555, macro F1: 0.526 +[INFO] Current question consumed time: 1750 ms, average: 2611.10 ms/ques, entity detection: 437.44 ms/ques (linking tool: 359.66 ms/ques), relation detection: 868.11 ms/ques (similarity: 703.51 ms/ques), EDG generation: 46.73 ms/ques [INFO] Entity Linking P: 0.7356459330143541, R: 0.687799043062201, micro F1: 0.7031499202551839, macro F1: 0.710918338627317 [INFO] Relation Linking P: 0.5932017543859649, R: 0.5388755980861244, micro F1: 0.5544657097288685, macro F1: 0.5647351737625299 [INFO] Type Linking P: 0.5574162679425837, R: 0.5574162679425837, micro F1: 0.5574162679425837, macro F1: 0.5574162679425837 @@ -11735,8 +11735,8 @@ Question 836: How many fictional characters were humans? [INFO] Golden Answer: [102] [INFO] Predicted Answer: [] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 837, P: 0.498, R: 0.554, macro F1: 0.507, macro F1*: 0.525 -[INFO] Current time: 308 ms, average: 2907.72 ms/ques, entity detection: 497.39 ms/ques (linking tool: 404.53 ms/ques), relation detection: 942.29 ms/ques (similarity: 709.81 ms/ques), EDG generation: 46.36 ms/ques +[INFO] Cumulative metrics, sample: 837, P: 0.498, R: 0.554, macro F1: 0.525 +[INFO] Current question consumed time: 266 ms, average: 2608.30 ms/ques, entity detection: 436.92 ms/ques (linking tool: 359.23 ms/ques), relation detection: 867.28 ms/ques (similarity: 702.83 ms/ques), EDG generation: 46.69 ms/ques [INFO] Entity Linking P: 0.7347670250896058, R: 0.6869772998805257, micro F1: 0.7023098367184393, macro F1: 0.7100689738260895 [INFO] Relation Linking P: 0.5924930306650736, R: 0.538231780167264, micro F1: 0.5538032656312235, macro F1: 0.5640604602932797 [INFO] Type Linking P: 0.5567502986857825, R: 0.5567502986857825, micro F1: 0.5567502986857825, macro F1: 0.5567502986857825 @@ -11749,8 +11749,8 @@ Question 837: Where was Girls (The Prodigy song) recorded ? [INFO] Golden Answer: [http://dbpedia.org/resource/England, http://dbpedia.org/resource/Essex] [INFO] Predicted Answer: [http://dbpedia.org/resource/England, http://dbpedia.org/resource/Essex] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 838, P: 0.499, R: 0.555, macro F1: 0.508, macro F1*: 0.525 -[INFO] Current time: 849 ms, average: 2905.27 ms/ques, entity detection: 496.95 ms/ques (linking tool: 404.14 ms/ques), relation detection: 941.58 ms/ques (similarity: 709.21 ms/ques), EDG generation: 46.32 ms/ques +[INFO] Cumulative metrics, sample: 838, P: 0.499, R: 0.555, macro F1: 0.525 +[INFO] Current question consumed time: 718 ms, average: 2606.04 ms/ques, entity detection: 436.53 ms/ques (linking tool: 358.88 ms/ques), relation detection: 866.61 ms/ques (similarity: 702.20 ms/ques), EDG generation: 46.65 ms/ques [INFO] Entity Linking P: 0.7350835322195705, R: 0.6873508353221957, micro F1: 0.7026650755767705, macro F1: 0.7104162995947525 [INFO] Relation Linking P: 0.5917859984089101, R: 0.5375894988066826, micro F1: 0.5531424025457448, macro F1: 0.5633873571187054 [INFO] Type Linking P: 0.5572792362768496, R: 0.5572792362768496, micro F1: 0.5572792362768496, macro F1: 0.5572792362768496 @@ -11763,8 +11763,8 @@ Question 838: How many TV shows' networks are headquarterd in NY? [INFO] Golden Answer: [1121] [INFO] Predicted Answer: [] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 839, P: 0.498, R: 0.554, macro F1: 0.507, macro F1*: 0.525 -[INFO] Current time: 548 ms, average: 2902.46 ms/ques, entity detection: 496.35 ms/ques (linking tool: 403.66 ms/ques), relation detection: 940.78 ms/ques (similarity: 708.57 ms/ques), EDG generation: 46.28 ms/ques +[INFO] Cumulative metrics, sample: 839, P: 0.498, R: 0.554, macro F1: 0.525 +[INFO] Current question consumed time: 453 ms, average: 2603.47 ms/ques, entity detection: 436.03 ms/ques (linking tool: 358.47 ms/ques), relation detection: 865.86 ms/ques (similarity: 701.55 ms/ques), EDG generation: 46.62 ms/ques [INFO] Entity Linking P: 0.734207389749702, R: 0.6865315852205006, micro F1: 0.7018275725069532, macro F1: 0.7095695578789066 [INFO] Relation Linking P: 0.5910806515693285, R: 0.5369487485101311, micro F1: 0.5524831148192302, macro F1: 0.5627158584809 [INFO] Type Linking P: 0.5566150178784267, R: 0.5566150178784267, micro F1: 0.5566150178784267, macro F1: 0.5566150178784267 @@ -11777,8 +11777,8 @@ Question 839: Does StandBy records distribute via Sony? [INFO] Golden Answer: [true] [INFO] Predicted Answer: [true] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 840, P: 0.499, R: 0.555, macro F1: 0.508, macro F1*: 0.525 -[INFO] Current time: 1613 ms, average: 2900.92 ms/ques, entity detection: 496.23 ms/ques (linking tool: 403.50 ms/ques), relation detection: 940.45 ms/ques (similarity: 708.15 ms/ques), EDG generation: 46.23 ms/ques +[INFO] Cumulative metrics, sample: 840, P: 0.499, R: 0.555, macro F1: 0.525 +[INFO] Current question consumed time: 1295 ms, average: 2601.92 ms/ques, entity detection: 435.82 ms/ques (linking tool: 358.27 ms/ques), relation detection: 865.52 ms/ques (similarity: 701.13 ms/ques), EDG generation: 46.56 ms/ques [INFO] Entity Linking P: 0.7345238095238096, R: 0.6863095238095238, micro F1: 0.7017857142857147, macro F1: 0.7095986195064536 [INFO] Relation Linking P: 0.5915674603174603, R: 0.5375, micro F1: 0.553015873015874, macro F1: 0.5632391705474037 [INFO] Type Linking P: 0.5571428571428572, R: 0.5571428571428572, micro F1: 0.5571428571428572, macro F1: 0.5571428571428572 @@ -11791,8 +11791,8 @@ Question 840: Give me some shows owned by BBC one and BBC hd? [INFO] Golden Answer: [http://dbpedia.org/resource/Doctors_(TV_series), http://dbpedia.org/resource/The_One_Show, http://dbpedia.org/resource/Lark_Rise_to_Candleford_(TV_series), http://dbpedia.org/resource/Doctor_Who, http://dbpedia.org/resource/Rock_&_Chips, http://dbpedia.org/resource/In_with_the_Flynns, http://dbpedia.org/resource/New_Year_Live, http://dbpedia.org/resource/Hunter_(British_TV_series), http://dbpedia.org/resource/Ashes_to_Ashes_(TV_series), http://dbpedia.org/resource/Luther_(TV_series), http://dbpedia.org/resource/Judge_John_Deed, http://dbpedia.org/resource/Michael_McIntyre's_Comedy_Roadshow, http://dbpedia.org/resource/QI, http://dbpedia.org/resource/Doctor_Who_Confidential, http://dbpedia.org/resource/The_Sarah_Jane_Adventures, http://dbpedia.org/resource/Waterloo_Road_(TV_series), http://dbpedia.org/resource/Friday_Night_with_Jonathan_Ross, http://dbpedia.org/resource/Last_of_the_Summer_Wine, http://dbpedia.org/resource/Mistresses_(UK_TV_series), http://dbpedia.org/resource/Apparitions_(TV_series), http://dbpedia.org/resource/Around_the_World_in_80_Days_(2009_TV_series), http://dbpedia.org/resource/Hotel_Babylon, http://dbpedia.org/resource/After_You've_Gone_(TV_series), http://dbpedia.org/resource/Big_&_Small, http://dbpedia.org/resource/The_No._1_Ladies'_Detective_Agency_(TV_series), http://dbpedia.org/resource/Land_Girls_(TV_series), http://dbpedia.org/resource/Sorted_(TV_series), http://dbpedia.org/resource/Britain_from_Above, http://dbpedia.org/resource/Doctor_Who_(series_5), http://dbpedia.org/resource/Survivors_(2008_TV_series), http://dbpedia.org/resource/Hope_Springs_(TV_series), http://dbpedia.org/resource/Prisoners'_Wives, http://dbpedia.org/resource/The_Day_of_the_Triffids_(2009_TV_miniseries)] [INFO] Predicted Answer: [http://dbpedia.org/resource/BBC] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 841, P: 0.498, R: 0.554, macro F1: 0.507, macro F1*: 0.525 -[INFO] Current time: 6058 ms, average: 2904.68 ms/ques, entity detection: 498.33 ms/ques (linking tool: 405.61 ms/ques), relation detection: 940.44 ms/ques (similarity: 708.37 ms/ques), EDG generation: 46.22 ms/ques +[INFO] Cumulative metrics, sample: 841, P: 0.498, R: 0.554, macro F1: 0.525 +[INFO] Current question consumed time: 5054 ms, average: 2604.83 ms/ques, entity detection: 437.52 ms/ques (linking tool: 359.99 ms/ques), relation detection: 865.48 ms/ques (similarity: 701.35 ms/ques), EDG generation: 46.55 ms/ques [INFO] Entity Linking P: 0.7348394768133175, R: 0.6866825208085613, micro F1: 0.7021403091557673, macro F1: 0.7099452912750998 [INFO] Relation Linking P: 0.590864050733254, R: 0.5368608799048752, micro F1: 0.5523583036068181, macro F1: 0.5625694450176206 [INFO] Type Linking P: 0.5564803804994055, R: 0.5564803804994055, micro F1: 0.5564803804994055, macro F1: 0.5564803804994055 @@ -11805,8 +11805,8 @@ Question 841: How many teams have rented stadiums in Canada? [INFO] Golden Answer: [111] [INFO] Predicted Answer: [22] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 842, P: 0.498, R: 0.553, macro F1: 0.506, macro F1*: 0.524 -[INFO] Current time: 19512 ms, average: 2924.40 ms/ques, entity detection: 503.95 ms/ques (linking tool: 411.21 ms/ques), relation detection: 946.14 ms/ques (similarity: 711.92 ms/ques), EDG generation: 46.17 ms/ques +[INFO] Cumulative metrics, sample: 842, P: 0.498, R: 0.553, macro F1: 0.524 +[INFO] Current question consumed time: 16637 ms, average: 2621.50 ms/ques, entity detection: 442.27 ms/ques (linking tool: 364.74 ms/ques), relation detection: 870.49 ms/ques (similarity: 704.93 ms/ques), EDG generation: 46.51 ms/ques [INFO] Entity Linking P: 0.7351543942992874, R: 0.6870546318289786, micro F1: 0.7024940617577201, macro F1: 0.7102911350348856 [INFO] Relation Linking P: 0.590162311955661, R: 0.5362232779097387, micro F1: 0.5517022961203493, macro F1: 0.5619013102848206 [INFO] Type Linking P: 0.5570071258907363, R: 0.5570071258907363, micro F1: 0.5570071258907363, macro F1: 0.5570071258907363 @@ -11819,8 +11819,8 @@ Question 842: List the awards won by the film director of Chicken Run? [INFO] Golden Answer: [http://dbpedia.org/resource/Academy_Awards, http://dbpedia.org/resource/Order_of_the_British_Empire, http://dbpedia.org/resource/Royal_Designers_for_Industry] [INFO] Predicted Answer: [http://dbpedia.org/resource/Academy_Awards, http://dbpedia.org/resource/Order_of_the_British_Empire, http://dbpedia.org/resource/Royal_Designers_for_Industry] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 843, P: 0.498, R: 0.554, macro F1: 0.507, macro F1*: 0.525 -[INFO] Current time: 1831 ms, average: 2923.10 ms/ques, entity detection: 503.63 ms/ques (linking tool: 410.91 ms/ques), relation detection: 946.04 ms/ques (similarity: 711.70 ms/ques), EDG generation: 46.23 ms/ques +[INFO] Cumulative metrics, sample: 843, P: 0.498, R: 0.554, macro F1: 0.525 +[INFO] Current question consumed time: 1631 ms, average: 2620.32 ms/ques, entity detection: 441.98 ms/ques (linking tool: 364.47 ms/ques), relation detection: 870.39 ms/ques (similarity: 704.72 ms/ques), EDG generation: 46.56 ms/ques [INFO] Entity Linking P: 0.7354685646500593, R: 0.6874258600237247, micro F1: 0.7028469750889684, macro F1: 0.7106361537552469 [INFO] Relation Linking P: 0.5906484776591537, R: 0.536773428232503, micro F1: 0.5522340846184272, macro F1: 0.562423714807401 [INFO] Type Linking P: 0.5575326215895611, R: 0.5575326215895611, micro F1: 0.5575326215895611, macro F1: 0.5575326215895611 @@ -11833,8 +11833,8 @@ Question 843: What territory of Sino French War was the battle of Raoul Magrin-V [INFO] Golden Answer: [http://dbpedia.org/resource/Tonkin] [INFO] Predicted Answer: [http://dbpedia.org/resource/Annam_(French_protectorate), http://dbpedia.org/resource/Tonkin, French protectorate over Tonkin and Annam. 1887 boundary convention signed between France and China places the Spratly and Paracel islands under French rule.@en] [INFO] P: 0.3333333333333333, R: 1.0, F: 0.5 -[INFO] Cumulative metrics, sample: 844, P: 0.498, R: 0.555, macro F1: 0.507, macro F1*: 0.525 -[INFO] Current time: 1516 ms, average: 2921.43 ms/ques, entity detection: 503.40 ms/ques (linking tool: 410.69 ms/ques), relation detection: 945.28 ms/ques (similarity: 711.08 ms/ques), EDG generation: 46.21 ms/ques +[INFO] Cumulative metrics, sample: 844, P: 0.498, R: 0.555, macro F1: 0.525 +[INFO] Current question consumed time: 1350 ms, average: 2618.82 ms/ques, entity detection: 441.76 ms/ques (linking tool: 364.26 ms/ques), relation detection: 869.70 ms/ques (similarity: 704.11 ms/ques), EDG generation: 46.55 ms/ques [INFO] Entity Linking P: 0.735781990521327, R: 0.6872037914691943, micro F1: 0.7028041074249609, macro F1: 0.7106637044169352 [INFO] Relation Linking P: 0.5911334913112164, R: 0.5367298578199052, micro F1: 0.5523696682464464, macro F1: 0.5626195673234303 [INFO] Type Linking P: 0.5580568720379147, R: 0.5580568720379147, micro F1: 0.5580568720379147, macro F1: 0.5580568720379147 @@ -11847,8 +11847,8 @@ Question 844: List the outflow of Lake Uniamsi? [INFO] Golden Answer: [http://dbpedia.org/resource/Nile, http://dbpedia.org/resource/Congo_River, http://dbpedia.org/resource/Zambezi, http://dbpedia.org/resource/Benué_River] [INFO] Predicted Answer: [http://dbpedia.org/resource/White_Nile] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 845, P: 0.498, R: 0.554, macro F1: 0.506, macro F1*: 0.524 -[INFO] Current time: 1545 ms, average: 2919.81 ms/ques, entity detection: 503.44 ms/ques (linking tool: 410.75 ms/ques), relation detection: 944.59 ms/ques (similarity: 710.35 ms/ques), EDG generation: 46.17 ms/ques +[INFO] Cumulative metrics, sample: 845, P: 0.498, R: 0.554, macro F1: 0.524 +[INFO] Current question consumed time: 1465 ms, average: 2617.45 ms/ques, entity detection: 441.87 ms/ques (linking tool: 364.38 ms/ques), relation detection: 869.06 ms/ques (similarity: 703.38 ms/ques), EDG generation: 46.50 ms/ques [INFO] Entity Linking P: 0.7349112426035503, R: 0.6863905325443787, micro F1: 0.7019723865877716, macro F1: 0.7098226822815306 [INFO] Relation Linking P: 0.5916173570019724, R: 0.5372781065088758, micro F1: 0.5528994082840246, macro F1: 0.5631399250365591 [INFO] Type Linking P: 0.5585798816568047, R: 0.5585798816568047, micro F1: 0.5585798816568047, macro F1: 0.5585798816568047 @@ -11861,8 +11861,8 @@ Question 845: Did John Byrne create Emma Frost? [INFO] Golden Answer: [true] [INFO] Predicted Answer: [true] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 846, P: 0.498, R: 0.554, macro F1: 0.507, macro F1*: 0.525 -[INFO] Current time: 1147 ms, average: 2917.71 ms/ques, entity detection: 503.35 ms/ques (linking tool: 410.58 ms/ques), relation detection: 943.66 ms/ques (similarity: 709.63 ms/ques), EDG generation: 46.12 ms/ques +[INFO] Cumulative metrics, sample: 846, P: 0.498, R: 0.554, macro F1: 0.525 +[INFO] Current question consumed time: 1057 ms, average: 2615.61 ms/ques, entity detection: 441.81 ms/ques (linking tool: 364.23 ms/ques), relation detection: 868.21 ms/ques (similarity: 702.67 ms/ques), EDG generation: 46.46 ms/ques [INFO] Entity Linking P: 0.735224586288416, R: 0.6867612293144209, micro F1: 0.7023246650906229, macro F1: 0.7101670567474143 [INFO] Relation Linking P: 0.5921000788022064, R: 0.5378250591016549, micro F1: 0.5534278959810884, macro F1: 0.5636590411054044 [INFO] Type Linking P: 0.5591016548463357, R: 0.5591016548463357, micro F1: 0.5591016548463357, macro F1: 0.5591016548463357 @@ -11875,8 +11875,8 @@ Question 846: Was Heinrich Himmler ever a commanders of Army Group Oberrhein? [INFO] Golden Answer: [true] [INFO] Predicted Answer: [true] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 847, P: 0.499, R: 0.555, macro F1: 0.508, macro F1*: 0.525 -[INFO] Current time: 992 ms, average: 2915.44 ms/ques, entity detection: 503.24 ms/ques (linking tool: 410.39 ms/ques), relation detection: 942.69 ms/ques (similarity: 708.88 ms/ques), EDG generation: 46.09 ms/ques +[INFO] Cumulative metrics, sample: 847, P: 0.499, R: 0.555, macro F1: 0.525 +[INFO] Current question consumed time: 916 ms, average: 2613.60 ms/ques, entity detection: 441.74 ms/ques (linking tool: 364.08 ms/ques), relation detection: 867.33 ms/ques (similarity: 701.93 ms/ques), EDG generation: 46.42 ms/ques [INFO] Entity Linking P: 0.7355371900826446, R: 0.6871310507674144, micro F1: 0.7026761117670212, macro F1: 0.710510613490621 [INFO] Relation Linking P: 0.591401023219205, R: 0.5371900826446281, micro F1: 0.5527744982290446, macro F1: 0.562993564079306 [INFO] Type Linking P: 0.5596221959858324, R: 0.5596221959858324, micro F1: 0.5596221959858324, macro F1: 0.5596221959858324 @@ -11889,8 +11889,8 @@ Question 847: List the universities from a country where Norwegian language is o [INFO] Golden Answer: [http://dbpedia.org/resource/University_of_Bergen, http://dbpedia.org/resource/Harstad_University_College, http://dbpedia.org/resource/Nesna_University_College, http://dbpedia.org/resource/Nord-Trøndelag_University_College, http://dbpedia.org/resource/Noroff, http://dbpedia.org/resource/Stord/Haugesund_University_College, http://dbpedia.org/resource/Sámi_University_College, http://dbpedia.org/resource/University_of_Nordland, http://dbpedia.org/resource/Østfold_University_College, http://dbpedia.org/resource/Nordic_Institute_of_Stage_and_Studio, http://dbpedia.org/resource/Royal_Norwegian_Naval_Academy, http://dbpedia.org/resource/Royal_Norwegian_Air_Force_Academy, http://dbpedia.org/resource/Nord_University, http://dbpedia.org/resource/Vestfold_University_College, http://dbpedia.org/resource/Barratt_Due_Institute_of_Music, http://dbpedia.org/resource/Buskerud_and_Vestfold_University_College, http://dbpedia.org/resource/Hedmark_University_College, http://dbpedia.org/resource/Norwegian_Defence_School_of_Engineering, http://dbpedia.org/resource/Oslo_and_Akershus_University_College, http://dbpedia.org/resource/Sør-Trøndelag_University_College, http://dbpedia.org/resource/Birralee_International_School_Trondheim, http://dbpedia.org/resource/North_European_Institute_of_Aviation, http://dbpedia.org/resource/Gjøvik_University_College, http://dbpedia.org/resource/Chateau_Neuf, http://dbpedia.org/resource/Akershus_University_College, http://dbpedia.org/resource/Hartvig_Nissen_School, http://dbpedia.org/resource/Rudolf_Steiner_University_College, http://dbpedia.org/resource/Norwegian_University_of_Science_and_Technology, http://dbpedia.org/resource/University_of_Oslo, http://dbpedia.org/resource/University_of_the_Arctic, http://dbpedia.org/resource/Norwegian_Police_University_College, http://dbpedia.org/resource/University_of_Stavanger, http://dbpedia.org/resource/International_School_of_Bergen, http://dbpedia.org/resource/University_of_Agder, http://dbpedia.org/resource/Bergen_University_College, http://dbpedia.org/resource/Norwegian_Confederation_of_Trade_Unions, http://dbpedia.org/resource/Queen_Maud_University_College, http://dbpedia.org/resource/Oslo_Business_School, http://dbpedia.org/resource/University_of_Tromsø, http://dbpedia.org/resource/BI_Norwegian_Business_School, http://dbpedia.org/resource/MF_Norwegian_School_of_Theology, http://dbpedia.org/resource/Norwegian_School_of_Information_Technology, http://dbpedia.org/resource/Norwegian_School_of_Sport_Sciences, http://dbpedia.org/resource/Norwegian_School_of_Veterinary_Science, http://dbpedia.org/resource/Norwegian_University_of_Life_Sciences, http://dbpedia.org/resource/Oslo_National_Academy_of_the_Arts, http://dbpedia.org/resource/Oslo_School_of_Architecture_and_Design, http://dbpedia.org/resource/Oslo_University_College, http://dbpedia.org/resource/University_Centre_in_Svalbard, http://dbpedia.org/resource/Norwegian_School_of_Economics, http://dbpedia.org/resource/North_European_Aviation_Resources, http://dbpedia.org/resource/Lillehammer_University_College, http://dbpedia.org/resource/Norwegian_Defence_Research_Establishment, http://dbpedia.org/resource/University_of_Oslo_Faculty_of_Law, http://dbpedia.org/resource/Molde_University_College, http://dbpedia.org/resource/Tromsø_University_College, http://dbpedia.org/resource/Aalesund_University_College, http://dbpedia.org/resource/St._Olav's_University_Hospital] [INFO] Predicted Answer: [http://dbpedia.org/resource/Norway, http://dbpedia.org/resource/Ladonia_(micronation), http://dbpedia.org/resource/Svalbard] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 848, P: 0.498, R: 0.554, macro F1: 0.507, macro F1*: 0.525 -[INFO] Current time: 1735 ms, average: 2914.04 ms/ques, entity detection: 502.79 ms/ques (linking tool: 410.00 ms/ques), relation detection: 942.22 ms/ques (similarity: 708.91 ms/ques), EDG generation: 46.06 ms/ques +[INFO] Cumulative metrics, sample: 848, P: 0.498, R: 0.554, macro F1: 0.525 +[INFO] Current question consumed time: 1667 ms, average: 2612.49 ms/ques, entity detection: 441.37 ms/ques (linking tool: 363.75 ms/ques), relation detection: 866.93 ms/ques (similarity: 701.98 ms/ques), EDG generation: 46.39 ms/ques [INFO] Entity Linking P: 0.7358490566037735, R: 0.6875, micro F1: 0.7030267295597488, macro F1: 0.7108533554266777 [INFO] Relation Linking P: 0.5918828616352201, R: 0.5371462264150944, micro F1: 0.5529088050314475, macro F1: 0.5631876963527048 [INFO] Type Linking P: 0.5589622641509434, R: 0.5589622641509434, micro F1: 0.5589622641509434, macro F1: 0.5589622641509434 @@ -11903,8 +11903,8 @@ Question 848: Name the incumbent of Linda Jenness ? [INFO] Golden Answer: [http://dbpedia.org/resource/Richard_Nixon] [INFO] Predicted Answer: [http://dbpedia.org/resource/Richard_Nixon] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 849, P: 0.499, R: 0.555, macro F1: 0.507, macro F1*: 0.525 -[INFO] Current time: 654 ms, average: 2911.38 ms/ques, entity detection: 502.43 ms/ques (linking tool: 409.62 ms/ques), relation detection: 941.26 ms/ques (similarity: 708.16 ms/ques), EDG generation: 46.02 ms/ques +[INFO] Cumulative metrics, sample: 849, P: 0.499, R: 0.555, macro F1: 0.525 +[INFO] Current question consumed time: 564 ms, average: 2610.07 ms/ques, entity detection: 441.04 ms/ques (linking tool: 363.41 ms/ques), relation detection: 866.05 ms/ques (similarity: 701.23 ms/ques), EDG generation: 46.35 ms/ques [INFO] Entity Linking P: 0.7361601884570083, R: 0.6878680800942285, micro F1: 0.7033765213977232, macro F1: 0.7111952854572257 [INFO] Relation Linking P: 0.5923635649784059, R: 0.5376914016489989, micro F1: 0.5534354142128003, macro F1: 0.5637049611659356 [INFO] Type Linking P: 0.5594817432273262, R: 0.5594817432273262, micro F1: 0.5594817432273262, macro F1: 0.5594817432273262 @@ -11917,8 +11917,8 @@ Question 849: Which companies makes cars assembled in Karachi, Pakistan? [INFO] Golden Answer: [http://dbpedia.org/resource/General_Motors] [INFO] Predicted Answer: [http://dbpedia.org/resource/Renault, http://dbpedia.org/resource/Toyota, http://dbpedia.org/resource/Hyundai_Motor_Company, http://dbpedia.org/resource/Honda, http://dbpedia.org/resource/Adam_Motor_Company, http://dbpedia.org/resource/Suzuki, http://dbpedia.org/resource/GM_Korea] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 850, P: 0.498, R: 0.554, macro F1: 0.507, macro F1*: 0.525 -[INFO] Current time: 11055 ms, average: 2920.96 ms/ques, entity detection: 503.94 ms/ques (linking tool: 411.06 ms/ques), relation detection: 944.39 ms/ques (similarity: 711.36 ms/ques), EDG generation: 45.98 ms/ques +[INFO] Cumulative metrics, sample: 850, P: 0.498, R: 0.554, macro F1: 0.525 +[INFO] Current question consumed time: 9956 ms, average: 2618.72 ms/ques, entity detection: 442.38 ms/ques (linking tool: 364.70 ms/ques), relation detection: 868.92 ms/ques (similarity: 704.43 ms/ques), EDG generation: 46.31 ms/ques [INFO] Entity Linking P: 0.7352941176470589, R: 0.6870588235294117, micro F1: 0.7025490196078434, macro F1: 0.7103585851213934 [INFO] Relation Linking P: 0.5922549019607842, R: 0.5376470588235294, micro F1: 0.5533725490196088, macro F1: 0.5636313895623325 [INFO] Type Linking P: 0.5588235294117647, R: 0.5588235294117647, micro F1: 0.5588235294117647, macro F1: 0.5588235294117647 @@ -11931,8 +11931,8 @@ Question 850: Which company founded by Fusajiro Yamauchi also provides services [INFO] Golden Answer: [http://dbpedia.org/resource/Nintendo] [INFO] Predicted Answer: [http://dbpedia.org/resource/Nintendo] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 851, P: 0.499, R: 0.555, macro F1: 0.507, macro F1*: 0.525 -[INFO] Current time: 2315 ms, average: 2920.25 ms/ques, entity detection: 504.17 ms/ques (linking tool: 411.18 ms/ques), relation detection: 943.73 ms/ques (similarity: 710.97 ms/ques), EDG generation: 45.96 ms/ques +[INFO] Cumulative metrics, sample: 851, P: 0.499, R: 0.555, macro F1: 0.525 +[INFO] Current question consumed time: 1845 ms, average: 2617.81 ms/ques, entity detection: 442.47 ms/ques (linking tool: 364.72 ms/ques), relation detection: 868.27 ms/ques (similarity: 704.04 ms/ques), EDG generation: 46.29 ms/ques [INFO] Entity Linking P: 0.7356051703877791, R: 0.6874265569917744, micro F1: 0.702898550724638, macro F1: 0.7107002884836511 [INFO] Relation Linking P: 0.5927340383862123, R: 0.5381903642773208, micro F1: 0.553897375636507, macro F1: 0.564146900159429 [INFO] Type Linking P: 0.5581668625146886, R: 0.5581668625146886, micro F1: 0.5581668625146886, macro F1: 0.5581668625146886 @@ -11945,8 +11945,8 @@ Question 851: In which countries does the sea lie on whose shore is the city of [INFO] Golden Answer: [http://dbpedia.org/resource/Antigua_and_Barbuda, http://dbpedia.org/resource/Aruba, http://dbpedia.org/resource/Belize, http://dbpedia.org/resource/Colombia, http://dbpedia.org/resource/Costa_Rica, http://dbpedia.org/resource/Cuba, http://dbpedia.org/resource/Curaçao, http://dbpedia.org/resource/Dominican_Republic, http://dbpedia.org/resource/Guatemala, http://dbpedia.org/resource/Haiti, http://dbpedia.org/resource/Honduras, http://dbpedia.org/resource/Jamaica, http://dbpedia.org/resource/Lesser_Antilles, http://dbpedia.org/resource/Mexico, http://dbpedia.org/resource/Nicaragua, http://dbpedia.org/resource/Panama, http://dbpedia.org/resource/Puerto_Rico, http://dbpedia.org/resource/Trinidad_and_Tobago, http://dbpedia.org/resource/United_States, http://dbpedia.org/resource/Venezuela] [INFO] Predicted Answer: [http://dbpedia.org/resource/Pakistan] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 852, P: 0.498, R: 0.554, macro F1: 0.507, macro F1*: 0.525 -[INFO] Current time: 5404 ms, average: 2923.17 ms/ques, entity detection: 504.05 ms/ques (linking tool: 410.96 ms/ques), relation detection: 945.10 ms/ques (similarity: 712.10 ms/ques), EDG generation: 45.95 ms/ques +[INFO] Cumulative metrics, sample: 852, P: 0.498, R: 0.554, macro F1: 0.525 +[INFO] Current question consumed time: 4968 ms, average: 2620.56 ms/ques, entity detection: 442.37 ms/ques (linking tool: 364.52 ms/ques), relation detection: 869.59 ms/ques (similarity: 705.20 ms/ques), EDG generation: 46.28 ms/ques [INFO] Entity Linking P: 0.7347417840375586, R: 0.6866197183098591, micro F1: 0.7020735524256655, macro F1: 0.7098661332154778 [INFO] Relation Linking P: 0.5926251956181533, R: 0.5381455399061033, micro F1: 0.5538341158059478, macro F1: 0.5640729740145454 [INFO] Type Linking P: 0.5575117370892019, R: 0.5575117370892019, micro F1: 0.5575117370892019, macro F1: 0.5575117370892019 @@ -11959,8 +11959,8 @@ Question 852: Tirana is the largest city of which country? [INFO] Golden Answer: [http://dbpedia.org/resource/Albania] [INFO] Predicted Answer: [http://dbpedia.org/resource/Albania] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 853, P: 0.499, R: 0.555, macro F1: 0.507, macro F1*: 0.525 -[INFO] Current time: 4604 ms, average: 2925.14 ms/ques, entity detection: 503.83 ms/ques (linking tool: 410.74 ms/ques), relation detection: 946.31 ms/ques (similarity: 713.87 ms/ques), EDG generation: 45.91 ms/ques +[INFO] Cumulative metrics, sample: 853, P: 0.499, R: 0.555, macro F1: 0.525 +[INFO] Current question consumed time: 4204 ms, average: 2622.42 ms/ques, entity detection: 442.18 ms/ques (linking tool: 364.33 ms/ques), relation detection: 870.68 ms/ques (similarity: 707.02 ms/ques), EDG generation: 46.24 ms/ques [INFO] Entity Linking P: 0.735052754982415, R: 0.6869871043376319, micro F1: 0.7024228214146154, macro F1: 0.7102076082764964 [INFO] Relation Linking P: 0.5931027745212973, R: 0.5386869871043376, micro F1: 0.5543571707698329, macro F1: 0.5645867412534197 [INFO] Type Linking P: 0.5568581477139508, R: 0.5568581477139508, micro F1: 0.5568581477139508, macro F1: 0.5568581477139508 @@ -11973,8 +11973,8 @@ Question 853: From which countries are the people from who are an academic autho [INFO] Golden Answer: [http://dbpedia.org/resource/France] [INFO] Predicted Answer: [http://dbpedia.org/resource/France] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 854, P: 0.499, R: 0.555, macro F1: 0.508, macro F1*: 0.526 -[INFO] Current time: 2166 ms, average: 2924.25 ms/ques, entity detection: 503.35 ms/ques (linking tool: 410.35 ms/ques), relation detection: 945.66 ms/ques (similarity: 713.39 ms/ques), EDG generation: 45.90 ms/ques +[INFO] Cumulative metrics, sample: 854, P: 0.499, R: 0.555, macro F1: 0.526 +[INFO] Current question consumed time: 1921 ms, average: 2621.60 ms/ques, entity detection: 441.76 ms/ques (linking tool: 363.98 ms/ques), relation detection: 870.07 ms/ques (similarity: 706.54 ms/ques), EDG generation: 46.22 ms/ques [INFO] Entity Linking P: 0.7353629976580797, R: 0.6873536299765808, micro F1: 0.702771272443404, macro F1: 0.7105482792185889 [INFO] Relation Linking P: 0.5935792349726775, R: 0.5392271662763466, micro F1: 0.5548790007806411, macro F1: 0.5650992940751168 [INFO] Type Linking P: 0.5562060889929742, R: 0.5562060889929742, micro F1: 0.5562060889929742, macro F1: 0.5562060889929742 @@ -11987,8 +11987,8 @@ Question 854: Where is the headquarters of Sigma TV [INFO] Golden Answer: [http://dbpedia.org/resource/Cyprus, http://dbpedia.org/resource/Nicosia] [INFO] Predicted Answer: [http://dbpedia.org/resource/Cyprus, http://dbpedia.org/resource/Nicosia] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 855, P: 0.500, R: 0.556, macro F1: 0.509, macro F1*: 0.526 -[INFO] Current time: 800 ms, average: 2921.76 ms/ques, entity detection: 502.97 ms/ques (linking tool: 410.02 ms/ques), relation detection: 944.78 ms/ques (similarity: 712.70 ms/ques), EDG generation: 45.86 ms/ques +[INFO] Cumulative metrics, sample: 855, P: 0.500, R: 0.556, macro F1: 0.526 +[INFO] Current question consumed time: 678 ms, average: 2619.33 ms/ques, entity detection: 441.40 ms/ques (linking tool: 363.66 ms/ques), relation detection: 869.27 ms/ques (similarity: 705.88 ms/ques), EDG generation: 46.17 ms/ques [INFO] Entity Linking P: 0.735672514619883, R: 0.6877192982456141, micro F1: 0.7031189083820666, macro F1: 0.7108881488849487 [INFO] Relation Linking P: 0.5940545808966862, R: 0.539766081871345, micro F1: 0.5553996101364532, macro F1: 0.5656106368100831 [INFO] Type Linking P: 0.5567251461988304, R: 0.5567251461988304, micro F1: 0.5567251461988304, macro F1: 0.5567251461988304 @@ -12001,8 +12001,8 @@ Question 855: Which scientist was advised by John Robert Woodyard? [INFO] Golden Answer: [http://dbpedia.org/resource/Douglas_Engelbart] [INFO] Predicted Answer: [http://dbpedia.org/resource/Douglas_Engelbart] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 856, P: 0.501, R: 0.556, macro F1: 0.509, macro F1*: 0.527 -[INFO] Current time: 919 ms, average: 2919.42 ms/ques, entity detection: 502.62 ms/ques (linking tool: 409.66 ms/ques), relation detection: 943.90 ms/ques (similarity: 712.08 ms/ques), EDG generation: 45.82 ms/ques +[INFO] Cumulative metrics, sample: 856, P: 0.501, R: 0.556, macro F1: 0.527 +[INFO] Current question consumed time: 837 ms, average: 2617.24 ms/ques, entity detection: 441.09 ms/ques (linking tool: 363.34 ms/ques), relation detection: 868.46 ms/ques (similarity: 705.27 ms/ques), EDG generation: 46.13 ms/ques [INFO] Entity Linking P: 0.735981308411215, R: 0.6880841121495327, micro F1: 0.7034657320872277, macro F1: 0.7112272201053416 [INFO] Relation Linking P: 0.5945288161993769, R: 0.540303738317757, micro F1: 0.5559190031152658, macro F1: 0.5661207737679579 [INFO] Type Linking P: 0.5560747663551402, R: 0.5560747663551402, micro F1: 0.5560747663551402, macro F1: 0.5560747663551402 @@ -12015,8 +12015,8 @@ Question 856: What is the launch site of the Juno I which was also the ground of [INFO] Golden Answer: [http://dbpedia.org/resource/Florida] [INFO] Predicted Answer: [http://dbpedia.org/resource/Florida] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 857, P: 0.501, R: 0.557, macro F1: 0.510, macro F1*: 0.527 -[INFO] Current time: 2414 ms, average: 2918.83 ms/ques, entity detection: 502.55 ms/ques (linking tool: 409.54 ms/ques), relation detection: 943.71 ms/ques (similarity: 711.78 ms/ques), EDG generation: 45.91 ms/ques +[INFO] Cumulative metrics, sample: 857, P: 0.501, R: 0.557, macro F1: 0.527 +[INFO] Current question consumed time: 2195 ms, average: 2616.75 ms/ques, entity detection: 441.03 ms/ques (linking tool: 363.25 ms/ques), relation detection: 868.28 ms/ques (similarity: 705.01 ms/ques), EDG generation: 46.28 ms/ques [INFO] Entity Linking P: 0.7362893815635939, R: 0.6884480746791132, micro F1: 0.7038117464021785, macro F1: 0.7115654956961841 [INFO] Relation Linking P: 0.5950019447685725, R: 0.5408401400233372, micro F1: 0.5564371839751079, macro F1: 0.5666297092377024 [INFO] Type Linking P: 0.5565927654609102, R: 0.5565927654609102, micro F1: 0.5565927654609102, macro F1: 0.5565927654609102 @@ -12029,8 +12029,8 @@ Question 857: Who has been married to both Penny Lancaster and Alana Stewart? [INFO] Golden Answer: [http://dbpedia.org/resource/Rod_Stewart] [INFO] Predicted Answer: [http://dbpedia.org/resource/Rod_Stewart] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 858, P: 0.502, R: 0.557, macro F1: 0.510, macro F1*: 0.528 -[INFO] Current time: 1607 ms, average: 2917.31 ms/ques, entity detection: 502.36 ms/ques (linking tool: 409.34 ms/ques), relation detection: 943.07 ms/ques (similarity: 711.24 ms/ques), EDG generation: 45.91 ms/ques +[INFO] Cumulative metrics, sample: 858, P: 0.502, R: 0.557, macro F1: 0.528 +[INFO] Current question consumed time: 1383 ms, average: 2615.31 ms/ques, entity detection: 440.84 ms/ques (linking tool: 363.07 ms/ques), relation detection: 867.68 ms/ques (similarity: 704.47 ms/ques), EDG generation: 46.29 ms/ques [INFO] Entity Linking P: 0.7365967365967366, R: 0.6888111888111889, micro F1: 0.7041569541569546, macro F1: 0.7119029784606237 [INFO] Relation Linking P: 0.5954739704739704, R: 0.5413752913752914, micro F1: 0.5569541569541578, macro F1: 0.5671374474877252 [INFO] Type Linking P: 0.5571095571095571, R: 0.5571095571095571, micro F1: 0.5571095571095571, macro F1: 0.5571095571095571 @@ -12043,8 +12043,8 @@ Question 858: Count the affiliations of International Union of Anarchists? [INFO] Golden Answer: [9] [INFO] Predicted Answer: [9] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 859, P: 0.502, R: 0.558, macro F1: 0.511, macro F1*: 0.529 -[INFO] Current time: 847 ms, average: 2914.90 ms/ques, entity detection: 502.00 ms/ques (linking tool: 409.03 ms/ques), relation detection: 942.07 ms/ques (similarity: 710.46 ms/ques), EDG generation: 45.87 ms/ques +[INFO] Cumulative metrics, sample: 859, P: 0.502, R: 0.558, macro F1: 0.529 +[INFO] Current question consumed time: 750 ms, average: 2613.14 ms/ques, entity detection: 440.53 ms/ques (linking tool: 362.82 ms/ques), relation detection: 866.76 ms/ques (similarity: 703.71 ms/ques), EDG generation: 46.25 ms/ques [INFO] Entity Linking P: 0.7369033760186263, R: 0.689173457508731, micro F1: 0.7045013581684132, macro F1: 0.7122396711886151 [INFO] Relation Linking P: 0.5959448971672487, R: 0.5419091967403958, micro F1: 0.5574699262708586, macro F1: 0.5676439927660075 [INFO] Type Linking P: 0.5576251455180442, R: 0.5576251455180442, micro F1: 0.5576251455180442, macro F1: 0.5576251455180442 @@ -12057,8 +12057,8 @@ Question 859: What is the alma mater of the scientist whose PhD advisor is Jean [INFO] Golden Answer: [http://dbpedia.org/resource/Stanford_University] [INFO] Predicted Answer: [http://dbpedia.org/resource/Stanford_University] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 860, P: 0.503, R: 0.558, macro F1: 0.511, macro F1*: 0.529 -[INFO] Current time: 1366 ms, average: 2913.09 ms/ques, entity detection: 501.64 ms/ques (linking tool: 408.66 ms/ques), relation detection: 941.45 ms/ques (similarity: 710.03 ms/ques), EDG generation: 45.87 ms/ques +[INFO] Cumulative metrics, sample: 860, P: 0.503, R: 0.558, macro F1: 0.529 +[INFO] Current question consumed time: 1116 ms, average: 2611.40 ms/ques, entity detection: 440.18 ms/ques (linking tool: 362.48 ms/ques), relation detection: 866.17 ms/ques (similarity: 703.27 ms/ques), EDG generation: 46.25 ms/ques [INFO] Entity Linking P: 0.7372093023255814, R: 0.6895348837209302, micro F1: 0.7048449612403104, macro F1: 0.7125755766570004 [INFO] Relation Linking P: 0.5964147286821705, R: 0.5424418604651163, micro F1: 0.5579844961240319, macro F1: 0.5681493493002282 [INFO] Type Linking P: 0.5569767441860465, R: 0.5569767441860465, micro F1: 0.5569767441860465, macro F1: 0.5569767441860465 @@ -12071,8 +12071,8 @@ Question 860: List the people residing in Wilton, Connecticut? [INFO] Golden Answer: [http://dbpedia.org/resource/Steve_Phillips, http://dbpedia.org/resource/Charles_Marvin_(Connecticut_politician), http://dbpedia.org/resource/Clapp_Raymond, http://dbpedia.org/resource/Christopher_Walken, http://dbpedia.org/resource/Joseph_Birchard, http://dbpedia.org/resource/Benjamin_Hickox, http://dbpedia.org/resource/Georgianne_Walken, http://dbpedia.org/resource/Gail_Lavielle, http://dbpedia.org/resource/Samuel_Marvin] [INFO] Predicted Answer: [http://dbpedia.org/resource/Robert_Vaughn, http://dbpedia.org/resource/Thea_Trinidad, http://dbpedia.org/resource/Peter_Salovey, http://dbpedia.org/resource/Bryan_F._Mahan, http://dbpedia.org/resource/Dolores_Hart, http://dbpedia.org/resource/Edward_M._Kennedy,_Jr., http://dbpedia.org/resource/Erin_Pac, http://dbpedia.org/resource/Garrett_Lucash, http://dbpedia.org/resource/Mae_Flexer, http://dbpedia.org/resource/Penny_Bacchiochi, http://dbpedia.org/resource/Robert_E._De_Forest, http://dbpedia.org/resource/Irving_Freese, http://dbpedia.org/resource/William_A._Collins, http://dbpedia.org/resource/Schuyler_Merritt, http://dbpedia.org/resource/Thomas_Robins_(politician), http://dbpedia.org/resource/George_Buxton, http://dbpedia.org/resource/Louis_Gloria, http://dbpedia.org/resource/Thomas_C._O'Connor, http://dbpedia.org/resource/Charles_E._Dow, http://dbpedia.org/resource/Leeman_M._Brundage, http://dbpedia.org/resource/Donnie_Sellers, http://dbpedia.org/resource/Douglas_Mintz, http://dbpedia.org/resource/Joseph_Clemmons, http://dbpedia.org/resource/Gigi_Fernández, http://dbpedia.org/resource/Christopher_Comstock, http://dbpedia.org/resource/John_Platt_(settler), http://dbpedia.org/resource/William_M._Jennings, http://dbpedia.org/resource/Nathaniel_Raymond, http://dbpedia.org/resource/Lezlye_Zupkus, http://dbpedia.org/resource/Edwin_O._Keeler, http://dbpedia.org/resource/Paul_Gerken, http://dbpedia.org/resource/Stew_Leonard,_Jr., http://dbpedia.org/resource/William_B._Reed_(politician), http://dbpedia.org/resource/Leander_P._Jones, http://dbpedia.org/resource/John_Atkin, http://dbpedia.org/resource/Julian_Lethbridge, http://dbpedia.org/resource/Mark_Sension, http://dbpedia.org/resource/Judith_Sheindlin, http://dbpedia.org/resource/Tony_Hwang, http://dbpedia.org/resource/Levi_Warner, http://dbpedia.org/resource/Robert_D._Scinto, http://dbpedia.org/resource/Lauren_Drain, http://dbpedia.org/resource/Alanna_Schepartz, http://dbpedia.org/resource/Alfred_N._Phillips, http://dbpedia.org/resource/Ebenezer_J._Hill, http://dbpedia.org/resource/Jeremiah_Donovan, http://dbpedia.org/resource/Jeffrey_A._Meyer, http://dbpedia.org/resource/Antonietta_Collins, http://dbpedia.org/resource/James_G._Gregory, http://dbpedia.org/resource/Robert_M._Wolfe, http://dbpedia.org/resource/Asa_Smith_(Connecticut), http://dbpedia.org/resource/Asa_Woodward, http://dbpedia.org/resource/Jennie_Cave, http://dbpedia.org/resource/Peter_L._Cunningham, http://dbpedia.org/resource/Edward_J._Kelley, http://dbpedia.org/resource/Ernesto_Zedillo, http://dbpedia.org/resource/Le_Roy_D._Downs, http://dbpedia.org/resource/Albert_P._Morano, http://dbpedia.org/resource/Samuel_Simons, http://dbpedia.org/resource/Jasper_McLevy, http://dbpedia.org/resource/Slyck_Wagner_Brown, http://dbpedia.org/resource/Diane_Farrell, http://dbpedia.org/resource/Phil_Maymin, http://dbpedia.org/resource/Lawrence_F._Cafero, http://dbpedia.org/resource/Taylor_Sherman, http://dbpedia.org/resource/Benjamin_P._Mead, http://dbpedia.org/resource/Charles_C._Swartz, http://dbpedia.org/resource/R._Jay_Walsh, http://dbpedia.org/resource/Clapp_Raymond, http://dbpedia.org/resource/Eliphalet_Lockwood, http://dbpedia.org/resource/Matthew_Mead_(politician), http://dbpedia.org/resource/Moses_Comstock, http://dbpedia.org/resource/George_Schaller, http://dbpedia.org/resource/David_Cannadine, http://dbpedia.org/resource/Donald_J._Irwin, http://dbpedia.org/resource/Joey_Styles, http://dbpedia.org/resource/Carl_Harstrom, http://dbpedia.org/resource/Antonio_Frasconi, http://dbpedia.org/resource/James_W._Hyatt, http://dbpedia.org/resource/David_Schultz_(professional_wrestler), http://dbpedia.org/resource/Rodney_A._Hawes,_Jr., http://dbpedia.org/resource/Frank_Esposito_(politician), http://dbpedia.org/resource/Richard_A._Moccia, http://dbpedia.org/resource/Sally_Bolster, http://dbpedia.org/resource/Eugénie_Rocherolle, http://dbpedia.org/resource/William_B._Hubbell, http://dbpedia.org/resource/John_Shostak, http://dbpedia.org/resource/Edward_J._Finnegan, http://dbpedia.org/resource/Stephen_St._John, http://dbpedia.org/resource/Albert_Pohlman, http://dbpedia.org/resource/Christian_Swartz, http://dbpedia.org/resource/Clarence_B._Coolidge, http://dbpedia.org/resource/Edwin_Adams_(politician), http://dbpedia.org/resource/Edwin_Wilcox, http://dbpedia.org/resource/Ferdinand_B._Smith, http://dbpedia.org/resource/Francis_Burnell, http://dbpedia.org/resource/Frank_Comstock_(politician), http://dbpedia.org/resource/Frank_T._Stack, http://dbpedia.org/resource/George_Brunjes, http://dbpedia.org/resource/George_Lockwood_(politician), http://dbpedia.org/resource/Jesse_Milton_Coburn, http://dbpedia.org/resource/John_H._Lee_(Connecticut), http://dbpedia.org/resource/Richard_H._Golden, http://dbpedia.org/resource/Robert_B._Oliver, http://dbpedia.org/resource/Samuel_Lynes, http://dbpedia.org/resource/Walter_C._Quintard, http://dbpedia.org/resource/William_Tammany, http://dbpedia.org/resource/Elbert_A._Woodward, http://dbpedia.org/resource/Ray_Dalio, http://dbpedia.org/resource/Mercedes_Martinez, http://dbpedia.org/resource/Derek_Poundstone, http://dbpedia.org/resource/Philip_Rubin, http://dbpedia.org/resource/John_Cavanagh_(hatter), http://dbpedia.org/resource/Paul_W._Kahn, http://dbpedia.org/resource/John_Hayes_(tennis), http://dbpedia.org/resource/A._Homer_Byington, http://dbpedia.org/resource/Anne_Hendricks_Bass, http://dbpedia.org/resource/Edwin_Lockwood, http://dbpedia.org/resource/John_H._Light, http://dbpedia.org/resource/William_C._Street, http://dbpedia.org/resource/William_J._Street, http://dbpedia.org/resource/James_T._Hubbell, http://dbpedia.org/resource/Frederick_Bruggerhof, http://dbpedia.org/resource/Nicholas_Hoyt, http://dbpedia.org/resource/William_O._Harbach, http://dbpedia.org/resource/Calvin_Barton, http://dbpedia.org/resource/Mortimer_M._Lee, http://dbpedia.org/resource/Wallace_Dann, http://dbpedia.org/resource/Frank_J._Cooke, http://dbpedia.org/resource/Stephanie_McMahon, http://dbpedia.org/resource/Edward_Woodruff_Seymour, http://dbpedia.org/resource/Bob_Duff, http://dbpedia.org/resource/Bruce_Morris, http://dbpedia.org/resource/Chris_Perone, http://dbpedia.org/resource/Harry_Rilling, http://dbpedia.org/resource/Terrie_Wood, http://dbpedia.org/resource/Joseph_Mann, http://dbpedia.org/resource/Olivia_Blatchford, http://dbpedia.org/resource/James_Richards_(politician), http://dbpedia.org/resource/Janet_M._Mills, http://dbpedia.org/resource/Harold_L._Nash, http://dbpedia.org/resource/Hezekiah_Rogers, http://dbpedia.org/resource/Heusted_W._R._Hoyt, http://dbpedia.org/resource/Edward_P._Weed, http://dbpedia.org/resource/Winfield_S._Hanford, http://dbpedia.org/resource/Robert_Genuario, http://dbpedia.org/resource/Andrew_J._Santaniello,_Jr., http://dbpedia.org/resource/Joseph_Santo, http://dbpedia.org/resource/Frank_Zullo, http://dbpedia.org/resource/Galen_A._Carter, http://dbpedia.org/resource/Arthur_C._Wheeler, http://dbpedia.org/resource/The_Young_and_the_Restless_characters_(1990s), http://dbpedia.org/resource/Jane_Geddes] [INFO] P: 0.006535947712418301, R: 0.1111111111111111, F: 0.012345679012345678 -[INFO] Cumulative metrics, sample: 861, P: 0.502, R: 0.558, macro F1: 0.511, macro F1*: 0.529 -[INFO] Current time: 4518 ms, average: 2914.96 ms/ques, entity detection: 501.61 ms/ques (linking tool: 408.59 ms/ques), relation detection: 942.64 ms/ques (similarity: 710.53 ms/ques), EDG generation: 45.83 ms/ques +[INFO] Cumulative metrics, sample: 861, P: 0.502, R: 0.558, macro F1: 0.529 +[INFO] Current question consumed time: 3798 ms, average: 2612.78 ms/ques, entity detection: 440.13 ms/ques (linking tool: 362.41 ms/ques), relation detection: 867.13 ms/ques (similarity: 703.64 ms/ques), EDG generation: 46.21 ms/ques [INFO] Entity Linking P: 0.7363530778164924, R: 0.6887340301974448, micro F1: 0.7040263259775459, macro F1: 0.7117479627468296 [INFO] Relation Linking P: 0.5968834688346883, R: 0.5429732868757259, micro F1: 0.5584978706929936, macro F1: 0.5686535212978859 [INFO] Type Linking P: 0.5574912891986062, R: 0.5574912891986062, micro F1: 0.5574912891986062, macro F1: 0.5574912891986062 @@ -12085,8 +12085,8 @@ Question 861: What is the Location of the Peetz Table Wind Energy Center which i [INFO] Golden Answer: [http://dbpedia.org/resource/Colorado] [INFO] Predicted Answer: [http://dbpedia.org/resource/Colorado] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 862, P: 0.503, R: 0.558, macro F1: 0.511, macro F1*: 0.529 -[INFO] Current time: 1725 ms, average: 2913.58 ms/ques, entity detection: 501.40 ms/ques (linking tool: 408.32 ms/ques), relation detection: 942.27 ms/ques (similarity: 710.16 ms/ques), EDG generation: 45.83 ms/ques +[INFO] Cumulative metrics, sample: 862, P: 0.503, R: 0.558, macro F1: 0.529 +[INFO] Current question consumed time: 1517 ms, average: 2611.51 ms/ques, entity detection: 439.92 ms/ques (linking tool: 362.15 ms/ques), relation detection: 866.84 ms/ques (similarity: 703.38 ms/ques), EDG generation: 46.22 ms/ques [INFO] Entity Linking P: 0.7366589327146171, R: 0.6890951276102089, micro F1: 0.7043696829079663, macro F1: 0.7120836550569286 [INFO] Relation Linking P: 0.5973511214230471, R: 0.5435034802784223, micro F1: 0.5590100541376652, macro F1: 0.5691565129464233 [INFO] Type Linking P: 0.5580046403712297, R: 0.5580046403712297, micro F1: 0.5580046403712297, macro F1: 0.5580046403712297 @@ -12099,8 +12099,8 @@ Question 862: Count the water bodies that flow into the North Sea? [INFO] Golden Answer: [17] [INFO] Predicted Answer: [17] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 863, P: 0.503, R: 0.559, macro F1: 0.512, macro F1*: 0.530 -[INFO] Current time: 2712 ms, average: 2913.34 ms/ques, entity detection: 501.27 ms/ques (linking tool: 408.21 ms/ques), relation detection: 942.14 ms/ques (similarity: 710.27 ms/ques), EDG generation: 45.80 ms/ques +[INFO] Cumulative metrics, sample: 863, P: 0.503, R: 0.559, macro F1: 0.530 +[INFO] Current question consumed time: 2303 ms, average: 2611.15 ms/ques, entity detection: 439.79 ms/ques (linking tool: 362.05 ms/ques), relation detection: 866.69 ms/ques (similarity: 703.50 ms/ques), EDG generation: 46.19 ms/ques [INFO] Entity Linking P: 0.7369640787949016, R: 0.6894553881807648, micro F1: 0.7047122441096952, macro F1: 0.7124185652038447 [INFO] Relation Linking P: 0.5978176902278872, R: 0.5440324449594438, micro F1: 0.5595210505986877, macro F1: 0.5696583284133475 [INFO] Type Linking P: 0.5585168018539977, R: 0.5585168018539977, micro F1: 0.5585168018539977, macro F1: 0.5585168018539977 @@ -12113,8 +12113,8 @@ Question 863: Who was the First Driver of 1999 San Marino Grand Prix ? [INFO] Golden Answer: [http://dbpedia.org/resource/Michael_Schumacher] [INFO] Predicted Answer: [http://dbpedia.org/resource/Michael_Schumacher] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 864, P: 0.504, R: 0.559, macro F1: 0.513, macro F1*: 0.530 -[INFO] Current time: 1139 ms, average: 2911.29 ms/ques, entity detection: 500.89 ms/ques (linking tool: 407.86 ms/ques), relation detection: 941.55 ms/ques (similarity: 709.75 ms/ques), EDG generation: 45.76 ms/ques +[INFO] Cumulative metrics, sample: 864, P: 0.504, R: 0.559, macro F1: 0.530 +[INFO] Current question consumed time: 937 ms, average: 2609.21 ms/ques, entity detection: 439.43 ms/ques (linking tool: 361.72 ms/ques), relation detection: 866.14 ms/ques (similarity: 703.00 ms/ques), EDG generation: 46.15 ms/ques [INFO] Entity Linking P: 0.7372685185185185, R: 0.6898148148148148, micro F1: 0.7050540123456793, macro F1: 0.7127526959238232 [INFO] Relation Linking P: 0.5982831790123456, R: 0.5445601851851852, micro F1: 0.5600308641975318, macro F1: 0.5701589718463506 [INFO] Type Linking P: 0.5590277777777778, R: 0.5590277777777778, micro F1: 0.5590277777777778, macro F1: 0.5590277777777778 @@ -12127,8 +12127,8 @@ Question 864: Bantam Spectra is the publisher of which series? [INFO] Golden Answer: [http://dbpedia.org/resource/A_Song_of_Ice_and_Fire, http://dbpedia.org/resource/Star_Wars_canon, http://dbpedia.org/resource/What_Might_Have_Been_(anthology_series), http://dbpedia.org/resource/The_Farseer_Trilogy, http://dbpedia.org/resource/The_Jenny_Casey_Trilogy, http://dbpedia.org/resource/Star_Wars:_X-wing_(book_series), http://dbpedia.org/resource/Thrawn_trilogy, http://dbpedia.org/resource/The_Hand_of_Thrawn, http://dbpedia.org/resource/Bantam_Spectra, http://dbpedia.org/resource/The_Millennial_Contest_series, http://dbpedia.org/resource/Dune_prequel_series, http://dbpedia.org/resource/Kim_Stanley_Robinson, http://dbpedia.org/resource/Jedi_Academy_trilogy, http://dbpedia.org/resource/The_Sword_of_Change, http://dbpedia.org/resource/Blade_Runner, http://dbpedia.org/resource/Bridge_trilogy, http://dbpedia.org/resource/Uplift_Universe, http://dbpedia.org/resource/The_Bounty_Hunter_Wars, http://dbpedia.org/resource/The_Queendom_of_Sol, http://dbpedia.org/resource/The_Nightrunner_Series, http://dbpedia.org/resource/The_Nightrunner_series, http://dbpedia.org/resource/Uplift_Storm] [INFO] Predicted Answer: [http://dbpedia.org/resource/The_Black_Fleet_Crisis] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 865, P: 0.503, R: 0.559, macro F1: 0.512, macro F1*: 0.530 -[INFO] Current time: 767 ms, average: 2908.81 ms/ques, entity detection: 500.49 ms/ques (linking tool: 407.49 ms/ques), relation detection: 940.67 ms/ques (similarity: 709.13 ms/ques), EDG generation: 45.73 ms/ques +[INFO] Cumulative metrics, sample: 865, P: 0.503, R: 0.559, macro F1: 0.530 +[INFO] Current question consumed time: 645 ms, average: 2606.94 ms/ques, entity detection: 439.06 ms/ques (linking tool: 361.38 ms/ques), relation detection: 865.34 ms/ques (similarity: 702.38 ms/ques), EDG generation: 46.11 ms/ques [INFO] Entity Linking P: 0.7375722543352601, R: 0.6901734104046243, micro F1: 0.705394990366089, macro F1: 0.7130860499403244 [INFO] Relation Linking P: 0.598747591522158, R: 0.5445086705202312, micro F1: 0.5601541425818892, macro F1: 0.570341516353457 [INFO] Type Linking P: 0.5595375722543353, R: 0.5595375722543353, micro F1: 0.5595375722543353, macro F1: 0.5595375722543353 @@ -12141,8 +12141,8 @@ Question 865: Where is the sister station of Al Arabiya 99 located? [INFO] Golden Answer: [http://dbpedia.org/resource/Dubai, http://dbpedia.org/resource/Media_City] [INFO] Predicted Answer: [http://dbpedia.org/resource/Arab_Idol_(season_1), http://dbpedia.org/resource/Arab_Idol_(season_2), http://dbpedia.org/resource/Ramez_Wakel_el-Gaw_(TV_show), http://dbpedia.org/resource/If_he_were_among_us, http://dbpedia.org/resource/Arab_Idol, http://dbpedia.org/resource/Man_sa_yarbah_2_malyoon, http://dbpedia.org/resource/The_Biggest_Winner, http://dbpedia.org/resource/The_Biggest_Winner_Arab_(season_1), http://dbpedia.org/resource/The_Biggest_Winner_Arab_(season_2), http://dbpedia.org/resource/The_Biggest_Winner_Arab_(season_3), http://dbpedia.org/resource/Shaklak_Mish_Ghareeb, http://dbpedia.org/resource/The_Biggest_Winner_Arab:_Couples, http://dbpedia.org/resource/The_Lost_Dream, http://dbpedia.org/resource/The_X_Factor_(Arabic_TV_series), http://dbpedia.org/resource/Redwall_(TV_series), http://dbpedia.org/resource/Galactik_Football, http://dbpedia.org/resource/Celebrity_Duets_Arab_World, http://dbpedia.org/resource/Middle_East_Broadcasting_Center] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 866, P: 0.503, R: 0.558, macro F1: 0.511, macro F1*: 0.529 -[INFO] Current time: 1444 ms, average: 2907.12 ms/ques, entity detection: 500.08 ms/ques (linking tool: 407.13 ms/ques), relation detection: 940.33 ms/ques (similarity: 708.73 ms/ques), EDG generation: 45.69 ms/ques +[INFO] Cumulative metrics, sample: 866, P: 0.503, R: 0.558, macro F1: 0.529 +[INFO] Current question consumed time: 1227 ms, average: 2605.35 ms/ques, entity detection: 438.68 ms/ques (linking tool: 361.05 ms/ques), relation detection: 865.01 ms/ques (similarity: 701.99 ms/ques), EDG generation: 46.08 ms/ques [INFO] Entity Linking P: 0.7367205542725174, R: 0.6893764434180139, micro F1: 0.704580446497306, macro F1: 0.7122626249403933 [INFO] Relation Linking P: 0.5980561970746728, R: 0.5438799076212472, micro F1: 0.5595073133179378, macro F1: 0.5696829233784529 [INFO] Type Linking P: 0.558891454965358, R: 0.558891454965358, micro F1: 0.558891454965358, macro F1: 0.558891454965358 @@ -12155,8 +12155,8 @@ Question 866: Which common prime minister did Graham Richardson and Bon McMullan [INFO] Golden Answer: [http://dbpedia.org/resource/Paul_Keating] [INFO] Predicted Answer: [http://dbpedia.org/resource/Paul_Keating] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 867, P: 0.503, R: 0.558, macro F1: 0.512, macro F1*: 0.530 -[INFO] Current time: 2918 ms, average: 2907.13 ms/ques, entity detection: 500.12 ms/ques (linking tool: 407.07 ms/ques), relation detection: 940.31 ms/ques (similarity: 708.50 ms/ques), EDG generation: 45.67 ms/ques +[INFO] Cumulative metrics, sample: 867, P: 0.503, R: 0.558, macro F1: 0.530 +[INFO] Current question consumed time: 2439 ms, average: 2605.16 ms/ques, entity detection: 438.64 ms/ques (linking tool: 360.96 ms/ques), relation detection: 864.99 ms/ques (similarity: 701.77 ms/ques), EDG generation: 46.06 ms/ques [INFO] Entity Linking P: 0.7370242214532872, R: 0.6897347174163783, micro F1: 0.7049211841599389, macro F1: 0.7125957711060075 [INFO] Relation Linking P: 0.5985198000768934, R: 0.544405997693195, micro F1: 0.5600153787005008, macro F1: 0.5701818430132916 [INFO] Type Linking P: 0.5594002306805075, R: 0.5594002306805075, micro F1: 0.5594002306805075, macro F1: 0.5594002306805075 @@ -12169,8 +12169,8 @@ Question 867: How many things are produced by companies located in Tehran? [INFO] Golden Answer: [89] [INFO] Predicted Answer: [1] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 868, P: 0.503, R: 0.558, macro F1: 0.511, macro F1*: 0.529 -[INFO] Current time: 8297 ms, average: 2913.34 ms/ques, entity detection: 500.40 ms/ques (linking tool: 407.32 ms/ques), relation detection: 943.10 ms/ques (similarity: 711.87 ms/ques), EDG generation: 45.64 ms/ques +[INFO] Cumulative metrics, sample: 868, P: 0.503, R: 0.558, macro F1: 0.529 +[INFO] Current question consumed time: 7625 ms, average: 2610.94 ms/ques, entity detection: 438.90 ms/ques (linking tool: 361.18 ms/ques), relation detection: 867.63 ms/ques (similarity: 705.16 ms/ques), EDG generation: 46.02 ms/ques [INFO] Entity Linking P: 0.7373271889400922, R: 0.6900921658986175, micro F1: 0.7052611367127499, macro F1: 0.7129281455611223 [INFO] Relation Linking P: 0.5978302611367127, R: 0.543778801843318, micro F1: 0.5593701996927812, macro F1: 0.5695249514890826 [INFO] Type Linking P: 0.5587557603686636, R: 0.5587557603686636, micro F1: 0.5587557603686636, macro F1: 0.5587557603686636 @@ -12183,8 +12183,8 @@ Question 868: Which is the common past team of the Charlie Justice (American foo [INFO] Golden Answer: [http://dbpedia.org/resource/Washington_Redskins, *@en] [INFO] Predicted Answer: [http://dbpedia.org/resource/Washington_Redskins, *@en] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 869, P: 0.503, R: 0.558, macro F1: 0.512, macro F1*: 0.529 -[INFO] Current time: 3195 ms, average: 2913.67 ms/ques, entity detection: 500.86 ms/ques (linking tool: 407.72 ms/ques), relation detection: 942.73 ms/ques (similarity: 711.49 ms/ques), EDG generation: 45.70 ms/ques +[INFO] Cumulative metrics, sample: 869, P: 0.503, R: 0.558, macro F1: 0.529 +[INFO] Current question consumed time: 2728 ms, average: 2611.08 ms/ques, entity detection: 439.26 ms/ques (linking tool: 361.51 ms/ques), relation detection: 867.27 ms/ques (similarity: 704.78 ms/ques), EDG generation: 46.13 ms/ques [INFO] Entity Linking P: 0.7376294591484465, R: 0.6904487917146145, micro F1: 0.70560030686613, macro F1: 0.7132597509896339 [INFO] Relation Linking P: 0.5982930571538166, R: 0.5443037974683544, micro F1: 0.5598772535481406, macro F1: 0.5700228942350086 [INFO] Type Linking P: 0.5592635212888377, R: 0.5592635212888377, micro F1: 0.5592635212888377, macro F1: 0.5592635212888377 @@ -12197,8 +12197,8 @@ Question 869: Name the officeholder who hasa child named Augustus and has an ass [INFO] Golden Answer: [http://dbpedia.org/resource/Julius_Caesar] [INFO] Predicted Answer: [http://dbpedia.org/resource/Julius_Caesar] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 870, P: 0.504, R: 0.559, macro F1: 0.512, macro F1*: 0.530 -[INFO] Current time: 2846 ms, average: 2913.59 ms/ques, entity detection: 500.86 ms/ques (linking tool: 407.70 ms/ques), relation detection: 942.71 ms/ques (similarity: 711.25 ms/ques), EDG generation: 45.74 ms/ques +[INFO] Cumulative metrics, sample: 870, P: 0.504, R: 0.559, macro F1: 0.530 +[INFO] Current question consumed time: 2366 ms, average: 2610.79 ms/ques, entity detection: 439.14 ms/ques (linking tool: 361.39 ms/ques), relation detection: 867.26 ms/ques (similarity: 704.55 ms/ques), EDG generation: 46.19 ms/ques [INFO] Entity Linking P: 0.7379310344827587, R: 0.6908045977011494, micro F1: 0.705938697318008, macro F1: 0.7135905900629734 [INFO] Relation Linking P: 0.5981800766283525, R: 0.5442528735632184, micro F1: 0.5598084291187749, macro F1: 0.5699436900146388 [INFO] Type Linking P: 0.5586206896551724, R: 0.5586206896551724, micro F1: 0.5586206896551724, macro F1: 0.5586206896551724 @@ -12211,8 +12211,8 @@ Question 870: Which party won both the general electoin and the andhra pradesh e [INFO] Golden Answer: [http://dbpedia.org/resource/Indian_National_Congress] [INFO] Predicted Answer: [http://dbpedia.org/resource/Y._S._Chowdary, http://dbpedia.org/resource/Nomula_Narsimhaiah, http://dbpedia.org/resource/Dokka_Manikya_Vara_Prasad, http://dbpedia.org/resource/Mudumala_Henry_Samuel, http://dbpedia.org/resource/Mynampally_Hanmanth_Rao, http://dbpedia.org/resource/Kunja_Bojji, http://dbpedia.org/resource/K._T._Rama_Rao, http://dbpedia.org/resource/Kommidi_Narasimha_Reddy, http://dbpedia.org/resource/Raashid_Alvi, http://dbpedia.org/resource/Byreddy_Rajasekhar_Reddy, http://dbpedia.org/resource/Kalvakuntla_Chandrashekar_Rao, http://dbpedia.org/resource/Uddaraju_Ramam, http://dbpedia.org/resource/Kanumuri_Bapi_Raju, http://dbpedia.org/resource/T_Ratna_Bai, http://dbpedia.org/resource/Sarve_Satyanarayana] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 871, P: 0.503, R: 0.558, macro F1: 0.512, macro F1*: 0.529 -[INFO] Current time: 5492 ms, average: 2916.55 ms/ques, entity detection: 501.09 ms/ques (linking tool: 408.03 ms/ques), relation detection: 943.54 ms/ques (similarity: 712.97 ms/ques), EDG generation: 45.83 ms/ques +[INFO] Cumulative metrics, sample: 871, P: 0.503, R: 0.558, macro F1: 0.529 +[INFO] Current question consumed time: 5072 ms, average: 2613.62 ms/ques, entity detection: 439.37 ms/ques (linking tool: 361.70 ms/ques), relation detection: 868.03 ms/ques (similarity: 706.30 ms/ques), EDG generation: 46.29 ms/ques [INFO] Entity Linking P: 0.7370838117106774, R: 0.6900114810562572, micro F1: 0.7051282051282055, macro F1: 0.712771312692063 [INFO] Relation Linking P: 0.5974933027171833, R: 0.5436280137772675, micro F1: 0.5591657099119796, macro F1: 0.5692893344577908 [INFO] Type Linking P: 0.5591274397244547, R: 0.5591274397244547, micro F1: 0.5591274397244547, macro F1: 0.5591274397244547 @@ -12225,8 +12225,8 @@ Question 871: Give me someone on the board of trinity house? [INFO] Golden Answer: [http://dbpedia.org/resource/Samuel_Pepys] [INFO] Predicted Answer: [http://dbpedia.org/resource/Trinity_House, http://dbpedia.org/resource/Royal_Society, http://dbpedia.org/resource/Freedom_of_the_City, http://dbpedia.org/resource/English_Tangier] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 872, P: 0.503, R: 0.557, macro F1: 0.511, macro F1*: 0.529 -[INFO] Current time: 1498 ms, average: 2914.92 ms/ques, entity detection: 500.71 ms/ques (linking tool: 407.67 ms/ques), relation detection: 943.01 ms/ques (similarity: 712.46 ms/ques), EDG generation: 45.88 ms/ques +[INFO] Cumulative metrics, sample: 872, P: 0.503, R: 0.557, macro F1: 0.529 +[INFO] Current question consumed time: 1328 ms, average: 2612.15 ms/ques, entity detection: 439.03 ms/ques (linking tool: 361.38 ms/ques), relation detection: 867.54 ms/ques (similarity: 705.83 ms/ques), EDG generation: 46.33 ms/ques [INFO] Entity Linking P: 0.7373853211009175, R: 0.6903669724770642, micro F1: 0.7054663608562695, macro F1: 0.7131019490807267 [INFO] Relation Linking P: 0.5979548929663608, R: 0.544151376146789, micro F1: 0.5596712538226309, macro F1: 0.5697858188520556 [INFO] Type Linking P: 0.5596330275229358, R: 0.5596330275229358, micro F1: 0.5596330275229358, macro F1: 0.5596330275229358 @@ -12239,8 +12239,8 @@ Question 872: Whose leaders are Willem-Alexander and Fredis Refunjol? [INFO] Golden Answer: [http://dbpedia.org/resource/Aruba] [INFO] Predicted Answer: [http://dbpedia.org/resource/Aruba, http://dbpedia.org/resource/Estates_of_Aruba] [INFO] P: 0.5, R: 1.0, F: 0.6666666666666666 -[INFO] Cumulative metrics, sample: 873, P: 0.503, R: 0.558, macro F1: 0.512, macro F1*: 0.529 -[INFO] Current time: 2634 ms, average: 2914.60 ms/ques, entity detection: 500.51 ms/ques (linking tool: 407.41 ms/ques), relation detection: 942.82 ms/ques (similarity: 712.61 ms/ques), EDG generation: 45.85 ms/ques +[INFO] Cumulative metrics, sample: 873, P: 0.503, R: 0.558, macro F1: 0.529 +[INFO] Current question consumed time: 2285 ms, average: 2611.77 ms/ques, entity detection: 438.84 ms/ques (linking tool: 361.14 ms/ques), relation detection: 867.35 ms/ques (similarity: 705.97 ms/ques), EDG generation: 46.32 ms/ques [INFO] Entity Linking P: 0.7376861397479955, R: 0.6901489117983963, micro F1: 0.7054219167621233, macro F1: 0.7131261920548211 [INFO] Relation Linking P: 0.5984154257350133, R: 0.5446735395189003, micro F1: 0.5601756395570838, macro F1: 0.5702811556148619 [INFO] Type Linking P: 0.5601374570446735, R: 0.5601374570446735, micro F1: 0.5601374570446735, macro F1: 0.5601374570446735 @@ -12253,8 +12253,8 @@ Question 873: Horses grandsired by Sundridge have won which awards? [INFO] Golden Answer: [http://dbpedia.org/resource/Hawthorne_Race_Course, http://dbpedia.org/resource/Blood-Horse_magazine_List_of_the_Top_100_U.S._Racehorses_of_the_20th_Century, http://dbpedia.org/resource/Virginia_Thoroughbred_Association_Hall_of_Fame, http://dbpedia.org/resource/National_Museum_of_Racing_and_Hall_of_Fame, http://dbpedia.org/resource/Kentucky, http://dbpedia.org/resource/Florence,_Kentucky] [INFO] Predicted Answer: [Richard Duckworth and Fabian Stedman@en] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 874, P: 0.502, R: 0.557, macro F1: 0.511, macro F1*: 0.528 -[INFO] Current time: 641 ms, average: 2912.00 ms/ques, entity detection: 500.19 ms/ques (linking tool: 407.10 ms/ques), relation detection: 941.89 ms/ques (similarity: 711.93 ms/ques), EDG generation: 45.82 ms/ques +[INFO] Cumulative metrics, sample: 874, P: 0.502, R: 0.557, macro F1: 0.528 +[INFO] Current question consumed time: 552 ms, average: 2609.41 ms/ques, entity detection: 438.54 ms/ques (linking tool: 360.86 ms/ques), relation detection: 866.50 ms/ques (similarity: 705.29 ms/ques), EDG generation: 46.28 ms/ques [INFO] Entity Linking P: 0.7368421052631579, R: 0.6893592677345538, micro F1: 0.7046147978642261, macro F1: 0.7123102581966347 [INFO] Relation Linking P: 0.5977307398932112, R: 0.5440503432494279, micro F1: 0.559534706331046, macro F1: 0.569628660013472 [INFO] Type Linking P: 0.5594965675057209, R: 0.5594965675057209, micro F1: 0.5594965675057209, macro F1: 0.5594965675057209 @@ -12267,8 +12267,8 @@ Question 874: What is the river whose source region is North-Western Province, Z [INFO] Golden Answer: [http://dbpedia.org/resource/Zambezi] [INFO] Predicted Answer: [http://dbpedia.org/resource/Keurbooms_River, http://dbpedia.org/resource/Mbhashe_River, http://dbpedia.org/resource/Mahavavy_River, http://dbpedia.org/resource/Krom_River, http://dbpedia.org/resource/Fiherenana_River, http://dbpedia.org/resource/Great_Kei_River, http://dbpedia.org/resource/Sundays_River, http://dbpedia.org/resource/Linta_River, http://dbpedia.org/resource/Jubba_River, http://dbpedia.org/resource/Limpopo_River, http://dbpedia.org/resource/Tana_River_(Kenya), http://dbpedia.org/resource/Zambezi, http://dbpedia.org/resource/Hutt_River_(Western_Australia), http://dbpedia.org/resource/Keiskamma_River, http://dbpedia.org/resource/Tsitsikamma_River, http://dbpedia.org/resource/Irodo_River, http://dbpedia.org/resource/Saharenana_River, http://dbpedia.org/resource/Gamtoos_River, http://dbpedia.org/resource/Great_Fish_River, http://dbpedia.org/resource/Balla_Balla_River, http://dbpedia.org/resource/Bowes_River, http://dbpedia.org/resource/Buller_River_(Western_Australia), http://dbpedia.org/resource/Cane_River_(Western_Australia), http://dbpedia.org/resource/De_Grey_River, http://dbpedia.org/resource/Harding_River, http://dbpedia.org/resource/Hill_River_(Western_Australia), http://dbpedia.org/resource/Maitland_River_(Western_Australia), http://dbpedia.org/resource/Prince_Regent_River, http://dbpedia.org/resource/Robe_River_(Australia), http://dbpedia.org/resource/Turner_River, http://dbpedia.org/resource/Yule_River, http://dbpedia.org/resource/Amanzimtoti_River, http://dbpedia.org/resource/Zigi_River, http://dbpedia.org/resource/Kelani_River, http://dbpedia.org/resource/Fortescue_River, http://dbpedia.org/resource/Umkomazi_River, http://dbpedia.org/resource/Great_Brak_River_(river), http://dbpedia.org/resource/Little_Brak_River_(river), http://dbpedia.org/resource/Gal_Oya, http://dbpedia.org/resource/Mzimvubu_River, http://dbpedia.org/resource/Kariega_River, http://dbpedia.org/resource/Tugela_River, http://dbpedia.org/resource/Umzimkulu_River, http://dbpedia.org/resource/Maputo_River, http://dbpedia.org/resource/Maningory_River, http://dbpedia.org/resource/Mdloti_River, http://dbpedia.org/resource/Mthatha_River, http://dbpedia.org/resource/Margaret_River, http://dbpedia.org/resource/Save_River_(Africa), http://dbpedia.org/resource/Chalumna_River, http://dbpedia.org/resource/Boesmans_River_(Eastern_Cape), http://dbpedia.org/resource/Tudor_Creek, http://dbpedia.org/resource/Progo_River, http://dbpedia.org/resource/Walawe_River, http://dbpedia.org/resource/Umgeni_River, http://dbpedia.org/resource/Mtamvuna_River, http://dbpedia.org/resource/Sambirano_River, http://dbpedia.org/resource/Fanambana_River, http://dbpedia.org/resource/Manampatrana_River, http://dbpedia.org/resource/Moore_River, http://dbpedia.org/resource/Murchison_River_(Western_Australia), http://dbpedia.org/resource/Pangani_River, http://dbpedia.org/resource/Gascoyne_River, http://dbpedia.org/resource/Buffalo_River_(Eastern_Cape), http://dbpedia.org/resource/Nahoon_River, http://dbpedia.org/resource/Komati_River, http://dbpedia.org/resource/Ohlanga_River, http://dbpedia.org/resource/Kowie_River, http://dbpedia.org/resource/Menarandra_River, http://dbpedia.org/resource/Kaaimans_River, http://dbpedia.org/resource/Mandrare_River, http://dbpedia.org/resource/Gin_Ganga, http://dbpedia.org/resource/Rivière_des_Remparts] [INFO] P: 0.0136986301369863, R: 1.0, F: 0.027027027027027025 -[INFO] Cumulative metrics, sample: 875, P: 0.502, R: 0.558, macro F1: 0.510, macro F1*: 0.528 -[INFO] Current time: 5100 ms, average: 2914.50 ms/ques, entity detection: 500.60 ms/ques (linking tool: 407.49 ms/ques), relation detection: 942.47 ms/ques (similarity: 712.79 ms/ques), EDG generation: 45.87 ms/ques +[INFO] Cumulative metrics, sample: 875, P: 0.502, R: 0.558, macro F1: 0.528 +[INFO] Current question consumed time: 4440 ms, average: 2611.51 ms/ques, entity detection: 438.86 ms/ques (linking tool: 361.17 ms/ques), relation detection: 867.01 ms/ques (similarity: 706.16 ms/ques), EDG generation: 46.33 ms/ques [INFO] Entity Linking P: 0.7371428571428571, R: 0.6891428571428572, micro F1: 0.7045714285714288, macro F1: 0.7123351648351649 [INFO] Relation Linking P: 0.5981904761904762, R: 0.544, micro F1: 0.5596571428571439, macro F1: 0.5698097223380305 [INFO] Type Linking P: 0.5588571428571428, R: 0.5588571428571428, micro F1: 0.5588571428571428, macro F1: 0.5588571428571428 @@ -12281,8 +12281,8 @@ Question 875: Who was on the first team in the GPs that were held at Watkins Gle [INFO] Golden Answer: [http://dbpedia.org/resource/Scuderia_Ferrari, http://dbpedia.org/resource/McLaren, http://dbpedia.org/resource/Cosworth, Lotus-Ford@en, Lotus-Climax@en, BRM@en, http://dbpedia.org/resource/British_Racing_Motors, http://dbpedia.org/resource/Team_Lotus, http://dbpedia.org/resource/Ford_Motor_Company, http://dbpedia.org/resource/Equipe_Matra_Sports, http://dbpedia.org/resource/Tyrrell_Racing, Brabham-Ford@en, http://dbpedia.org/resource/Williams_Grand_Prix_Engineering, McLaren-Ford@en] [INFO] Predicted Answer: [5260.91334912, less than one acre@en, Less than@en, 4046.8564224, 18170.385336576, 11209.792290048, 182108.539008] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 876, P: 0.501, R: 0.557, macro F1: 0.510, macro F1*: 0.528 -[INFO] Current time: 6003 ms, average: 2918.03 ms/ques, entity detection: 500.53 ms/ques (linking tool: 407.38 ms/ques), relation detection: 944.36 ms/ques (similarity: 713.64 ms/ques), EDG generation: 45.95 ms/ques +[INFO] Cumulative metrics, sample: 876, P: 0.501, R: 0.557, macro F1: 0.528 +[INFO] Current question consumed time: 5175 ms, average: 2614.43 ms/ques, entity detection: 438.80 ms/ques (linking tool: 361.09 ms/ques), relation detection: 868.70 ms/ques (similarity: 707.00 ms/ques), EDG generation: 46.41 ms/ques [INFO] Entity Linking P: 0.7374429223744292, R: 0.6894977168949772, micro F1: 0.704908675799087, macro F1: 0.7126648401826484 [INFO] Relation Linking P: 0.5980783866057838, R: 0.5439497716894978, micro F1: 0.5595890410958914, macro F1: 0.5697313143876506 [INFO] Type Linking P: 0.5582191780821918, R: 0.5582191780821918, micro F1: 0.5582191780821918, macro F1: 0.5582191780821918 @@ -12295,8 +12295,8 @@ Question 876: Name the movie whose screenplay is by Akiva Goldsman and directed [INFO] Golden Answer: [http://dbpedia.org/resource/Batman_Forever, http://dbpedia.org/resource/The_Client_(1994_film), http://dbpedia.org/resource/A_Time_to_Kill_(1996_film)] [INFO] Predicted Answer: [http://dbpedia.org/resource/Batman_&_Robin_(film), http://dbpedia.org/resource/Batman_Forever, http://dbpedia.org/resource/The_Client_(1994_film), http://dbpedia.org/resource/A_Time_to_Kill_(1996_film)] [INFO] P: 0.75, R: 1.0, F: 0.8571428571428571 -[INFO] Cumulative metrics, sample: 877, P: 0.501, R: 0.558, macro F1: 0.510, macro F1*: 0.528 -[INFO] Current time: 1973 ms, average: 2916.95 ms/ques, entity detection: 500.39 ms/ques (linking tool: 407.17 ms/ques), relation detection: 943.91 ms/ques (similarity: 713.60 ms/ques), EDG generation: 45.95 ms/ques +[INFO] Cumulative metrics, sample: 877, P: 0.501, R: 0.558, macro F1: 0.528 +[INFO] Current question consumed time: 1848 ms, average: 2613.56 ms/ques, entity detection: 438.69 ms/ques (linking tool: 360.90 ms/ques), relation detection: 868.26 ms/ques (similarity: 706.93 ms/ques), EDG generation: 46.42 ms/ques [INFO] Entity Linking P: 0.7377423033067275, R: 0.6898517673888256, micro F1: 0.7052451539338657, macro F1: 0.7129937595855754 [INFO] Relation Linking P: 0.597966552641581, R: 0.5438996579247435, micro F1: 0.5595210946408219, macro F1: 0.5696530826862628 [INFO] Type Linking P: 0.5575826681870011, R: 0.5575826681870011, micro F1: 0.5575826681870011, macro F1: 0.5575826681870011 @@ -12309,8 +12309,8 @@ Question 877: What ethinicity of Linda Hogan is also the state of origin of Holm [INFO] Golden Answer: [http://dbpedia.org/resource/Chickasaw] [INFO] Predicted Answer: [http://dbpedia.org/resource/Chickasaw] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 878, P: 0.502, R: 0.558, macro F1: 0.511, macro F1*: 0.529 -[INFO] Current time: 1664 ms, average: 2915.52 ms/ques, entity detection: 500.36 ms/ques (linking tool: 407.00 ms/ques), relation detection: 943.20 ms/ques (similarity: 713.00 ms/ques), EDG generation: 46.00 ms/ques +[INFO] Cumulative metrics, sample: 878, P: 0.502, R: 0.558, macro F1: 0.529 +[INFO] Current question consumed time: 1497 ms, average: 2612.29 ms/ques, entity detection: 438.66 ms/ques (linking tool: 360.74 ms/ques), relation detection: 867.59 ms/ques (similarity: 706.33 ms/ques), EDG generation: 46.48 ms/ques [INFO] Entity Linking P: 0.7380410022779044, R: 0.6902050113895216, micro F1: 0.7055808656036449, macro F1: 0.713321925646587 [INFO] Relation Linking P: 0.598424449506454, R: 0.5444191343963554, micro F1: 0.5600227790432811, macro F1: 0.5701457756613272 [INFO] Type Linking P: 0.5580865603644647, R: 0.5580865603644647, micro F1: 0.5580865603644647, macro F1: 0.5580865603644647 @@ -12323,8 +12323,8 @@ Question 878: What are the track and field events of Jamiatu Muslim Mindanao? [INFO] Golden Answer: [http://dbpedia.org/resource/Association_football, http://dbpedia.org/resource/Badminton, http://dbpedia.org/resource/Baseball, http://dbpedia.org/resource/Basketball, http://dbpedia.org/resource/Chess, http://dbpedia.org/resource/Volleyball] [INFO] Predicted Answer: [http://dbpedia.org/resource/Association_football, http://dbpedia.org/resource/Badminton, http://dbpedia.org/resource/Baseball, http://dbpedia.org/resource/Basketball, http://dbpedia.org/resource/Chess, http://dbpedia.org/resource/Volleyball] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 879, P: 0.503, R: 0.559, macro F1: 0.511, macro F1*: 0.529 -[INFO] Current time: 1951 ms, average: 2914.42 ms/ques, entity detection: 500.44 ms/ques (linking tool: 407.10 ms/ques), relation detection: 942.47 ms/ques (similarity: 712.50 ms/ques), EDG generation: 45.98 ms/ques +[INFO] Cumulative metrics, sample: 879, P: 0.503, R: 0.559, macro F1: 0.529 +[INFO] Current question consumed time: 1726 ms, average: 2611.28 ms/ques, entity detection: 438.72 ms/ques (linking tool: 360.82 ms/ques), relation detection: 866.92 ms/ques (similarity: 705.83 ms/ques), EDG generation: 46.46 ms/ques [INFO] Entity Linking P: 0.7383390216154722, R: 0.6905574516496018, micro F1: 0.7059158134243461, macro F1: 0.7136493409563561 [INFO] Relation Linking P: 0.5988813045127038, R: 0.5449374288964732, micro F1: 0.56052332195677, macro F1: 0.5706373374784951 [INFO] Type Linking P: 0.558589306029579, R: 0.558589306029579, micro F1: 0.558589306029579, macro F1: 0.558589306029579 @@ -12337,8 +12337,8 @@ Question 879: How many cities are close to Lake Victoria? [INFO] Golden Answer: [12] [INFO] Predicted Answer: [5] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 880, P: 0.502, R: 0.558, macro F1: 0.511, macro F1*: 0.529 -[INFO] Current time: 1529 ms, average: 2912.85 ms/ques, entity detection: 500.15 ms/ques (linking tool: 406.80 ms/ques), relation detection: 941.89 ms/ques (similarity: 712.23 ms/ques), EDG generation: 45.94 ms/ques +[INFO] Cumulative metrics, sample: 880, P: 0.502, R: 0.558, macro F1: 0.529 +[INFO] Current question consumed time: 1346 ms, average: 2609.84 ms/ques, entity detection: 438.44 ms/ques (linking tool: 360.55 ms/ques), relation detection: 866.38 ms/ques (similarity: 705.56 ms/ques), EDG generation: 46.42 ms/ques [INFO] Entity Linking P: 0.7386363636363636, R: 0.6909090909090909, micro F1: 0.7062500000000003, macro F1: 0.7139760080936551 [INFO] Relation Linking P: 0.5993371212121211, R: 0.5454545454545454, micro F1: 0.5610227272727283, macro F1: 0.5711277720542342 [INFO] Type Linking P: 0.5579545454545455, R: 0.5579545454545455, micro F1: 0.5579545454545455, macro F1: 0.5579545454545455 @@ -12351,8 +12351,8 @@ Question 880: Is greasy pop recording the distributor of festival records? [INFO] Golden Answer: [true] [INFO] Predicted Answer: [true] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 881, P: 0.503, R: 0.559, macro F1: 0.511, macro F1*: 0.529 -[INFO] Current time: 925 ms, average: 2910.59 ms/ques, entity detection: 499.83 ms/ques (linking tool: 406.50 ms/ques), relation detection: 941.04 ms/ques (similarity: 711.56 ms/ques), EDG generation: 45.98 ms/ques +[INFO] Cumulative metrics, sample: 881, P: 0.503, R: 0.559, macro F1: 0.529 +[INFO] Current question consumed time: 828 ms, average: 2607.82 ms/ques, entity detection: 438.16 ms/ques (linking tool: 360.29 ms/ques), relation detection: 865.60 ms/ques (similarity: 704.90 ms/ques), EDG generation: 46.46 ms/ques [INFO] Entity Linking P: 0.738933030646992, R: 0.6912599318955732, micro F1: 0.7065834279228153, macro F1: 0.7143019296254257 [INFO] Relation Linking P: 0.5986568293605751, R: 0.5448354143019296, micro F1: 0.5603859250851315, macro F1: 0.5704794998952624 [INFO] Type Linking P: 0.5584562996594779, R: 0.5584562996594779, micro F1: 0.5584562996594779, macro F1: 0.5584562996594779 @@ -12365,8 +12365,8 @@ Question 881: What is the baseball team whose club manager's debut team is Houst [INFO] Golden Answer: [http://dbpedia.org/resource/San_Francisco_Giants, http://dbpedia.org/resource/Seattle_Mariners] [INFO] Predicted Answer: [http://dbpedia.org/resource/Baltimore_Orioles, http://dbpedia.org/resource/Atlanta_Braves, http://dbpedia.org/resource/Chicago_White_Sox, http://dbpedia.org/resource/Cleveland_Indians, http://dbpedia.org/resource/Arizona_Diamondbacks, http://dbpedia.org/resource/Colorado_Rockies, http://dbpedia.org/resource/Houston_Astros, http://dbpedia.org/resource/Kansas_City_Royals, http://dbpedia.org/resource/Los_Angeles_Angels_of_Anaheim, http://dbpedia.org/resource/Los_Angeles_Dodgers, http://dbpedia.org/resource/Milwaukee_Brewers, http://dbpedia.org/resource/New_York_Mets, http://dbpedia.org/resource/Philadelphia_Phillies, http://dbpedia.org/resource/San_Diego_Padres, http://dbpedia.org/resource/San_Francisco_Giants, http://dbpedia.org/resource/Seattle_Mariners, http://dbpedia.org/resource/Tampa_Bay_Rays, http://dbpedia.org/resource/Washington_Nationals, http://dbpedia.org/resource/Detroit_Tigers, http://dbpedia.org/resource/Miami_Marlins, http://dbpedia.org/resource/Minnesota_Twins] [INFO] P: 0.09523809523809523, R: 1.0, F: 0.17391304347826084 -[INFO] Cumulative metrics, sample: 882, P: 0.502, R: 0.559, macro F1: 0.511, macro F1*: 0.529 -[INFO] Current time: 3533 ms, average: 2911.30 ms/ques, entity detection: 499.73 ms/ques (linking tool: 406.36 ms/ques), relation detection: 941.77 ms/ques (similarity: 712.10 ms/ques), EDG generation: 45.96 ms/ques +[INFO] Cumulative metrics, sample: 882, P: 0.502, R: 0.559, macro F1: 0.529 +[INFO] Current question consumed time: 3008 ms, average: 2608.27 ms/ques, entity detection: 438.06 ms/ques (linking tool: 360.17 ms/ques), relation detection: 866.15 ms/ques (similarity: 705.44 ms/ques), EDG generation: 46.44 ms/ques [INFO] Entity Linking P: 0.7392290249433107, R: 0.691609977324263, micro F1: 0.7069160997732429, macro F1: 0.7146271081068454 [INFO] Relation Linking P: 0.5985449735449735, R: 0.5447845804988662, micro F1: 0.5603174603174613, macro F1: 0.5704008457913093 [INFO] Type Linking P: 0.5578231292517006, R: 0.5578231292517006, micro F1: 0.5578231292517006, macro F1: 0.5578231292517006 @@ -12379,8 +12379,8 @@ Question 882: What cities are near to Nihoa which is also known as Bird Island ? [INFO] Golden Answer: [http://dbpedia.org/resource/Kauai, http://dbpedia.org/resource/Hawaii] [INFO] Predicted Answer: [http://dbpedia.org/resource/Kauai, http://dbpedia.org/resource/Hawaii] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 883, P: 0.503, R: 0.560, macro F1: 0.511, macro F1*: 0.530 -[INFO] Current time: 4306 ms, average: 2912.88 ms/ques, entity detection: 499.56 ms/ques (linking tool: 406.13 ms/ques), relation detection: 942.30 ms/ques (similarity: 713.07 ms/ques), EDG generation: 45.96 ms/ques +[INFO] Cumulative metrics, sample: 883, P: 0.503, R: 0.560, macro F1: 0.530 +[INFO] Current question consumed time: 3506 ms, average: 2609.29 ms/ques, entity detection: 437.84 ms/ques (linking tool: 359.92 ms/ques), relation detection: 866.54 ms/ques (similarity: 706.40 ms/ques), EDG generation: 46.43 ms/ques [INFO] Entity Linking P: 0.739524348810872, R: 0.6919592298980748, micro F1: 0.7072480181200456, macro F1: 0.7149515460813969 [INFO] Relation Linking P: 0.5989996224990563, R: 0.5453001132502832, micro F1: 0.560815402038506, macro F1: 0.570889867018482 [INFO] Type Linking P: 0.5583238958097395, R: 0.5583238958097395, micro F1: 0.5583238958097395, macro F1: 0.5583238958097395 @@ -12393,8 +12393,8 @@ Question 883: Is Sloth in the Chordate phylum? [INFO] Golden Answer: [true] [INFO] Predicted Answer: [false] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 884, P: 0.502, R: 0.559, macro F1: 0.511, macro F1*: 0.529 -[INFO] Current time: 10811 ms, average: 2921.81 ms/ques, entity detection: 505.41 ms/ques (linking tool: 411.97 ms/ques), relation detection: 943.60 ms/ques (similarity: 712.65 ms/ques), EDG generation: 45.92 ms/ques +[INFO] Cumulative metrics, sample: 884, P: 0.502, R: 0.559, macro F1: 0.529 +[INFO] Current question consumed time: 9058 ms, average: 2616.58 ms/ques, entity detection: 442.80 ms/ques (linking tool: 364.89 ms/ques), relation detection: 867.64 ms/ques (similarity: 705.98 ms/ques), EDG generation: 46.38 ms/ques [INFO] Entity Linking P: 0.7386877828054299, R: 0.6911764705882353, micro F1: 0.7064479638009052, macro F1: 0.7141427773641102 [INFO] Relation Linking P: 0.5983220211161387, R: 0.544683257918552, micro F1: 0.560180995475114, macro F1: 0.5702440640014927 [INFO] Type Linking P: 0.5588235294117647, R: 0.5588235294117647, micro F1: 0.5588235294117647, macro F1: 0.5588235294117647 @@ -12407,8 +12407,8 @@ Question 884: Which shareholder of Naval Air facility Adak is also the operator [INFO] Golden Answer: [http://dbpedia.org/resource/United_States_Navy] [INFO] Predicted Answer: [http://dbpedia.org/resource/United_States_Navy] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 885, P: 0.503, R: 0.559, macro F1: 0.511, macro F1*: 0.530 -[INFO] Current time: 2615 ms, average: 2921.47 ms/ques, entity detection: 505.25 ms/ques (linking tool: 411.81 ms/ques), relation detection: 942.92 ms/ques (similarity: 712.08 ms/ques), EDG generation: 47.01 ms/ques +[INFO] Cumulative metrics, sample: 885, P: 0.503, R: 0.559, macro F1: 0.530 +[INFO] Current question consumed time: 2278 ms, average: 2616.20 ms/ques, entity detection: 442.60 ms/ques (linking tool: 364.69 ms/ques), relation detection: 867.00 ms/ques (similarity: 705.43 ms/ques), EDG generation: 47.51 ms/ques [INFO] Entity Linking P: 0.7389830508474576, R: 0.6915254237288135, micro F1: 0.7067796610169494, macro F1: 0.7144670254638926 [INFO] Relation Linking P: 0.5987758945386064, R: 0.5451977401129944, micro F1: 0.5606779661016958, macro F1: 0.5707321473995441 [INFO] Type Linking P: 0.559322033898305, R: 0.559322033898305, micro F1: 0.559322033898305, macro F1: 0.559322033898305 @@ -12421,8 +12421,8 @@ Question 885: What other awards have been given to Golden Globe winners? [INFO] Golden Answer: [http://dbpedia.org/resource/Film_Independent_Spirit_Awards, http://dbpedia.org/resource/Golden_Globe_Award, http://dbpedia.org/resource/Emmy_Award, Juvenile Academy Award, Academy Honorary Award, Emmy, 2 Golden Globes@en, http://dbpedia.org/resource/Hollywood_Walk_of_Fame, Golden Globe for New Star Of The Year – Actress@en, Theatre World Award for Promising Personality@en, 1956, 1961, 1964, Golden Globe Award@en, Cannes Film Festival, Golden Palm@en, Oscar Award (Nominated)@en, César Awards 1996@en, http://dbpedia.org/resource/César_Award, *@en, 1989, 1990, http://dbpedia.org/resource/Jussi_Awards, s@en, http://dbpedia.org/resource/Hugo_Award, http://dbpedia.org/resource/Peabody_Award, Motion Picture 6730 Hollywood Boulevard@en, http://dbpedia.org/resource/Academy_Awards, http://dbpedia.org/resource/British_Academy_Film_Awards, http://dbpedia.org/resource/Tony_Award, http://dbpedia.org/resource/Primetime_Emmy_Award, http://dbpedia.org/resource/Screen_Actors_Guild_Award, http://dbpedia.org/resource/Academy_Award_for_Best_Supporting_Actor, Emmy Award@en, Monk@en, Six Emmy Awards, Golden Globe Award@en] [INFO] Predicted Answer: [http://dbpedia.org/resource/David_Kirkpatrick_(producer), http://dbpedia.org/resource/Jeff_Kutash, http://dbpedia.org/resource/Mickey_Rooney, http://dbpedia.org/resource/Cantinflas, http://dbpedia.org/resource/Jayne_Mansfield, http://dbpedia.org/resource/Henri_Verneuil, http://dbpedia.org/resource/Jack_Klugman, http://dbpedia.org/resource/Omar_Sharif, http://dbpedia.org/resource/Uli_Edel, http://dbpedia.org/resource/James_Dean, http://dbpedia.org/resource/Rod_Serling, http://dbpedia.org/resource/Gilbert_Roland, http://dbpedia.org/resource/Geoffrey_Rush, http://dbpedia.org/resource/Tommy_Lee_Jones, http://dbpedia.org/resource/Tony_Shalhoub, http://dbpedia.org/resource/Alan_Alda, http://dbpedia.org/resource/Kvitka_Cisyk, http://dbpedia.org/resource/List_of_awards_and_nominations_received_by_Lionel_Richie, http://dbpedia.org/resource/List_of_awards_and_nominations_received_by_Felicity_Jones, http://dbpedia.org/resource/List_of_awards_and_nominations_received_by_Jon_Bon_Jovi, http://dbpedia.org/resource/List_of_awards_and_nominations_received_by_Justin_Timberlake, http://dbpedia.org/resource/List_of_awards_and_nominations_received_by_Cher, http://dbpedia.org/resource/List_of_awards_and_nominations_received_by_Sia_Furler, http://dbpedia.org/resource/List_of_awards_and_nominations_received_by_Miley_Cyrus, http://dbpedia.org/resource/List_of_awards_and_nominations_received_by_Carrie_Underwood, http://dbpedia.org/resource/List_of_awards_and_nominations_received_by_Björk] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 886, P: 0.502, R: 0.559, macro F1: 0.511, macro F1*: 0.529 -[INFO] Current time: 1338 ms, average: 2919.68 ms/ques, entity detection: 504.92 ms/ques (linking tool: 411.54 ms/ques), relation detection: 942.36 ms/ques (similarity: 711.72 ms/ques), EDG generation: 46.97 ms/ques +[INFO] Cumulative metrics, sample: 886, P: 0.502, R: 0.559, macro F1: 0.529 +[INFO] Current question consumed time: 1189 ms, average: 2614.59 ms/ques, entity detection: 442.31 ms/ques (linking tool: 364.45 ms/ques), relation detection: 866.48 ms/ques (similarity: 705.07 ms/ques), EDG generation: 47.48 ms/ques [INFO] Entity Linking P: 0.7392776523702032, R: 0.6918735891647856, micro F1: 0.707110609480813, macro F1: 0.7147905377017895 [INFO] Relation Linking P: 0.5992287434161023, R: 0.5451467268623025, micro F1: 0.5607975921745683, macro F1: 0.5709098046913338 [INFO] Type Linking P: 0.5598194130925508, R: 0.5598194130925508, micro F1: 0.5598194130925508, macro F1: 0.5598194130925508 @@ -12435,8 +12435,8 @@ Question 886: Which allegiance of the John Kotelawala is also the sovereign sta [INFO] Golden Answer: [http://dbpedia.org/resource/Sri_Lanka] [INFO] Predicted Answer: [http://dbpedia.org/resource/Sri_Lanka] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 887, P: 0.503, R: 0.559, macro F1: 0.511, macro F1*: 0.529 -[INFO] Current time: 1972 ms, average: 2918.61 ms/ques, entity detection: 504.84 ms/ques (linking tool: 411.29 ms/ques), relation detection: 942.05 ms/ques (similarity: 711.36 ms/ques), EDG generation: 47.05 ms/ques +[INFO] Cumulative metrics, sample: 887, P: 0.503, R: 0.559, macro F1: 0.529 +[INFO] Current question consumed time: 1600 ms, average: 2613.45 ms/ques, entity detection: 442.18 ms/ques (linking tool: 364.21 ms/ques), relation detection: 866.15 ms/ques (similarity: 704.71 ms/ques), EDG generation: 47.55 ms/ques [INFO] Entity Linking P: 0.7395715896279594, R: 0.6922209695603156, micro F1: 0.7074408117249157, macro F1: 0.7151133165851451 [INFO] Relation Linking P: 0.5996805712138293, R: 0.5456595264937993, micro F1: 0.561292747087562, macro F1: 0.5713960721204038 [INFO] Type Linking P: 0.560315670800451, R: 0.560315670800451, micro F1: 0.560315670800451, macro F1: 0.560315670800451 @@ -12449,8 +12449,8 @@ Question 887: List the television shows whose network's sister names is N-tv. [INFO] Golden Answer: [http://dbpedia.org/resource/Into_the_West_(miniseries), http://dbpedia.org/resource/The_World_of_David_the_Gnome, http://dbpedia.org/resource/King_of_Texas, http://dbpedia.org/resource/Bull_(2000_TV_series), http://dbpedia.org/resource/Saved_(TV_series), http://dbpedia.org/resource/Purgatory_(1999_film), http://dbpedia.org/resource/Doctor's_Diary, http://dbpedia.org/resource/Maple_Avenue_(TV_series), http://dbpedia.org/resource/NBA_on_TNT, http://dbpedia.org/resource/Hanna-Barbera's_50th:_A_Yabba_Dabba_Doo_Celebration, http://dbpedia.org/resource/The_Lost_World_(TV_series), http://dbpedia.org/resource/Gifted_Hands:_The_Ben_Carson_Story, http://dbpedia.org/resource/Dr._Drew_On_Call, http://dbpedia.org/resource/Nancy_Grace_(TV_series), http://dbpedia.org/resource/The_Mists_of_Avalon_(miniseries), http://dbpedia.org/resource/Witchblade_(2000_film), http://dbpedia.org/resource/Shaqtin'_a_Fool, http://dbpedia.org/resource/Ich_bin_Boes, http://dbpedia.org/resource/Die_Camper, http://dbpedia.org/resource/Babylon_5, http://dbpedia.org/resource/Leverage_(TV_series), http://dbpedia.org/resource/Perception_(U.S._TV_series), http://dbpedia.org/resource/Alarm_für_Cobra_11_–_Die_Autobahnpolizei, http://dbpedia.org/resource/The_New_Adventures_of_Robin_Hood, http://dbpedia.org/resource/Wer_wird_Millionär%3F_(Germany), http://dbpedia.org/resource/72_Hours_(TV_series), http://dbpedia.org/resource/Animal_Kingdom_(TV_series), http://dbpedia.org/resource/Boston's_Finest, http://dbpedia.org/resource/Dark_Blue_(TV_series), http://dbpedia.org/resource/Good_Behavior_(TV_series), http://dbpedia.org/resource/Heartland_(2007_U.S._TV_series), http://dbpedia.org/resource/Memphis_Beat, http://dbpedia.org/resource/Mob_City, http://dbpedia.org/resource/Murder_in_the_First_(TV_series), http://dbpedia.org/resource/NASCAR_on_TNT, http://dbpedia.org/resource/On_the_Menu, http://dbpedia.org/resource/Saving_Grace_(TV_series), http://dbpedia.org/resource/The_Company_(miniseries), http://dbpedia.org/resource/The_Grid_(miniseries), http://dbpedia.org/resource/The_Hero_(2013_TV_series), http://dbpedia.org/resource/Witchblade_(TV_series), http://dbpedia.org/resource/The_Pretender:_Island_of_the_Haunted, http://dbpedia.org/resource/The_Nightmare_Years, http://dbpedia.org/resource/Major_League_Baseball_on_TBS, http://dbpedia.org/resource/Domino_Day, http://dbpedia.org/resource/NASCAR_on_NBC, http://dbpedia.org/resource/The_Librarian:_Quest_for_the_Spear, http://dbpedia.org/resource/The_Heidi_Chronicles_(film), http://dbpedia.org/resource/The_Librarian:_Return_to_King_Solomon's_Mines, http://dbpedia.org/resource/RTL_Samstag_Nacht, http://dbpedia.org/resource/Duel_of_Hearts, http://dbpedia.org/resource/Der_Heiland_auf_dem_Eiland, http://dbpedia.org/resource/Die_Feuerengel, http://dbpedia.org/resource/Guten_Morgen,_Mallorca, http://dbpedia.org/resource/Pirates_of_Silicon_Valley, http://dbpedia.org/resource/Dallas_(2012_TV_series), http://dbpedia.org/resource/Deutschland_sucht_den_Superstar_(season_4), http://dbpedia.org/resource/WCW_Monday_Nitro, http://dbpedia.org/resource/NCAA_March_Madness_(CBS/Turner), http://dbpedia.org/resource/Morning_Express_with_Robin_Meade, http://dbpedia.org/resource/112_–_Sie_retten_dein_Leben, http://dbpedia.org/resource/Unter_Uns, http://dbpedia.org/resource/Hinter_Gittern_–_Der_Frauenknast, http://dbpedia.org/resource/40th_Daytime_Emmy_Awards, http://dbpedia.org/resource/Freedom_Song_(film), http://dbpedia.org/resource/7_Tage,_7_Köpfe, http://dbpedia.org/resource/Final_Verdict, http://dbpedia.org/resource/Arme_Millionäre, http://dbpedia.org/resource/RTL_aktuell, http://dbpedia.org/resource/Evening_Express_(HLN), http://dbpedia.org/resource/Lasko_–_Die_Faust_Gottes, http://dbpedia.org/resource/Crusade_(TV_series), http://dbpedia.org/resource/Ultra_Seven, http://dbpedia.org/resource/The_Rudy_and_Gogo_World_Famous_Cartoon_Show, http://dbpedia.org/resource/Agent_X_(TV_series), http://dbpedia.org/resource/X_Factor_(German_TV_series), http://dbpedia.org/resource/14_Hours_(2005_film), http://dbpedia.org/resource/Ebenezer_(film), http://dbpedia.org/resource/Jonny_Quest_vs._The_Cyber_Insects, http://dbpedia.org/resource/Der_Schwächste_fliegt!, http://dbpedia.org/resource/Prime_News_(Headline_News), http://dbpedia.org/resource/News_to_Me, http://dbpedia.org/resource/Deutschland_sucht_den_Superstar, http://dbpedia.org/resource/Hawthorne_(TV_series), http://dbpedia.org/resource/Never_Forget_(1991_film), http://dbpedia.org/resource/Salem's_Lot_(2004_miniseries), http://dbpedia.org/resource/Ich_bin_ein_Star_–_Holt_mich_hier_raus!, http://dbpedia.org/resource/Post_Mortem_(TV_series), http://dbpedia.org/resource/Mortal_Kombat:_Conquest, http://dbpedia.org/resource/Dollar_for_the_Dead, http://dbpedia.org/resource/Ich_bin_ein_Star_–_Holt_mich_hier_raus!_(Season_10), http://dbpedia.org/resource/Ich_bin_ein_Star_–_Holt_mich_hier_raus!_(season_9), http://dbpedia.org/resource/Alles_Atze, http://dbpedia.org/resource/Bernds_Hexe, http://dbpedia.org/resource/Mein_Leben_&_Ich, http://dbpedia.org/resource/Ritas_Welt, http://dbpedia.org/resource/Schulmädchen, http://dbpedia.org/resource/Verschollen, http://dbpedia.org/resource/Der_Lehrer, http://dbpedia.org/resource/Legends_(TV_series), http://dbpedia.org/resource/The_Librarian:_Curse_of_the_Judas_Chalice, http://dbpedia.org/resource/A_Christmas_Carol_(1999_film), http://dbpedia.org/resource/The_Closer, http://dbpedia.org/resource/The_Ron_Clark_Story, http://dbpedia.org/resource/39th_Daytime_Emmy_Awards, http://dbpedia.org/resource/Nightmares_&_Dreamscapes:_From_the_Stories_of_Stephen_King, http://dbpedia.org/resource/James_Dean_(film), http://dbpedia.org/resource/Not_Just_Another_Cable_News_Show, http://dbpedia.org/resource/Mitten_im_Leben_(TV_series)] [INFO] Predicted Answer: [http://dbpedia.org/resource/Into_the_West_(miniseries), http://dbpedia.org/resource/The_World_of_David_the_Gnome, http://dbpedia.org/resource/King_of_Texas, http://dbpedia.org/resource/Bull_(2000_TV_series), http://dbpedia.org/resource/Saved_(TV_series), http://dbpedia.org/resource/Riders_of_the_Purple_Sage_(1996_film), http://dbpedia.org/resource/Purgatory_(1999_film), http://dbpedia.org/resource/Doctor's_Diary, http://dbpedia.org/resource/Maple_Avenue_(TV_series), http://dbpedia.org/resource/NBA_on_TNT, http://dbpedia.org/resource/Hanna-Barbera's_50th:_A_Yabba_Dabba_Doo_Celebration, http://dbpedia.org/resource/The_Lost_World_(TV_series), http://dbpedia.org/resource/Gifted_Hands:_The_Ben_Carson_Story, http://dbpedia.org/resource/Dr._Drew_On_Call, http://dbpedia.org/resource/Nancy_Grace_(TV_series), http://dbpedia.org/resource/The_Mists_of_Avalon_(miniseries), http://dbpedia.org/resource/Witchblade_(2000_film), http://dbpedia.org/resource/Shaqtin'_a_Fool, http://dbpedia.org/resource/Ich_bin_Boes, http://dbpedia.org/resource/Die_Camper, http://dbpedia.org/resource/Babylon_5, http://dbpedia.org/resource/Leverage_(TV_series), http://dbpedia.org/resource/Perception_(U.S._TV_series), http://dbpedia.org/resource/Alarm_für_Cobra_11_–_Die_Autobahnpolizei, http://dbpedia.org/resource/The_New_Adventures_of_Robin_Hood, http://dbpedia.org/resource/Wer_wird_Millionär%3F_(Germany), http://dbpedia.org/resource/72_Hours_(TV_series), http://dbpedia.org/resource/Animal_Kingdom_(TV_series), http://dbpedia.org/resource/Boston's_Finest, http://dbpedia.org/resource/Dark_Blue_(TV_series), http://dbpedia.org/resource/Good_Behavior_(TV_series), http://dbpedia.org/resource/Heartland_(2007_U.S._TV_series), http://dbpedia.org/resource/Memphis_Beat, http://dbpedia.org/resource/Mob_City, http://dbpedia.org/resource/Murder_in_the_First_(TV_series), http://dbpedia.org/resource/NASCAR_on_TNT, http://dbpedia.org/resource/On_the_Menu, http://dbpedia.org/resource/Saving_Grace_(TV_series), http://dbpedia.org/resource/The_Company_(miniseries), http://dbpedia.org/resource/The_Grid_(miniseries), http://dbpedia.org/resource/The_Hero_(2013_TV_series), http://dbpedia.org/resource/Witchblade_(TV_series), http://dbpedia.org/resource/The_Pretender:_Island_of_the_Haunted, http://dbpedia.org/resource/The_Nightmare_Years, http://dbpedia.org/resource/Die_Cleveren, http://dbpedia.org/resource/Local_Edition, http://dbpedia.org/resource/Major_League_Baseball_on_TBS, http://dbpedia.org/resource/Domino_Day, http://dbpedia.org/resource/NASCAR_on_NBC, http://dbpedia.org/resource/The_Librarian:_Quest_for_the_Spear, http://dbpedia.org/resource/The_Heidi_Chronicles_(film), http://dbpedia.org/resource/The_Librarian:_Return_to_King_Solomon's_Mines, http://dbpedia.org/resource/RTL_Samstag_Nacht, http://dbpedia.org/resource/Duel_of_Hearts, http://dbpedia.org/resource/Der_Heiland_auf_dem_Eiland, http://dbpedia.org/resource/Die_Feuerengel, http://dbpedia.org/resource/Guten_Morgen,_Mallorca, http://dbpedia.org/resource/Pirates_of_Silicon_Valley, http://dbpedia.org/resource/Dallas_(2012_TV_series), http://dbpedia.org/resource/Deutschland_sucht_den_Superstar_(season_4), http://dbpedia.org/resource/WCW_Monday_Nitro, http://dbpedia.org/resource/NCAA_March_Madness_(CBS/Turner), http://dbpedia.org/resource/Morning_Express_with_Robin_Meade, http://dbpedia.org/resource/112_–_Sie_retten_dein_Leben, http://dbpedia.org/resource/Unter_Uns, http://dbpedia.org/resource/Hinter_Gittern_–_Der_Frauenknast, http://dbpedia.org/resource/Freedom_Song_(film), http://dbpedia.org/resource/7_Tage,_7_Köpfe, http://dbpedia.org/resource/Final_Verdict, http://dbpedia.org/resource/Arme_Millionäre, http://dbpedia.org/resource/RTL_aktuell, http://dbpedia.org/resource/Evening_Express_(HLN), http://dbpedia.org/resource/Lasko_–_Die_Faust_Gottes, http://dbpedia.org/resource/Babylon_5:_A_Call_to_Arms, http://dbpedia.org/resource/Babylon_5:_In_the_Beginning, http://dbpedia.org/resource/Babylon_5:_The_River_of_Souls, http://dbpedia.org/resource/Babylon_5:_Thirdspace, http://dbpedia.org/resource/Crusade_(TV_series), http://dbpedia.org/resource/Ultra_Seven, http://dbpedia.org/resource/The_Rudy_and_Gogo_World_Famous_Cartoon_Show, http://dbpedia.org/resource/Agent_X_(TV_series), http://dbpedia.org/resource/World_Idol, http://dbpedia.org/resource/X_Factor_(German_TV_series), http://dbpedia.org/resource/14_Hours_(2005_film), http://dbpedia.org/resource/Ebenezer_(film), http://dbpedia.org/resource/Jonny_Quest_vs._The_Cyber_Insects, http://dbpedia.org/resource/Der_Schwächste_fliegt!, http://dbpedia.org/resource/Prime_News_(Headline_News), http://dbpedia.org/resource/News_to_Me, http://dbpedia.org/resource/Deutschland_sucht_den_Superstar, http://dbpedia.org/resource/Hawthorne_(TV_series), http://dbpedia.org/resource/Never_Forget_(1991_film), http://dbpedia.org/resource/Salem's_Lot_(2004_miniseries), http://dbpedia.org/resource/Ich_bin_ein_Star_–_Holt_mich_hier_raus!, http://dbpedia.org/resource/Post_Mortem_(TV_series), http://dbpedia.org/resource/Mortal_Kombat:_Conquest, http://dbpedia.org/resource/Dollar_for_the_Dead, http://dbpedia.org/resource/Ich_bin_ein_Star_–_Holt_mich_hier_raus!_(Season_10), http://dbpedia.org/resource/Ich_bin_ein_Star_–_Holt_mich_hier_raus!_(season_9), http://dbpedia.org/resource/Alles_Atze, http://dbpedia.org/resource/Bernds_Hexe, http://dbpedia.org/resource/Mein_Leben_&_Ich, http://dbpedia.org/resource/Ritas_Welt, http://dbpedia.org/resource/Schulmädchen, http://dbpedia.org/resource/Verschollen, http://dbpedia.org/resource/Der_Lehrer, http://dbpedia.org/resource/Legends_(TV_series), http://dbpedia.org/resource/The_Librarian:_Curse_of_the_Judas_Chalice, http://dbpedia.org/resource/A_Christmas_Carol_(1999_film), http://dbpedia.org/resource/The_Closer, http://dbpedia.org/resource/Galidor:_Defenders_of_the_Outer_Dimension, http://dbpedia.org/resource/The_Ron_Clark_Story, http://dbpedia.org/resource/The_Heart_of_Justice, http://dbpedia.org/resource/Nightmares_&_Dreamscapes:_From_the_Stories_of_Stephen_King, http://dbpedia.org/resource/Archibald_the_Koala, http://dbpedia.org/resource/James_Dean_(film), http://dbpedia.org/resource/Not_Just_Another_Cable_News_Show, http://dbpedia.org/resource/Mitten_im_Leben_(TV_series), http://dbpedia.org/resource/2013_Viña_del_Mar_International_Song_Festival, http://dbpedia.org/resource/2006_Viña_del_Mar_International_Song_Festival, http://dbpedia.org/resource/2016_Viña_del_Mar_International_Song_Festival, http://dbpedia.org/resource/American_Morning, http://dbpedia.org/resource/Crossfire_(TV_series), http://dbpedia.org/resource/Planet_in_Peril, http://dbpedia.org/resource/Your_Bottom_Line, http://dbpedia.org/resource/Manuel_Rodríguez_(2010_telenovela), http://dbpedia.org/resource/Anderson_Cooper_360°, http://dbpedia.org/resource/State_of_the_Union_(TV_series), http://dbpedia.org/resource/International_Desk, http://dbpedia.org/resource/Both_Sides_with_Jesse_Jackson, http://dbpedia.org/resource/New_Year's_Eve_Live_(CNN_program), http://dbpedia.org/resource/Anthony_Bourdain:_Parts_Unknown, http://dbpedia.org/resource/CNN_Daybreak, http://dbpedia.org/resource/CNN_Live_Today, http://dbpedia.org/resource/CNN_Newsroom, http://dbpedia.org/resource/CNN_Newsroom_(CNNI), http://dbpedia.org/resource/CNN_Presents, http://dbpedia.org/resource/CNN_Special_Investigations_Unit, http://dbpedia.org/resource/CNN_Tonight, http://dbpedia.org/resource/Campbell_Brown_(TV_series), http://dbpedia.org/resource/Connie_Chung_Tonight, http://dbpedia.org/resource/Early_Start, http://dbpedia.org/resource/In_the_Arena, http://dbpedia.org/resource/Lou_Dobbs_Tonight, http://dbpedia.org/resource/NewsNight_with_Aaron_Brown, http://dbpedia.org/resource/Piers_Morgan_Live, http://dbpedia.org/resource/Reliable_Sources, http://dbpedia.org/resource/Rick's_List, http://dbpedia.org/resource/Sanjay_Gupta_MD, http://dbpedia.org/resource/Somebody's_Gotta_Do_It, http://dbpedia.org/resource/Starting_Point, http://dbpedia.org/resource/The_Lead_with_Jake_Tapper, http://dbpedia.org/resource/Morgan_Spurlock_Inside_Man, http://dbpedia.org/resource/This_Week_in_Politics, http://dbpedia.org/resource/Larry_King_Live, http://dbpedia.org/resource/Fareed_Zakaria_GPS, http://dbpedia.org/resource/Late_Edition_with_Wolf_Blitzer, http://dbpedia.org/resource/La_mañana_de_Chilevisión, http://dbpedia.org/resource/New_Day_(TV_series), http://dbpedia.org/resource/CNN_Heroes, http://dbpedia.org/resource/The_Situation_Room_with_Wolf_Blitzer, http://dbpedia.org/resource/Amanpour, http://dbpedia.org/resource/The_Hunt_with_John_Walsh, http://dbpedia.org/resource/The_Real_Football_Factories_International, http://dbpedia.org/resource/This_Is_Life_with_Lisa_Ling, http://dbpedia.org/resource/(Get_to)_The_Point, http://dbpedia.org/resource/Your_Money, http://dbpedia.org/resource/Achieving_the_Perfect_10, http://dbpedia.org/resource/The_Next_List, http://dbpedia.org/resource/TalkAsia, http://dbpedia.org/resource/CNN_World_News, http://dbpedia.org/resource/Mala_Conducta, http://dbpedia.org/resource/32._Gün, http://dbpedia.org/resource/D._L._Hughley_Breaks_the_News, http://dbpedia.org/resource/Chicagoland_(TV_series), http://dbpedia.org/resource/Chilevisión_Noticias, http://dbpedia.org/resource/CNN:_The_Seventies, http://dbpedia.org/resource/CNN:_The_Sixties, http://dbpedia.org/resource/2011_Viña_del_Mar_International_Song_Festival, http://dbpedia.org/resource/2015_Viña_del_Mar_International_Song_Festival, http://dbpedia.org/resource/40th_Daytime_Emmy_Awards, http://dbpedia.org/resource/39th_Daytime_Emmy_Awards, http://dbpedia.org/resource/Tech_Toyz, http://dbpedia.org/resource/Real_Heroes_Award] [INFO] P: 0.592391304347826, R: 1.0, F: 0.7440273037542662 -[INFO] Cumulative metrics, sample: 888, P: 0.503, R: 0.560, macro F1: 0.512, macro F1*: 0.530 -[INFO] Current time: 3057 ms, average: 2918.77 ms/ques, entity detection: 504.43 ms/ques (linking tool: 410.94 ms/ques), relation detection: 942.50 ms/ques (similarity: 711.78 ms/ques), EDG generation: 47.02 ms/ques +[INFO] Cumulative metrics, sample: 888, P: 0.503, R: 0.560, macro F1: 0.530 +[INFO] Current question consumed time: 2568 ms, average: 2613.40 ms/ques, entity detection: 441.80 ms/ques (linking tool: 363.88 ms/ques), relation detection: 866.48 ms/ques (similarity: 705.13 ms/ques), EDG generation: 47.52 ms/ques [INFO] Entity Linking P: 0.7398648648648649, R: 0.6925675675675675, micro F1: 0.7077702702702705, macro F1: 0.7154353646098929 [INFO] Relation Linking P: 0.6001313813813813, R: 0.5461711711711712, micro F1: 0.5617867867867877, macro F1: 0.5718812344886843 [INFO] Type Linking P: 0.5596846846846847, R: 0.5596846846846847, micro F1: 0.5596846846846847, macro F1: 0.5596846846846847 @@ -12463,8 +12463,8 @@ Question 888: God Defend New Zealand is the national anthem of which country? [INFO] Golden Answer: [http://dbpedia.org/resource/New_Zealand] [INFO] Predicted Answer: [http://dbpedia.org/resource/New_Zealand] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 889, P: 0.503, R: 0.560, macro F1: 0.512, macro F1*: 0.530 -[INFO] Current time: 1705 ms, average: 2917.40 ms/ques, entity detection: 504.88 ms/ques (linking tool: 411.42 ms/ques), relation detection: 941.71 ms/ques (similarity: 711.25 ms/ques), EDG generation: 46.98 ms/ques +[INFO] Cumulative metrics, sample: 889, P: 0.503, R: 0.560, macro F1: 0.530 +[INFO] Current question consumed time: 1405 ms, average: 2612.04 ms/ques, entity detection: 442.14 ms/ques (linking tool: 364.26 ms/ques), relation detection: 865.75 ms/ques (similarity: 704.60 ms/ques), EDG generation: 47.48 ms/ques [INFO] Entity Linking P: 0.7401574803149606, R: 0.6929133858267716, micro F1: 0.7080989876265469, macro F1: 0.7157566842606212 [INFO] Relation Linking P: 0.5994563179602549, R: 0.545556805399325, micro F1: 0.5611548556430456, macro F1: 0.5712379485106318 [INFO] Type Linking P: 0.5590551181102362, R: 0.5590551181102362, micro F1: 0.5590551181102362, macro F1: 0.5590551181102362 @@ -12477,8 +12477,8 @@ Question 889: Where do the rivers starting from Zambia end? [INFO] Golden Answer: [http://dbpedia.org/resource/Lualaba_River, http://dbpedia.org/resource/Indian_Ocean, http://dbpedia.org/resource/Zambezi] [INFO] Predicted Answer: [http://dbpedia.org/resource/Luangwa_River, http://dbpedia.org/resource/Kabompo_River, http://dbpedia.org/resource/Kafue_River] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 890, P: 0.503, R: 0.560, macro F1: 0.512, macro F1*: 0.530 -[INFO] Current time: 3664 ms, average: 2918.24 ms/ques, entity detection: 504.81 ms/ques (linking tool: 411.43 ms/ques), relation detection: 942.72 ms/ques (similarity: 711.61 ms/ques), EDG generation: 46.94 ms/ques +[INFO] Cumulative metrics, sample: 890, P: 0.503, R: 0.560, macro F1: 0.530 +[INFO] Current question consumed time: 3065 ms, average: 2612.54 ms/ques, entity detection: 442.02 ms/ques (linking tool: 364.21 ms/ques), relation detection: 866.61 ms/ques (similarity: 704.97 ms/ques), EDG generation: 47.44 ms/ques [INFO] Entity Linking P: 0.7404494382022472, R: 0.6932584269662921, micro F1: 0.7084269662921351, macro F1: 0.7160772780106373 [INFO] Relation Linking P: 0.5987827715355805, R: 0.5449438202247191, micro F1: 0.5605243445692893, macro F1: 0.5705961081190468 [INFO] Type Linking P: 0.5584269662921348, R: 0.5584269662921348, micro F1: 0.5584269662921348, macro F1: 0.5584269662921348 @@ -12491,8 +12491,8 @@ Question 890: Which political party of kumta had affiliation with Indian general [INFO] Golden Answer: [http://dbpedia.org/resource/Indian_National_Congress] [INFO] Predicted Answer: [http://dbpedia.org/resource/All_India_Christian_Council] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 891, P: 0.502, R: 0.559, macro F1: 0.511, macro F1*: 0.529 -[INFO] Current time: 4821 ms, average: 2920.38 ms/ques, entity detection: 505.08 ms/ques (linking tool: 411.68 ms/ques), relation detection: 943.25 ms/ques (similarity: 712.40 ms/ques), EDG generation: 46.97 ms/ques +[INFO] Cumulative metrics, sample: 891, P: 0.502, R: 0.559, macro F1: 0.529 +[INFO] Current question consumed time: 4287 ms, average: 2614.43 ms/ques, entity detection: 442.32 ms/ques (linking tool: 364.49 ms/ques), relation detection: 867.08 ms/ques (similarity: 705.75 ms/ques), EDG generation: 47.43 ms/ques [INFO] Entity Linking P: 0.7396184062850729, R: 0.6924803591470258, micro F1: 0.7076318742985412, macro F1: 0.7152735998086049 [INFO] Relation Linking P: 0.5981107369996258, R: 0.5443322109988776, micro F1: 0.5598952487841387, macro F1: 0.5699557084466349 [INFO] Type Linking P: 0.5578002244668911, R: 0.5578002244668911, micro F1: 0.5578002244668911, macro F1: 0.5578002244668911 @@ -12505,8 +12505,8 @@ Question 891: Who was member of boards at Royal Society and Trinity House? [INFO] Golden Answer: [http://dbpedia.org/resource/Samuel_Pepys] [INFO] Predicted Answer: [http://dbpedia.org/resource/Antonie_van_Leeuwenhoek] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 892, P: 0.502, R: 0.558, macro F1: 0.510, macro F1*: 0.529 -[INFO] Current time: 3229 ms, average: 2920.72 ms/ques, entity detection: 505.32 ms/ques (linking tool: 411.81 ms/ques), relation detection: 943.28 ms/ques (similarity: 712.12 ms/ques), EDG generation: 46.97 ms/ques +[INFO] Cumulative metrics, sample: 892, P: 0.502, R: 0.558, macro F1: 0.529 +[INFO] Current question consumed time: 2765 ms, average: 2614.59 ms/ques, entity detection: 442.50 ms/ques (linking tool: 364.59 ms/ques), relation detection: 867.08 ms/ques (similarity: 705.49 ms/ques), EDG generation: 47.43 ms/ques [INFO] Entity Linking P: 0.7399103139013453, R: 0.6922645739910314, micro F1: 0.7075859491778776, macro F1: 0.7152949022842751 [INFO] Relation Linking P: 0.5974402092675635, R: 0.5437219730941704, micro F1: 0.5592675635276542, macro F1: 0.5693167446479279 [INFO] Type Linking P: 0.5582959641255605, R: 0.5582959641255605, micro F1: 0.5582959641255605, macro F1: 0.5582959641255605 @@ -12519,8 +12519,8 @@ Question 892: Where is the king buried whose predecessor was Iyasu II? [INFO] Golden Answer: [http://dbpedia.org/resource/Ethiopia, http://dbpedia.org/resource/Gondar] [INFO] Predicted Answer: [http://dbpedia.org/resource/Ethiopia, http://dbpedia.org/resource/Gondar] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 893, P: 0.502, R: 0.559, macro F1: 0.511, macro F1*: 0.529 -[INFO] Current time: 1243 ms, average: 2918.85 ms/ques, entity detection: 504.93 ms/ques (linking tool: 411.45 ms/ques), relation detection: 942.64 ms/ques (similarity: 711.65 ms/ques), EDG generation: 46.94 ms/ques +[INFO] Cumulative metrics, sample: 893, P: 0.502, R: 0.559, macro F1: 0.529 +[INFO] Current question consumed time: 1034 ms, average: 2612.82 ms/ques, entity detection: 442.13 ms/ques (linking tool: 364.26 ms/ques), relation detection: 866.48 ms/ques (similarity: 705.03 ms/ques), EDG generation: 47.39 ms/ques [INFO] Entity Linking P: 0.7402015677491601, R: 0.6926091825307951, micro F1: 0.7079134005225832, macro F1: 0.7156149584257218 [INFO] Relation Linking P: 0.5978910041060097, R: 0.5442329227323628, micro F1: 0.5597611048898853, macro F1: 0.5698015092648498 [INFO] Type Linking P: 0.5576707726763718, R: 0.5576707726763718, micro F1: 0.5576707726763718, macro F1: 0.5576707726763718 @@ -12533,8 +12533,8 @@ Question 893: In how many different places have people beatified by Pope Pius X [INFO] Golden Answer: [17] [INFO] Predicted Answer: [] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 894, P: 0.502, R: 0.558, macro F1: 0.510, macro F1*: 0.528 -[INFO] Current time: 1504 ms, average: 2917.26 ms/ques, entity detection: 504.73 ms/ques (linking tool: 411.30 ms/ques), relation detection: 941.86 ms/ques (similarity: 711.04 ms/ques), EDG generation: 46.90 ms/ques +[INFO] Cumulative metrics, sample: 894, P: 0.502, R: 0.558, macro F1: 0.528 +[INFO] Current question consumed time: 1446 ms, average: 2611.52 ms/ques, entity detection: 441.99 ms/ques (linking tool: 364.14 ms/ques), relation detection: 865.77 ms/ques (similarity: 704.43 ms/ques), EDG generation: 47.36 ms/ques [INFO] Entity Linking P: 0.7393736017897091, R: 0.691834451901566, micro F1: 0.7071215510812828, macro F1: 0.7148144942664089 [INFO] Relation Linking P: 0.5972222222222222, R: 0.5436241610738255, micro F1: 0.5591349739000755, macro F1: 0.5691641473976632 [INFO] Type Linking P: 0.5570469798657718, R: 0.5570469798657718, micro F1: 0.5570469798657718, macro F1: 0.5570469798657718 @@ -12547,8 +12547,8 @@ Question 894: What faiths are followed by the relatives of Meadow Soprano? [INFO] Golden Answer: [http://dbpedia.org/resource/Catholic_Church, http://dbpedia.org/resource/Buddhism, http://dbpedia.org/resource/Born_again_(Christianity)] [INFO] Predicted Answer: [http://dbpedia.org/resource/Catholic_Church, http://dbpedia.org/resource/Buddhism, http://dbpedia.org/resource/Born_again_(Christianity), Roman Catholicism@en] [INFO] P: 0.75, R: 1.0, F: 0.8571428571428571 -[INFO] Cumulative metrics, sample: 895, P: 0.502, R: 0.559, macro F1: 0.511, macro F1*: 0.529 -[INFO] Current time: 1946 ms, average: 2916.18 ms/ques, entity detection: 504.62 ms/ques (linking tool: 411.18 ms/ques), relation detection: 941.45 ms/ques (similarity: 710.73 ms/ques), EDG generation: 46.95 ms/ques +[INFO] Cumulative metrics, sample: 895, P: 0.502, R: 0.559, macro F1: 0.529 +[INFO] Current question consumed time: 1712 ms, average: 2610.51 ms/ques, entity detection: 441.86 ms/ques (linking tool: 364.03 ms/ques), relation detection: 865.38 ms/ques (similarity: 704.12 ms/ques), EDG generation: 47.41 ms/ques [INFO] Entity Linking P: 0.7396648044692737, R: 0.6921787709497207, micro F1: 0.7074487895716948, macro F1: 0.7151343681134844 [INFO] Relation Linking P: 0.5976722532588454, R: 0.5441340782122905, micro F1: 0.5596275605214163, macro F1: 0.5696479895693864 [INFO] Type Linking P: 0.5575418994413408, R: 0.5575418994413408, micro F1: 0.5575418994413408, macro F1: 0.5575418994413408 @@ -12561,8 +12561,8 @@ Question 895: Where is the headquarters of the public transit system which owns [INFO] Golden Answer: [http://dbpedia.org/resource/Minneapolis, http://dbpedia.org/resource/Minnesota] [INFO] Predicted Answer: [] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 896, P: 0.501, R: 0.558, macro F1: 0.510, macro F1*: 0.528 -[INFO] Current time: 2677 ms, average: 2915.91 ms/ques, entity detection: 504.40 ms/ques (linking tool: 410.96 ms/ques), relation detection: 941.39 ms/ques (similarity: 710.73 ms/ques), EDG generation: 46.96 ms/ques +[INFO] Cumulative metrics, sample: 896, P: 0.501, R: 0.558, macro F1: 0.528 +[INFO] Current question consumed time: 2262 ms, average: 2610.12 ms/ques, entity detection: 441.63 ms/ques (linking tool: 363.81 ms/ques), relation detection: 865.31 ms/ques (similarity: 704.19 ms/ques), EDG generation: 47.43 ms/ques [INFO] Entity Linking P: 0.7388392857142857, R: 0.69140625, micro F1: 0.7066592261904764, macro F1: 0.7143362270776433 [INFO] Relation Linking P: 0.5970052083333333, R: 0.5435267857142857, micro F1: 0.5590029761904771, macro F1: 0.5690122217238848 [INFO] Type Linking P: 0.5580357142857143, R: 0.5580357142857143, micro F1: 0.5580357142857143, macro F1: 0.5580357142857143 @@ -12575,8 +12575,8 @@ Question 896: What is the nationality of Aishath Saffa ? [INFO] Golden Answer: [http://dbpedia.org/resource/Maldives] [INFO] Predicted Answer: [http://dbpedia.org/resource/Maldives] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 897, P: 0.502, R: 0.559, macro F1: 0.511, macro F1*: 0.529 -[INFO] Current time: 395 ms, average: 2913.10 ms/ques, entity detection: 503.94 ms/ques (linking tool: 410.57 ms/ques), relation detection: 940.47 ms/ques (similarity: 710.01 ms/ques), EDG generation: 46.93 ms/ques +[INFO] Cumulative metrics, sample: 897, P: 0.502, R: 0.559, macro F1: 0.529 +[INFO] Current question consumed time: 313 ms, average: 2607.56 ms/ques, entity detection: 441.22 ms/ques (linking tool: 363.46 ms/ques), relation detection: 864.46 ms/ques (similarity: 703.48 ms/ques), EDG generation: 47.39 ms/ques [INFO] Entity Linking P: 0.7391304347826086, R: 0.6917502787068004, micro F1: 0.7069862504645116, macro F1: 0.7146559170745752 [INFO] Relation Linking P: 0.5974544778892604, R: 0.5440356744704571, micro F1: 0.5594946116685257, macro F1: 0.5694951448717359 [INFO] Type Linking P: 0.5585284280936454, R: 0.5585284280936454, micro F1: 0.5585284280936454, macro F1: 0.5585284280936454 @@ -12589,8 +12589,8 @@ Question 897: who was the president under whom winston bryant served as a lieute [INFO] Golden Answer: [http://dbpedia.org/resource/Bill_Clinton, http://dbpedia.org/resource/Frank_D._White, http://dbpedia.org/resource/Presidency_of_Bill_Clinton] [INFO] Predicted Answer: [http://dbpedia.org/resource/Bill_Clinton, http://dbpedia.org/resource/Frank_D._White, http://dbpedia.org/resource/Presidency_of_Bill_Clinton] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 898, P: 0.502, R: 0.559, macro F1: 0.511, macro F1*: 0.529 -[INFO] Current time: 1173 ms, average: 2911.16 ms/ques, entity detection: 503.50 ms/ques (linking tool: 410.17 ms/ques), relation detection: 939.87 ms/ques (similarity: 709.68 ms/ques), EDG generation: 46.90 ms/ques +[INFO] Cumulative metrics, sample: 898, P: 0.502, R: 0.559, macro F1: 0.529 +[INFO] Current question consumed time: 983 ms, average: 2605.76 ms/ques, entity detection: 440.82 ms/ques (linking tool: 363.09 ms/ques), relation detection: 863.89 ms/ques (similarity: 703.15 ms/ques), EDG generation: 47.37 ms/ques [INFO] Entity Linking P: 0.7394209354120267, R: 0.6920935412026726, micro F1: 0.7073125463994063, macro F1: 0.714974891261882 [INFO] Relation Linking P: 0.5979027468448403, R: 0.544543429844098, micro F1: 0.559985152190053, macro F1: 0.5699769829397325 [INFO] Type Linking P: 0.5579064587973274, R: 0.5579064587973274, micro F1: 0.5579064587973274, macro F1: 0.5579064587973274 @@ -12603,8 +12603,8 @@ Question 898: What are the bands associated with the artists of My Favorite Girl [INFO] Golden Answer: [http://dbpedia.org/resource/Bruce_Hornsby, http://dbpedia.org/resource/Erick_Sermon, http://dbpedia.org/resource/Digital_Underground, http://dbpedia.org/resource/Teddy_Riley, http://dbpedia.org/resource/Usher_(singer), http://dbpedia.org/resource/Tupac_Shakur, http://dbpedia.org/resource/K-Ci_&_JoJo, http://dbpedia.org/resource/Tim_&_Bob, http://dbpedia.org/resource/Mike_City, http://dbpedia.org/resource/Eric_Roberson] [INFO] Predicted Answer: [http://dbpedia.org/resource/Tank_(American_singer), http://dbpedia.org/resource/Def_Squad, http://dbpedia.org/resource/Tim_&_Bob, http://dbpedia.org/resource/Big_Bert, http://dbpedia.org/resource/Mike_City, http://dbpedia.org/resource/Chopmaster_J, http://dbpedia.org/resource/Bruce_Hornsby, http://dbpedia.org/resource/Erick_Sermon, http://dbpedia.org/resource/Digital_Underground, http://dbpedia.org/resource/Teddy_Riley, http://dbpedia.org/resource/Usher_(singer), http://dbpedia.org/resource/Tupac_Shakur, http://dbpedia.org/resource/K-Ci_&_JoJo, http://dbpedia.org/resource/Eric_Roberson] [INFO] P: 0.7142857142857143, R: 1.0, F: 0.8333333333333333 -[INFO] Cumulative metrics, sample: 899, P: 0.503, R: 0.560, macro F1: 0.512, macro F1*: 0.530 -[INFO] Current time: 1318 ms, average: 2909.39 ms/ques, entity detection: 503.20 ms/ques (linking tool: 409.88 ms/ques), relation detection: 939.34 ms/ques (similarity: 709.19 ms/ques), EDG generation: 46.95 ms/ques +[INFO] Cumulative metrics, sample: 899, P: 0.503, R: 0.560, macro F1: 0.530 +[INFO] Current question consumed time: 1041 ms, average: 2604.01 ms/ques, entity detection: 440.47 ms/ques (linking tool: 362.78 ms/ques), relation detection: 863.37 ms/ques (similarity: 702.67 ms/ques), EDG generation: 47.42 ms/ques [INFO] Entity Linking P: 0.7397107897664071, R: 0.6924360400444939, micro F1: 0.7076381164256583, macro F1: 0.7152931520459627 [INFO] Relation Linking P: 0.5983500185391175, R: 0.5450500556173526, micro F1: 0.5604746014089739, macro F1: 0.5704577396043796 [INFO] Type Linking P: 0.5583982202447163, R: 0.5583982202447163, micro F1: 0.5583982202447163, macro F1: 0.5583982202447163 @@ -12617,8 +12617,8 @@ Question 899: On what subjects was Broca's Brain written? [INFO] Golden Answer: [http://dbpedia.org/resource/Brain, http://dbpedia.org/resource/Intellect, http://dbpedia.org/resource/Paul_Broca, http://dbpedia.org/resource/Outline_of_space_science] [INFO] Predicted Answer: [http://dbpedia.org/resource/Aquitaine, Aquitaine@en] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 900, P: 0.502, R: 0.559, macro F1: 0.511, macro F1*: 0.529 -[INFO] Current time: 1582 ms, average: 2907.92 ms/ques, entity detection: 502.86 ms/ques (linking tool: 409.52 ms/ques), relation detection: 938.80 ms/ques (similarity: 708.93 ms/ques), EDG generation: 46.91 ms/ques +[INFO] Cumulative metrics, sample: 900, P: 0.502, R: 0.559, macro F1: 0.529 +[INFO] Current question consumed time: 1292 ms, average: 2602.56 ms/ques, entity detection: 440.15 ms/ques (linking tool: 362.45 ms/ques), relation detection: 862.87 ms/ques (similarity: 702.41 ms/ques), EDG generation: 47.38 ms/ques [INFO] Entity Linking P: 0.7388888888888889, R: 0.6916666666666667, micro F1: 0.7068518518518521, macro F1: 0.7144983818770226 [INFO] Relation Linking P: 0.5976851851851851, R: 0.5444444444444444, micro F1: 0.5598518518518528, macro F1: 0.5698238976714857 [INFO] Type Linking P: 0.5588888888888889, R: 0.5588888888888889, micro F1: 0.5588888888888889, macro F1: 0.5588888888888889 @@ -12631,8 +12631,8 @@ Question 900: Which parent organization of GetTV is also the label of next thing [INFO] Golden Answer: [http://dbpedia.org/resource/Sony] [INFO] Predicted Answer: [http://dbpedia.org/resource/Sony] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 901, P: 0.503, R: 0.560, macro F1: 0.512, macro F1*: 0.530 -[INFO] Current time: 1142 ms, average: 2905.96 ms/ques, entity detection: 502.42 ms/ques (linking tool: 409.13 ms/ques), relation detection: 938.28 ms/ques (similarity: 708.62 ms/ques), EDG generation: 46.95 ms/ques +[INFO] Cumulative metrics, sample: 901, P: 0.503, R: 0.560, macro F1: 0.530 +[INFO] Current question consumed time: 988 ms, average: 2600.77 ms/ques, entity detection: 439.75 ms/ques (linking tool: 362.10 ms/ques), relation detection: 862.37 ms/ques (similarity: 702.10 ms/ques), EDG generation: 47.43 ms/ques [INFO] Entity Linking P: 0.7391786903440621, R: 0.6914539400665927, micro F1: 0.7068072512023679, macro F1: 0.7145202856235077 [INFO] Relation Linking P: 0.5981317055123936, R: 0.5443951165371809, micro F1: 0.5599704032556428, macro F1: 0.56999971159167 [INFO] Type Linking P: 0.5593784683684795, R: 0.5593784683684795, micro F1: 0.5593784683684795, macro F1: 0.5593784683684795 @@ -12645,8 +12645,8 @@ Question 901: Which teams have used the stadium whose architect was Populous ? [INFO] Golden Answer: [http://dbpedia.org/resource/Pensacola_Blue_Wahoos, http://dbpedia.org/resource/West_Florida_Argonauts, http://dbpedia.org/resource/Missouri_Tigers, http://dbpedia.org/resource/Missouri_Tigers_men's_basketball, http://dbpedia.org/resource/Arizona_League_Cubs, http://dbpedia.org/resource/Chicago_Cubs, http://dbpedia.org/resource/Mesa_Solar_Sox, http://dbpedia.org/resource/Chicago_White_Sox, http://dbpedia.org/resource/Colorado_Rapids, http://dbpedia.org/resource/Colorado_Crush, http://dbpedia.org/resource/Colorado_Mammoth, http://dbpedia.org/resource/Colorado_Avalanche, http://dbpedia.org/resource/Denver_Nuggets, http://dbpedia.org/resource/Iowa_Cubs, http://dbpedia.org/resource/Nashville_Predators, http://dbpedia.org/resource/Nashville_Kats, http://dbpedia.org/resource/Gold_Coast_Chargers, http://dbpedia.org/resource/Gold_Coast_Clippers, http://dbpedia.org/resource/Gold_Coast_Cougars, http://dbpedia.org/resource/Gold_Coast_Football_Club, http://dbpedia.org/resource/Brisbane_Bears, http://dbpedia.org/resource/North_Melbourne_Football_Club, http://dbpedia.org/resource/Queensland_Country_(NRC_team), http://dbpedia.org/resource/Nerang_Bulls_RUC, http://dbpedia.org/resource/2018_Commonwealth_Games, http://dbpedia.org/resource/Gold_Coast_Titans, http://dbpedia.org/resource/Daikyo_Dolphins, http://dbpedia.org/resource/George_Mason_Patriots, http://dbpedia.org/resource/Washington_Commandos, http://dbpedia.org/resource/Kansas_City_Command, http://dbpedia.org/resource/Auckland_Rugby_Football_Union, http://dbpedia.org/resource/Blues_(Super_Rugby), http://dbpedia.org/resource/Auckland_cricket_team, http://dbpedia.org/resource/Australian_Open, http://dbpedia.org/resource/Davis_Cup, http://dbpedia.org/resource/Melbourne_United, http://dbpedia.org/resource/Melbourne_Vixens, http://dbpedia.org/resource/Minnesota_Wild, http://dbpedia.org/resource/Minnesota_Swarm, http://dbpedia.org/resource/Jacksonville_Sharks, http://dbpedia.org/resource/Jacksonville_Bullies, http://dbpedia.org/resource/Southern_Professional_Hockey_League, http://dbpedia.org/resource/Jacksonville_Barracudas, http://dbpedia.org/resource/Jacksonville_Dolphins, http://dbpedia.org/resource/Jacksonville_Giants, http://dbpedia.org/resource/Louisville_Cardinals, http://dbpedia.org/resource/Louisville_Cardinals_men's_basketball, http://dbpedia.org/resource/Manchester_Wolves, http://dbpedia.org/resource/Manchester_Monarchs_(AHL), http://dbpedia.org/resource/Manchester_Monarchs_(ECHL), http://dbpedia.org/resource/Delaware_87ers, http://dbpedia.org/resource/Delaware_Fightin'_Blue_Hens, http://dbpedia.org/resource/Santa_Clara_University, http://dbpedia.org/resource/Wichita_Force, http://dbpedia.org/resource/Wichita_State_Shockers_men's_basketball, http://dbpedia.org/resource/Wichita_Thunder, http://dbpedia.org/resource/Akron_RubberDucks, http://dbpedia.org/resource/Louletano_D.C., http://dbpedia.org/resource/S.C._Farense, http://dbpedia.org/resource/Gibraltar_national_football_team, http://dbpedia.org/resource/Algarve_United_F.C., http://dbpedia.org/resource/Minnesota–Duluth_Bulldogs_women's_ice_hockey, http://dbpedia.org/resource/Minnesota–Duluth_Bulldogs_men's_ice_hockey, http://dbpedia.org/resource/A-League, http://dbpedia.org/resource/Australian_Sevens, http://dbpedia.org/resource/AFF_Championship, http://dbpedia.org/resource/Jacksonville_Jaguars, http://dbpedia.org/resource/TaxSlayer_Bowl, http://dbpedia.org/resource/Jacksonville_Armada_FC, http://dbpedia.org/resource/Cleveland_Browns, http://dbpedia.org/resource/New_England_Patriots, http://dbpedia.org/resource/Massachusetts_Interscholastic_Athletic_Association, http://dbpedia.org/resource/Boston_Cannons, http://dbpedia.org/resource/New_England_Revolution, http://dbpedia.org/resource/2016_NHL_Winter_Classic, http://dbpedia.org/resource/UMass_Minutemen_football, http://dbpedia.org/resource/2016_Outdoor_Women's_Classic, http://dbpedia.org/resource/Pittsburgh_Panthers, http://dbpedia.org/resource/Pittsburgh_Steelers, http://dbpedia.org/resource/Baltimore_Ravens, http://dbpedia.org/resource/Chesapeake_Bayhawks, http://dbpedia.org/resource/Houston_Texans, http://dbpedia.org/resource/Houston_Livestock_Show_and_Rodeo, http://dbpedia.org/resource/Texas_Bowl, http://dbpedia.org/resource/Houston_Cougars_football, http://dbpedia.org/resource/Tennessee_Titans, http://dbpedia.org/resource/Tennessee_State_Tigers_football, http://dbpedia.org/resource/Music_City_Bowl, http://dbpedia.org/resource/Buffalo_Bills, http://dbpedia.org/resource/Miami_Dolphins, http://dbpedia.org/resource/Russell_Athletic_Bowl, http://dbpedia.org/resource/Orange_Bowl, http://dbpedia.org/resource/Miami_Marlins, http://dbpedia.org/resource/Miami_Hurricanes_football, http://dbpedia.org/resource/Florida_Atlantic_University, http://dbpedia.org/resource/Minnesota_Vikings, http://dbpedia.org/resource/Minnesota_Golden_Gophers_football, http://dbpedia.org/resource/Freedom_Bowl, http://dbpedia.org/resource/Los_Angeles_Rams, http://dbpedia.org/resource/California_Surf, http://dbpedia.org/resource/Los_Angeles_Angels_of_Anaheim, http://dbpedia.org/resource/Southern_California_Sun, http://dbpedia.org/resource/Albuquerque_Isotopes, http://dbpedia.org/resource/New_Mexico_Lobos_baseball, http://dbpedia.org/resource/Sydney_Thunder_(WBBL), http://dbpedia.org/resource/2000_Summer_Olympics, http://dbpedia.org/resource/Greater_Western_Sydney_Giants, http://dbpedia.org/resource/Canterbury-Bankstown_Bulldogs, http://dbpedia.org/resource/Sydney_Thunder, http://dbpedia.org/resource/Sydney_Storm, http://dbpedia.org/resource/Sydney_Royal_Easter_Show, http://dbpedia.org/resource/Quebec_Remparts, http://dbpedia.org/resource/Orlando_Predators, http://dbpedia.org/resource/Orlando_Magic, http://dbpedia.org/resource/Orlando_Solar_Bears_(ECHL), http://dbpedia.org/resource/Arsenal_F.C., http://dbpedia.org/resource/Northwest_Arkansas_Naturals, http://dbpedia.org/resource/2015_Rugby_World_Cup, http://dbpedia.org/resource/2012_Summer_Olympics, http://dbpedia.org/resource/2017_World_Championships_in_Athletics, http://dbpedia.org/resource/West_Ham_United_F.C., http://dbpedia.org/resource/2012_Summer_Paralympics, http://dbpedia.org/resource/Euroleague_Final_Four, http://dbpedia.org/resource/ATP_World_Tour_Finals, http://dbpedia.org/resource/Anschutz_Entertainment_Group, http://dbpedia.org/resource/Chicago_Bulls, http://dbpedia.org/resource/Chicago_Blackhawks, http://dbpedia.org/resource/CBS_Sports_Classic, http://dbpedia.org/resource/Toronto_Rock, http://dbpedia.org/resource/Toronto_Phantoms, http://dbpedia.org/resource/Toronto_Raptors, http://dbpedia.org/resource/Raptors_905, http://dbpedia.org/resource/Toronto_Maple_Leafs, http://dbpedia.org/resource/Toronto_Marlies, http://dbpedia.org/resource/Manchester_City_F.C., http://dbpedia.org/resource/Summer_Olympic_Games, http://dbpedia.org/resource/The_Championships,_Wimbledon, http://dbpedia.org/resource/UEFA_Euro_2004, http://dbpedia.org/resource/UEFA_Euro_2004_Final, http://dbpedia.org/resource/2014_UEFA_Champions_League_Final, http://dbpedia.org/resource/S.L._Benfica, http://dbpedia.org/resource/Portugal_national_football_team, http://dbpedia.org/resource/S.L._Benfica_B, http://dbpedia.org/resource/Western_Athletic_Conference_Baseball_Tournament, http://dbpedia.org/resource/Mesa_Miners, http://dbpedia.org/resource/Oakland_Athletics, http://dbpedia.org/resource/Arizona_State_Sun_Devils_baseball, http://dbpedia.org/resource/Tulsa_Drillers, http://dbpedia.org/resource/Tulsa_Roughnecks_FC, http://dbpedia.org/resource/2010_FIFA_World_Cup, http://dbpedia.org/resource/1999_All-Africa_Games, http://dbpedia.org/resource/Kaizer_Chiefs_F.C., http://dbpedia.org/resource/South_Africa_national_rugby_union_team, http://dbpedia.org/resource/South_Africa_national_football_team, http://dbpedia.org/resource/Syracuse_Chiefs, http://dbpedia.org/resource/Syracuse_Salty_Dogs, http://dbpedia.org/resource/South_Carolina_Gamecocks_baseball, http://dbpedia.org/resource/Harrisburg_Senators, http://dbpedia.org/resource/Harrisburg_City_Islanders, http://dbpedia.org/resource/Hershey_Bears, http://dbpedia.org/resource/Harrisburg_Stampede, http://dbpedia.org/resource/Norfolk_Tides, http://dbpedia.org/resource/Wisconsin_Badgers_men's_basketball, http://dbpedia.org/resource/Wisconsin_Badgers_men's_ice_hockey, http://dbpedia.org/resource/Columbia_Fireflies, http://dbpedia.org/resource/Chinese_Taipei_Ice_Hockey_League, http://dbpedia.org/resource/Indianapolis_Indians, http://dbpedia.org/resource/Atlantic_Coast_Conference_Baseball_Tournament, http://dbpedia.org/resource/1975_Atlantic_Coast_Conference_Baseball_Tournament, http://dbpedia.org/resource/Duke_Blue_Devils, http://dbpedia.org/resource/1998_Atlantic_Coast_Conference_Baseball_Tournament, http://dbpedia.org/resource/1999_Atlantic_Coast_Conference_Baseball_Tournament, http://dbpedia.org/resource/2009_Atlantic_Coast_Conference_Baseball_Tournament, http://dbpedia.org/resource/2011_Atlantic_Coast_Conference_Baseball_Tournament, http://dbpedia.org/resource/2013_Atlantic_Coast_Conference_Baseball_Tournament, http://dbpedia.org/resource/International_League, http://dbpedia.org/resource/Triple-A_All-Star_Game, http://dbpedia.org/resource/Durham_Bulls, http://dbpedia.org/resource/North_Carolina_Central_University, http://dbpedia.org/resource/Presidential_Cup_Bowl, http://dbpedia.org/resource/Maryland_Terrapins, http://dbpedia.org/resource/Philadelphia/Baltimore_Stars, http://dbpedia.org/resource/West_Virginia_Mountaineers, http://dbpedia.org/resource/South_Florida_Bulls_baseball, http://dbpedia.org/resource/UCF_Knights, http://dbpedia.org/resource/Orlando_Fantasy, http://dbpedia.org/resource/Southern_United_FC, http://dbpedia.org/resource/Crusaders_(rugby_union), http://dbpedia.org/resource/South_Island_rugby_league_team, http://dbpedia.org/resource/Canterbury_rugby_league_team, http://dbpedia.org/resource/C.D._Guadalajara, http://dbpedia.org/resource/2019_Special_Olympics_World_Summer_Games, http://dbpedia.org/resource/2011_Pan_American_Games, http://dbpedia.org/resource/Atlanta_Braves, http://dbpedia.org/resource/Arizona_Cardinals, http://dbpedia.org/resource/Fiesta_Bowl, http://dbpedia.org/resource/England_national_football_team, http://dbpedia.org/resource/NFL_International_Series, http://dbpedia.org/resource/Republic_of_Ireland_national_football_team, http://dbpedia.org/resource/Leinster_Rugby, http://dbpedia.org/resource/Ireland_national_rugby_union_team, http://dbpedia.org/resource/Dayton_Wings_(basketball_team), http://dbpedia.org/resource/Dayton_Bombers, http://dbpedia.org/resource/Wright_State_Raiders, http://dbpedia.org/resource/Dayton_Bulldogs, http://dbpedia.org/resource/Dayton_Warbirds, http://dbpedia.org/resource/Mississippi_Braves, http://dbpedia.org/resource/Mississippi_State_Bulldogs_baseball, http://dbpedia.org/resource/Conference_USA_Baseball_Tournament, http://dbpedia.org/resource/Tacoma_Rainiers, http://dbpedia.org/resource/Fresno_Grizzlies, http://dbpedia.org/resource/Fresno_Fuego, http://dbpedia.org/resource/El_Paso_Chihuahuas, http://dbpedia.org/resource/Tottenham_Hotspur_F.C., http://dbpedia.org/resource/River_City_Rascals, http://dbpedia.org/resource/Troy_Trojans_women's_volleyball, http://dbpedia.org/resource/Troy_Trojans, http://dbpedia.org/resource/Buffalo_Bisons, http://dbpedia.org/resource/Colorado_Rockies, http://dbpedia.org/resource/Hong_Kong_Pegasus_FC, http://dbpedia.org/resource/Hong_Kong_Sevens, http://dbpedia.org/resource/Kitchee_SC, http://dbpedia.org/resource/South_China_AA, http://dbpedia.org/resource/1956_AFC_Asian_Cup, http://dbpedia.org/resource/Hong_Kong_national_football_team, http://dbpedia.org/resource/Alba_Berlin, http://dbpedia.org/resource/Eisbären_Berlin, http://dbpedia.org/resource/EuroBasket_2015, http://dbpedia.org/resource/2001_FA_Cup_Final, http://dbpedia.org/resource/2006_FA_Cup_Final, http://dbpedia.org/resource/2004_FA_Cup_Final, http://dbpedia.org/resource/2006_Football_League_Cup_Final, http://dbpedia.org/resource/1999_Rugby_World_Cup_Final, http://dbpedia.org/resource/2002_Football_League_Cup_Final, http://dbpedia.org/resource/2004_Football_League_Cup_Final, http://dbpedia.org/resource/Football_Association_of_Wales, http://dbpedia.org/resource/2005_Football_League_Cup_Final, http://dbpedia.org/resource/2001_Football_League_Cup_Final, http://dbpedia.org/resource/2003_Football_League_Cup_Final, http://dbpedia.org/resource/2006_Heineken_Cup_Final, http://dbpedia.org/resource/2002_FA_Cup_Final, http://dbpedia.org/resource/2003_FA_Cup_Final, http://dbpedia.org/resource/2005_FA_Cup_Final, http://dbpedia.org/resource/2007_Football_League_Cup_Final, http://dbpedia.org/resource/Challenge_Cup, http://dbpedia.org/resource/Welsh_Rugby_Union, http://dbpedia.org/resource/2014_Heineken_Cup_Final, http://dbpedia.org/resource/2011_Heineken_Cup_Final, http://dbpedia.org/resource/2008_Heineken_Cup_Final, http://dbpedia.org/resource/2002_Heineken_Cup_Final, http://dbpedia.org/resource/Houston_Astros, http://dbpedia.org/resource/Georgia_Tech_Yellow_Jackets_men's_basketball, http://dbpedia.org/resource/Atlanta_Dream, http://dbpedia.org/resource/Atlanta_Hawks, http://dbpedia.org/resource/Atlanta_Thrashers, http://dbpedia.org/resource/Georgia_Force, http://dbpedia.org/resource/New_South_Wales_cricket_team, http://dbpedia.org/resource/Sydney_Swans, http://dbpedia.org/resource/New_South_Wales_rugby_league_team, http://dbpedia.org/resource/St._George_Illawarra_Dragons, http://dbpedia.org/resource/New_South_Wales_Waratahs, http://dbpedia.org/resource/South_Sydney_Rabbitohs, http://dbpedia.org/resource/Wests_Tigers, http://dbpedia.org/resource/New_York_Yankees, http://dbpedia.org/resource/Pinstripe_Bowl, http://dbpedia.org/resource/New_York_City_FC, http://dbpedia.org/resource/Houston_Dynamo, http://dbpedia.org/resource/Houston_Dash, http://dbpedia.org/resource/Texas_Southern_Tigers_football, http://dbpedia.org/resource/Los_Angeles_Kiss, http://dbpedia.org/resource/Anaheim_Bullfrogs, http://dbpedia.org/resource/Anaheim_Piranhas, http://dbpedia.org/resource/Anaheim_Storm, http://dbpedia.org/resource/Los_Angeles_Clippers, http://dbpedia.org/resource/Anaheim_Ducks, http://dbpedia.org/resource/Anaheim_Splash, http://dbpedia.org/resource/UCLA_Bruins, http://dbpedia.org/resource/2014_Asian_Para_Games, http://dbpedia.org/resource/2014_Asian_Games, http://dbpedia.org/resource/Washington_Redskins, http://dbpedia.org/resource/National_Collegiate_Athletic_Association, http://dbpedia.org/resource/National_Football_League, http://dbpedia.org/resource/Tampa_Bay_Buccaneers, http://dbpedia.org/resource/Major_League_Soccer, http://dbpedia.org/resource/Tampa_Bay_Mutiny, http://dbpedia.org/resource/Outback_Bowl, http://dbpedia.org/resource/South_Florida_Bulls_football, http://dbpedia.org/resource/2011_Asian_Winter_Games, http://dbpedia.org/resource/Kazakhstan_national_football_team, http://dbpedia.org/resource/FC_Astana, http://dbpedia.org/resource/FC_Bayterek, http://dbpedia.org/resource/Lehigh_Valley_IronPigs, http://dbpedia.org/resource/D.C._United, http://dbpedia.org/resource/Orlando_City_SC, http://dbpedia.org/resource/NCAA_Division_I_Women's_Soccer_Championship, http://dbpedia.org/resource/Orlando_Pride, http://dbpedia.org/resource/Tampa_Bay_Rays, http://dbpedia.org/resource/Evansville_IceMen, http://dbpedia.org/resource/Evansville_Purple_Aces, http://dbpedia.org/resource/Evansville_SPHL_team, http://dbpedia.org/resource/Georgia_Tech_Yellow_Jackets, http://dbpedia.org/resource/Peach_Bowl, http://dbpedia.org/resource/Atlanta_Chiefs, http://dbpedia.org/resource/Atlanta_Beat_(WUSA), http://dbpedia.org/resource/Olympique_Lyonnais, http://dbpedia.org/resource/UEFA_Euro_2016, http://dbpedia.org/resource/2019_FIFA_Women's_World_Cup, http://dbpedia.org/resource/Southend_United_F.C., http://dbpedia.org/resource/Texas_Copperheads, http://dbpedia.org/resource/Sacramento_Mountain_Lions, http://dbpedia.org/resource/San_Francisco_Demons, http://dbpedia.org/resource/Foster_Farms_Bowl, http://dbpedia.org/resource/San_Francisco_Giants, http://dbpedia.org/resource/California_Golden_Bears_football, http://dbpedia.org/resource/Carolina_Panthers, http://dbpedia.org/resource/ACC_Championship_Game, http://dbpedia.org/resource/Belk_Bowl, http://dbpedia.org/resource/St._Louis_Cardinals, http://dbpedia.org/resource/New_York_Mets, http://dbpedia.org/resource/Philadelphia_Phillies, http://dbpedia.org/resource/Boston_Red_Sox, http://dbpedia.org/resource/Gulf_Coast_League_Red_Sox, http://dbpedia.org/resource/Detroit_Tigers, http://dbpedia.org/resource/AIK_Fotboll, http://dbpedia.org/resource/Sweden_national_football_team, http://dbpedia.org/resource/Cincinnati_Reds, http://dbpedia.org/resource/Miami_Beach_Bowl, http://dbpedia.org/resource/Washington_Nationals, http://dbpedia.org/resource/Baltimore_Orioles, http://dbpedia.org/resource/Pittsburgh_Pirates, http://dbpedia.org/resource/San_Diego_Padres, http://dbpedia.org/resource/USA_Sevens, http://dbpedia.org/resource/San_Diego_Comic-Con_International, http://dbpedia.org/resource/Cleveland_Indians, http://dbpedia.org/resource/Minnesota_Twins, http://dbpedia.org/resource/Minnesota_Golden_Gophers_baseball, http://dbpedia.org/resource/Fort_Wayne_TinCaps, http://dbpedia.org/resource/Tampa_Bay_Storm, http://dbpedia.org/resource/St._Petersburg_Bowl, http://dbpedia.org/resource/Tampa_Bay_Lightning, http://dbpedia.org/resource/Virginia_Destroyers, http://dbpedia.org/resource/Sporting_Kansas_City, http://dbpedia.org/resource/NCAA_Division_II_Football_Championship, http://dbpedia.org/resource/FC_Kansas_City, http://dbpedia.org/resource/NCAA_Division_I_Men's_Soccer_Championship, http://dbpedia.org/resource/San_Antonio_Talons, http://dbpedia.org/resource/U.S._Army_All-American_Bowl, http://dbpedia.org/resource/San_Antonio_Spurs, http://dbpedia.org/resource/San_Antonio_Texans, http://dbpedia.org/resource/Alamo_Bowl, http://dbpedia.org/resource/UTSA_Roadrunners_football, http://dbpedia.org/resource/Pittsburgh_Power, http://dbpedia.org/resource/Pittsburgh_Penguins, http://dbpedia.org/resource/Nashville_Sounds, http://dbpedia.org/resource/Arizona_Sting, http://dbpedia.org/resource/Arizona_Coyotes, http://dbpedia.org/resource/Jiangsu_Suning_F.C., http://dbpedia.org/resource/New_York_Titans_(lacrosse), http://dbpedia.org/resource/New_York_Liberty, http://dbpedia.org/resource/Seton_Hall_Pirates_men's_basketball, http://dbpedia.org/resource/Brooklyn_Nets, http://dbpedia.org/resource/New_Jersey_Devils, http://dbpedia.org/resource/NJIT_Highlanders, http://dbpedia.org/resource/New_Jersey_Ironmen, http://dbpedia.org/resource/New_Jersey_Rockets_(ice_hockey), http://dbpedia.org/resource/Houston_Comets, http://dbpedia.org/resource/Houston_Rockets, http://dbpedia.org/resource/Houston_Aeros_(1994–2013), http://dbpedia.org/resource/Creighton_Bluejays_baseball, http://dbpedia.org/resource/Omaha_Mammoths, http://dbpedia.org/resource/College_World_Series, http://dbpedia.org/resource/Omaha_Nighthawks, http://dbpedia.org/resource/Red_Sky_Music_Festival, http://dbpedia.org/resource/VMI_Keydets_football, http://dbpedia.org/resource/Northern_Illinois_Huskies, http://dbpedia.org/resource/Illinois_High_School_Association, http://dbpedia.org/resource/UNC_Asheville_Bulldogs, http://dbpedia.org/resource/New_Mexico_State_Aggies, http://dbpedia.org/resource/Buffalo_Bulls_football, http://dbpedia.org/resource/Buffalo_Bulls, http://dbpedia.org/resource/Midland_RockHounds, http://dbpedia.org/resource/Milton_Keynes_Dons_F.C., http://dbpedia.org/resource/2014_FIFA_World_Cup, http://dbpedia.org/resource/América_Futebol_Clube_(RN), http://dbpedia.org/resource/New_York_Cosmos_(2010), http://dbpedia.org/resource/Seattle_Reign_FC, http://dbpedia.org/resource/FK_Pacific, http://dbpedia.org/resource/Hibernian_Saints, http://dbpedia.org/resource/Seattle_Sounders_FC_2, http://dbpedia.org/resource/Seattle_Sounders_(1994–2008), http://dbpedia.org/resource/Seattle_Sounders_Women, http://dbpedia.org/resource/Colorado_Gold_Kings, http://dbpedia.org/resource/Colorado_College_Tigers_men's_ice_hockey, http://dbpedia.org/resource/Penn_State_Nittany_Lions_football, http://dbpedia.org/resource/Brisbane_International, http://dbpedia.org/resource/Fed_Cup, http://dbpedia.org/resource/Tennis, http://dbpedia.org/resource/WTA_International_tournaments, http://dbpedia.org/resource/San_Antonio_Missions, http://dbpedia.org/resource/New_Orleans_Storm, http://dbpedia.org/resource/New_Orleans_Zephyrs, http://dbpedia.org/resource/Tulane_Green_Wave_baseball, http://dbpedia.org/resource/Sun_Belt_Conference_Baseball_Tournament, http://dbpedia.org/resource/Hurricanes_(rugby_union), http://dbpedia.org/resource/Wellington_Phoenix_FC, http://dbpedia.org/resource/Wellington_Rugby_Football_Union, http://dbpedia.org/resource/St_Kilda_Football_Club, http://dbpedia.org/resource/Wellington_cricket_team, http://dbpedia.org/resource/San_Diego_Toreros_baseball, http://dbpedia.org/resource/Arizona_League_Padres, http://dbpedia.org/resource/Seattle_Mariners, http://dbpedia.org/resource/Arizona_League_Mariners, http://dbpedia.org/resource/Arizona_United_SC, http://dbpedia.org/resource/Peoria_Javelinas, http://dbpedia.org/resource/Surprise_Saguaros, http://dbpedia.org/resource/Tulsa_Golden_Hurricane, http://dbpedia.org/resource/Salt_Lake_Bees, http://dbpedia.org/resource/Utah_Utes, http://dbpedia.org/resource/C.F._Monterrey, http://dbpedia.org/resource/Minnesota_United_FC_(MLS), http://dbpedia.org/resource/Memphis_Redbirds, http://dbpedia.org/resource/Civil_Rights_Game, http://dbpedia.org/resource/Gfinity_3, http://dbpedia.org/resource/London_Lions_(basketball), http://dbpedia.org/resource/London_GD_Handball_Club, http://dbpedia.org/resource/Great_Lakes_Loons, http://dbpedia.org/resource/UMass_Lowell_River_Hawks_baseball, http://dbpedia.org/resource/Lowell_Spinners, http://dbpedia.org/resource/Kane_County_Cougars, http://dbpedia.org/resource/South_Bend_Cubs, http://dbpedia.org/resource/Staten_Island_Yankees, http://dbpedia.org/resource/New_York_Storm, http://dbpedia.org/resource/Montreal_Expos, http://dbpedia.org/resource/Gulf_Coast_League_Marlins, http://dbpedia.org/resource/Jupiter_Hammerheads, http://dbpedia.org/resource/Gulf_Coast_League_Cardinals, http://dbpedia.org/resource/Palm_Beach_Cardinals, http://dbpedia.org/resource/Cal_State_San_Bernardino_Coyotes, http://dbpedia.org/resource/Inland_Empire_66ers, http://dbpedia.org/resource/Connecticut_Tigers, http://dbpedia.org/resource/Northeast_Conference_Baseball_Tournament, http://dbpedia.org/resource/Connecticut_Defenders, http://dbpedia.org/resource/Atlantic_10_Conference_Baseball_Tournament, http://dbpedia.org/resource/Texas_Rangers_(baseball), http://dbpedia.org/resource/Surprise_Fightin'_Falcons, http://dbpedia.org/resource/Kansas_City_Royals, http://dbpedia.org/resource/Salt_River_Rafters, http://dbpedia.org/resource/2018_FIFA_World_Cup, http://dbpedia.org/resource/FC_Rostov] [INFO] Predicted Answer: [http://dbpedia.org/resource/Game_design, http://dbpedia.org/resource/Peter_Molyneux__1] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 902, P: 0.502, R: 0.559, macro F1: 0.511, macro F1*: 0.529 -[INFO] Current time: 1584 ms, average: 2904.49 ms/ques, entity detection: 502.01 ms/ques (linking tool: 408.76 ms/ques), relation detection: 937.97 ms/ques (similarity: 708.57 ms/ques), EDG generation: 46.92 ms/ques +[INFO] Cumulative metrics, sample: 902, P: 0.502, R: 0.559, macro F1: 0.529 +[INFO] Current question consumed time: 1341 ms, average: 2599.37 ms/ques, entity detection: 439.36 ms/ques (linking tool: 361.76 ms/ques), relation detection: 862.06 ms/ques (similarity: 702.05 ms/ques), EDG generation: 47.40 ms/ques [INFO] Entity Linking P: 0.738359201773836, R: 0.6906873614190687, micro F1: 0.7060236511456025, macro F1: 0.7137281345307986 [INFO] Relation Linking P: 0.5974685883222468, R: 0.5437915742793792, micro F1: 0.5593495934959359, macro F1: 0.5693677828648499 [INFO] Type Linking P: 0.5587583148558758, R: 0.5587583148558758, micro F1: 0.5587583148558758, macro F1: 0.5587583148558758 @@ -12659,8 +12659,8 @@ Question 902: Name the route end of the North-South Transport Corridor and Mumba [INFO] Golden Answer: [http://dbpedia.org/resource/Mumbai] [INFO] Predicted Answer: [http://dbpedia.org/resource/Mumbai] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 903, P: 0.503, R: 0.559, macro F1: 0.512, macro F1*: 0.530 -[INFO] Current time: 2395 ms, average: 2903.93 ms/ques, entity detection: 502.13 ms/ques (linking tool: 408.84 ms/ques), relation detection: 937.60 ms/ques (similarity: 708.18 ms/ques), EDG generation: 46.91 ms/ques +[INFO] Cumulative metrics, sample: 903, P: 0.503, R: 0.559, macro F1: 0.530 +[INFO] Current question consumed time: 1933 ms, average: 2598.63 ms/ques, entity detection: 439.39 ms/ques (linking tool: 361.76 ms/ques), relation detection: 861.70 ms/ques (similarity: 701.67 ms/ques), EDG generation: 47.39 ms/ques [INFO] Entity Linking P: 0.7386489479512736, R: 0.6910299003322259, micro F1: 0.7063492063492065, macro F1: 0.7140463881047168 [INFO] Relation Linking P: 0.5979143595422665, R: 0.544296788482835, micro F1: 0.5598375784422306, macro F1: 0.5698471184584775 [INFO] Type Linking P: 0.5592469545957918, R: 0.5592469545957918, micro F1: 0.5592469545957918, macro F1: 0.5592469545957918 @@ -12673,8 +12673,8 @@ Question 903: Is Albert P Clark buried in Colorado? [INFO] Golden Answer: [true] [INFO] Predicted Answer: [true] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 904, P: 0.503, R: 0.560, macro F1: 0.512, macro F1*: 0.530 -[INFO] Current time: 6665 ms, average: 2908.09 ms/ques, entity detection: 502.62 ms/ques (linking tool: 409.25 ms/ques), relation detection: 941.84 ms/ques (similarity: 710.25 ms/ques), EDG generation: 46.86 ms/ques +[INFO] Cumulative metrics, sample: 904, P: 0.503, R: 0.560, macro F1: 0.530 +[INFO] Current question consumed time: 5887 ms, average: 2602.27 ms/ques, entity detection: 439.79 ms/ques (linking tool: 362.11 ms/ques), relation detection: 865.46 ms/ques (similarity: 703.82 ms/ques), EDG generation: 47.35 ms/ques [INFO] Entity Linking P: 0.7389380530973452, R: 0.6913716814159292, micro F1: 0.7066740412979353, macro F1: 0.7143639337754689 [INFO] Relation Linking P: 0.5983591445427728, R: 0.5448008849557522, micro F1: 0.5603244837758122, macro F1: 0.5703253841218914 [INFO] Type Linking P: 0.5597345132743363, R: 0.5597345132743363, micro F1: 0.5597345132743363, macro F1: 0.5597345132743363 @@ -12687,8 +12687,8 @@ Question 904: Which hockey team is head coached by Bill Peters? [INFO] Golden Answer: [http://dbpedia.org/resource/Carolina_Hurricanes] [INFO] Predicted Answer: [http://dbpedia.org/resource/Carolina_Hurricanes] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 905, P: 0.504, R: 0.560, macro F1: 0.513, macro F1*: 0.531 -[INFO] Current time: 1096 ms, average: 2906.08 ms/ques, entity detection: 502.30 ms/ques (linking tool: 408.92 ms/ques), relation detection: 941.12 ms/ques (similarity: 709.78 ms/ques), EDG generation: 46.83 ms/ques +[INFO] Cumulative metrics, sample: 905, P: 0.504, R: 0.560, macro F1: 0.531 +[INFO] Current question consumed time: 917 ms, average: 2600.41 ms/ques, entity detection: 439.49 ms/ques (linking tool: 361.81 ms/ques), relation detection: 864.79 ms/ques (similarity: 703.35 ms/ques), EDG generation: 47.31 ms/ques [INFO] Entity Linking P: 0.7392265193370166, R: 0.6917127071823205, micro F1: 0.706998158379374, macro F1: 0.7146807739072933 [INFO] Relation Linking P: 0.5988029465930018, R: 0.5453038674033149, micro F1: 0.5608103130755074, macro F1: 0.5708025834565403 [INFO] Type Linking P: 0.5591160220994476, R: 0.5591160220994476, micro F1: 0.5591160220994476, macro F1: 0.5591160220994476 @@ -12701,8 +12701,8 @@ Question 905: Count the number of religions followed by the alumini of Ananda Co [INFO] Golden Answer: [4] [INFO] Predicted Answer: [4, 6, 5] [INFO] P: 0.3333333333333333, R: 1.0, F: 0.5 -[INFO] Cumulative metrics, sample: 906, P: 0.504, R: 0.561, macro F1: 0.513, macro F1*: 0.531 -[INFO] Current time: 2596 ms, average: 2905.74 ms/ques, entity detection: 501.97 ms/ques (linking tool: 408.58 ms/ques), relation detection: 941.12 ms/ques (similarity: 709.56 ms/ques), EDG generation: 46.88 ms/ques +[INFO] Cumulative metrics, sample: 906, P: 0.504, R: 0.561, macro F1: 0.531 +[INFO] Current question consumed time: 2160 ms, average: 2599.92 ms/ques, entity detection: 439.19 ms/ques (linking tool: 361.51 ms/ques), relation detection: 864.75 ms/ques (similarity: 703.13 ms/ques), EDG generation: 47.38 ms/ques [INFO] Entity Linking P: 0.739514348785872, R: 0.6920529801324503, micro F1: 0.7073215599705668, macro F1: 0.7149969108538808 [INFO] Relation Linking P: 0.5992457689477556, R: 0.5458057395143487, micro F1: 0.5612950699043424, macro F1: 0.5712787200475511 [INFO] Type Linking P: 0.5596026490066225, R: 0.5596026490066225, micro F1: 0.5596026490066225, macro F1: 0.5596026490066225 @@ -12715,8 +12715,8 @@ Question 906: Which team did Wilfred McDonalds and Cam Plante play for? [INFO] Golden Answer: [http://dbpedia.org/resource/Toronto_Maple_Leafs] [INFO] Predicted Answer: [http://dbpedia.org/resource/Toronto_Maple_Leafs] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 907, P: 0.504, R: 0.561, macro F1: 0.513, macro F1*: 0.531 -[INFO] Current time: 3243 ms, average: 2906.11 ms/ques, entity detection: 501.76 ms/ques (linking tool: 408.31 ms/ques), relation detection: 941.25 ms/ques (similarity: 710.03 ms/ques), EDG generation: 46.86 ms/ques +[INFO] Cumulative metrics, sample: 907, P: 0.504, R: 0.561, macro F1: 0.531 +[INFO] Current question consumed time: 2710 ms, average: 2600.04 ms/ques, entity detection: 438.97 ms/ques (linking tool: 361.25 ms/ques), relation detection: 864.81 ms/ques (similarity: 703.60 ms/ques), EDG generation: 47.36 ms/ques [INFO] Entity Linking P: 0.7398015435501654, R: 0.6923925027563396, micro F1: 0.7076442484380744, macro F1: 0.7153123469584356 [INFO] Relation Linking P: 0.5996876148474825, R: 0.5463065049614112, micro F1: 0.5617787578096298, macro F1: 0.5717537974638228 [INFO] Type Linking P: 0.5600882028665931, R: 0.5600882028665931, micro F1: 0.5600882028665931, macro F1: 0.5600882028665931 @@ -12729,8 +12729,8 @@ Question 907: What are the television shows whose network's stockholder is Warne [INFO] Golden Answer: [http://dbpedia.org/resource/Kung_Fu:_The_Legend_Continues, http://dbpedia.org/resource/Island_City_(film), http://dbpedia.org/resource/Babylon_5, http://dbpedia.org/resource/Time_Trax, http://dbpedia.org/resource/Babylon_5:_The_Gathering, http://dbpedia.org/resource/Pointman] [INFO] Predicted Answer: [http://dbpedia.org/resource/Kung_Fu:_The_Legend_Continues, http://dbpedia.org/resource/Island_City_(film), http://dbpedia.org/resource/Babylon_5, http://dbpedia.org/resource/Time_Trax, http://dbpedia.org/resource/Babylon_5:_The_Gathering, http://dbpedia.org/resource/Pointman] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 908, P: 0.505, R: 0.562, macro F1: 0.514, macro F1*: 0.532 -[INFO] Current time: 3598 ms, average: 2906.88 ms/ques, entity detection: 502.07 ms/ques (linking tool: 408.65 ms/ques), relation detection: 941.40 ms/ques (similarity: 710.25 ms/ques), EDG generation: 46.83 ms/ques +[INFO] Cumulative metrics, sample: 908, P: 0.505, R: 0.562, macro F1: 0.532 +[INFO] Current question consumed time: 3015 ms, average: 2600.50 ms/ques, entity detection: 439.15 ms/ques (linking tool: 361.47 ms/ques), relation detection: 864.90 ms/ques (similarity: 703.82 ms/ques), EDG generation: 47.33 ms/ques [INFO] Entity Linking P: 0.7400881057268722, R: 0.6927312775330396, micro F1: 0.7079662261380325, macro F1: 0.7156270845537319 [INFO] Relation Linking P: 0.6001284875183553, R: 0.5468061674008811, micro F1: 0.5622613803230553, macro F1: 0.5722278192581197 [INFO] Type Linking P: 0.5594713656387665, R: 0.5594713656387665, micro F1: 0.5594713656387665, macro F1: 0.5594713656387665 @@ -12743,8 +12743,8 @@ Question 908: Name the Fox TV show judged by Rand Jackson ? [INFO] Golden Answer: [http://dbpedia.org/resource/American_Idol] [INFO] Predicted Answer: [http://dbpedia.org/resource/Derrick_O'Connor, http://dbpedia.org/resource/Cindy_O'Callaghan, http://dbpedia.org/resource/Ray_Winstone, http://dbpedia.org/resource/Larry_Lamb, http://dbpedia.org/resource/Bernard_Hill] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 909, P: 0.504, R: 0.561, macro F1: 0.513, macro F1*: 0.531 -[INFO] Current time: 3861 ms, average: 2907.93 ms/ques, entity detection: 502.07 ms/ques (linking tool: 408.61 ms/ques), relation detection: 942.03 ms/ques (similarity: 710.37 ms/ques), EDG generation: 46.79 ms/ques +[INFO] Cumulative metrics, sample: 909, P: 0.504, R: 0.561, macro F1: 0.531 +[INFO] Current question consumed time: 3077 ms, average: 2601.02 ms/ques, entity detection: 439.10 ms/ques (linking tool: 361.40 ms/ques), relation detection: 865.44 ms/ques (similarity: 703.95 ms/ques), EDG generation: 47.29 ms/ques [INFO] Entity Linking P: 0.7392739273927392, R: 0.6919691969196919, micro F1: 0.7071873854052073, macro F1: 0.7148398160338708 [INFO] Relation Linking P: 0.5994682801613495, R: 0.5462046204620462, micro F1: 0.5616428309497626, macro F1: 0.5715983057055806 [INFO] Type Linking P: 0.5588558855885588, R: 0.5588558855885588, micro F1: 0.5588558855885588, macro F1: 0.5588558855885588 @@ -12757,8 +12757,8 @@ Question 909: What is the total number of software whose programming language is [INFO] Golden Answer: [441] [INFO] Predicted Answer: [] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 910, P: 0.504, R: 0.561, macro F1: 0.513, macro F1*: 0.531 -[INFO] Current time: 2524 ms, average: 2907.50 ms/ques, entity detection: 501.51 ms/ques (linking tool: 408.16 ms/ques), relation detection: 941.46 ms/ques (similarity: 709.88 ms/ques), EDG generation: 46.84 ms/ques +[INFO] Cumulative metrics, sample: 910, P: 0.504, R: 0.561, macro F1: 0.531 +[INFO] Current question consumed time: 2182 ms, average: 2600.56 ms/ques, entity detection: 438.62 ms/ques (linking tool: 361.00 ms/ques), relation detection: 864.89 ms/ques (similarity: 703.46 ms/ques), EDG generation: 47.36 ms/ques [INFO] Entity Linking P: 0.7384615384615385, R: 0.6912087912087912, micro F1: 0.7064102564102566, macro F1: 0.714054277774493 [INFO] Relation Linking P: 0.5988095238095238, R: 0.5456043956043956, micro F1: 0.561025641025642, macro F1: 0.5709701756993106 [INFO] Type Linking P: 0.5593406593406594, R: 0.5593406593406594, micro F1: 0.5593406593406594, macro F1: 0.5593406593406594 @@ -12771,8 +12771,8 @@ Question 910: Name the city of Phil-Ellena ? [INFO] Golden Answer: [http://dbpedia.org/resource/Philadelphia] [INFO] Predicted Answer: [http://dbpedia.org/resource/Philadelphia] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 911, P: 0.504, R: 0.561, macro F1: 0.513, macro F1*: 0.531 -[INFO] Current time: 745 ms, average: 2905.13 ms/ques, entity detection: 501.16 ms/ques (linking tool: 407.85 ms/ques), relation detection: 940.60 ms/ques (similarity: 709.20 ms/ques), EDG generation: 46.80 ms/ques +[INFO] Cumulative metrics, sample: 911, P: 0.504, R: 0.561, macro F1: 0.531 +[INFO] Current question consumed time: 596 ms, average: 2598.36 ms/ques, entity detection: 438.28 ms/ques (linking tool: 360.70 ms/ques), relation detection: 864.09 ms/ques (similarity: 702.79 ms/ques), EDG generation: 47.32 ms/ques [INFO] Entity Linking P: 0.7387486278814489, R: 0.6915477497255763, micro F1: 0.7067325283571169, macro F1: 0.7143693562015546 [INFO] Relation Linking P: 0.5992499085254299, R: 0.5461031833150384, micro F1: 0.5615075009147467, macro F1: 0.5714434875643825 [INFO] Type Linking P: 0.5598243688254665, R: 0.5598243688254665, micro F1: 0.5598243688254665, macro F1: 0.5598243688254665 @@ -12785,8 +12785,8 @@ Question 911: how many companies are located in a spanish speaking region? [INFO] Golden Answer: [4783] [INFO] Predicted Answer: [1] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 912, P: 0.504, R: 0.560, macro F1: 0.513, macro F1*: 0.530 -[INFO] Current time: 1249 ms, average: 2903.31 ms/ques, entity detection: 500.71 ms/ques (linking tool: 407.45 ms/ques), relation detection: 939.93 ms/ques (similarity: 708.79 ms/ques), EDG generation: 46.77 ms/ques +[INFO] Cumulative metrics, sample: 912, P: 0.504, R: 0.560, macro F1: 0.530 +[INFO] Current question consumed time: 1017 ms, average: 2596.63 ms/ques, entity detection: 437.87 ms/ques (linking tool: 360.34 ms/ques), relation detection: 863.47 ms/ques (similarity: 702.39 ms/ques), EDG generation: 47.29 ms/ques [INFO] Entity Linking P: 0.7379385964912281, R: 0.6907894736842105, micro F1: 0.7059576023391815, macro F1: 0.7135860564688774 [INFO] Relation Linking P: 0.5985928362573099, R: 0.5455043859649122, micro F1: 0.560891812865498, macro F1: 0.5708169047929302 [INFO] Type Linking P: 0.5592105263157895, R: 0.5592105263157895, micro F1: 0.5592105263157895, macro F1: 0.5592105263157895 @@ -12799,8 +12799,8 @@ Question 912: What is the combatant of the Northwest Indian War and also the eth [INFO] Golden Answer: [http://dbpedia.org/resource/Chickasaw] [INFO] Predicted Answer: [http://dbpedia.org/resource/Chickasaw] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 913, P: 0.504, R: 0.561, macro F1: 0.513, macro F1*: 0.531 -[INFO] Current time: 1910 ms, average: 2902.23 ms/ques, entity detection: 500.73 ms/ques (linking tool: 407.35 ms/ques), relation detection: 939.39 ms/ques (similarity: 708.29 ms/ques), EDG generation: 46.79 ms/ques +[INFO] Cumulative metrics, sample: 913, P: 0.504, R: 0.561, macro F1: 0.531 +[INFO] Current question consumed time: 1536 ms, average: 2595.47 ms/ques, entity detection: 437.83 ms/ques (linking tool: 360.23 ms/ques), relation detection: 862.94 ms/ques (similarity: 701.90 ms/ques), EDG generation: 47.32 ms/ques [INFO] Entity Linking P: 0.7382256297918949, R: 0.6911281489594743, micro F1: 0.7062796641109895, macro F1: 0.7139009538677176 [INFO] Relation Linking P: 0.5990324936108068, R: 0.5460021905805038, micro F1: 0.5613727637824033, macro F1: 0.5712893386655794 [INFO] Type Linking P: 0.5596933187294633, R: 0.5596933187294633, micro F1: 0.5596933187294633, macro F1: 0.5596933187294633 @@ -12813,8 +12813,8 @@ Question 913: Which airlines fly to Gimhae International Airport? [INFO] Golden Answer: [http://dbpedia.org/resource/Yeongnam_Air, http://dbpedia.org/resource/Korean_Air, http://dbpedia.org/resource/Asiana_Airlines, http://dbpedia.org/resource/Air_Busan] [INFO] Predicted Answer: [http://dbpedia.org/resource/Yeongnam_Air, http://dbpedia.org/resource/Korean_Air, http://dbpedia.org/resource/Asiana_Airlines, http://dbpedia.org/resource/Air_Busan] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 914, P: 0.505, R: 0.561, macro F1: 0.514, macro F1*: 0.532 -[INFO] Current time: 1816 ms, average: 2901.04 ms/ques, entity detection: 500.44 ms/ques (linking tool: 407.06 ms/ques), relation detection: 939.04 ms/ques (similarity: 708.23 ms/ques), EDG generation: 46.75 ms/ques +[INFO] Cumulative metrics, sample: 914, P: 0.505, R: 0.561, macro F1: 0.532 +[INFO] Current question consumed time: 1530 ms, average: 2594.30 ms/ques, entity detection: 437.55 ms/ques (linking tool: 359.96 ms/ques), relation detection: 862.60 ms/ques (similarity: 701.85 ms/ques), EDG generation: 47.27 ms/ques [INFO] Entity Linking P: 0.7385120350109409, R: 0.6914660831509847, micro F1: 0.7066010211524436, macro F1: 0.7142151585721723 [INFO] Relation Linking P: 0.599471188913202, R: 0.5464989059080962, micro F1: 0.5618526622903001, macro F1: 0.5717607297868936 [INFO] Type Linking P: 0.5590809628008753, R: 0.5590809628008753, micro F1: 0.5590809628008753, macro F1: 0.5590809628008753 @@ -12827,8 +12827,8 @@ Question 914: GetTV is owned by which organisation? [INFO] Golden Answer: [http://dbpedia.org/resource/Sony] [INFO] Predicted Answer: [http://dbpedia.org/resource/Sony_Pictures_Television] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 915, P: 0.504, R: 0.561, macro F1: 0.513, macro F1*: 0.531 -[INFO] Current time: 880 ms, average: 2898.83 ms/ques, entity detection: 499.99 ms/ques (linking tool: 406.69 ms/ques), relation detection: 938.45 ms/ques (similarity: 707.85 ms/ques), EDG generation: 46.71 ms/ques +[INFO] Cumulative metrics, sample: 915, P: 0.504, R: 0.561, macro F1: 0.531 +[INFO] Current question consumed time: 753 ms, average: 2592.29 ms/ques, entity detection: 437.14 ms/ques (linking tool: 359.62 ms/ques), relation detection: 862.04 ms/ques (similarity: 701.47 ms/ques), EDG generation: 47.23 ms/ques [INFO] Entity Linking P: 0.7387978142076502, R: 0.6918032786885245, micro F1: 0.706921675774135, macro F1: 0.7145286728700422 [INFO] Relation Linking P: 0.598816029143898, R: 0.5459016393442623, micro F1: 0.5612386156648461, macro F1: 0.5711358546723724 [INFO] Type Linking P: 0.5584699453551912, R: 0.5584699453551912, micro F1: 0.5584699453551912, macro F1: 0.5584699453551912 @@ -12841,8 +12841,8 @@ Question 915: Name the islands that belong to the archipelago whose largest city [INFO] Golden Answer: [http://dbpedia.org/resource/Manuae_(Society_Islands), http://dbpedia.org/resource/Motu_One_(Society_Islands), http://dbpedia.org/resource/Bora_Bora, http://dbpedia.org/resource/Leeward_Islands_(Society_Islands), http://dbpedia.org/resource/Maiao, http://dbpedia.org/resource/Maupiti, http://dbpedia.org/resource/Tetiaroa, http://dbpedia.org/resource/Windward_Islands_(Society_Islands), http://dbpedia.org/resource/Tahiti, http://dbpedia.org/resource/Huahine, http://dbpedia.org/resource/Raiatea, http://dbpedia.org/resource/Mo'orea, http://dbpedia.org/resource/Maupihaa, http://dbpedia.org/resource/Tūpai] [INFO] Predicted Answer: [http://dbpedia.org/resource/Mehetia, http://dbpedia.org/resource/Tetiaroa, http://dbpedia.org/resource/Tahiti, http://dbpedia.org/resource/Mo'orea, http://dbpedia.org/resource/Bora_Bora, http://dbpedia.org/resource/Huahine, http://dbpedia.org/resource/Raiatea] [INFO] P: 0.8571428571428571, R: 0.42857142857142855, F: 0.5714285714285714 -[INFO] Cumulative metrics, sample: 916, P: 0.504, R: 0.561, macro F1: 0.513, macro F1*: 0.531 -[INFO] Current time: 3698 ms, average: 2899.70 ms/ques, entity detection: 499.65 ms/ques (linking tool: 406.39 ms/ques), relation detection: 939.20 ms/ques (similarity: 709.17 ms/ques), EDG generation: 46.69 ms/ques +[INFO] Cumulative metrics, sample: 916, P: 0.504, R: 0.561, macro F1: 0.531 +[INFO] Current question consumed time: 3406 ms, average: 2593.18 ms/ques, entity detection: 436.83 ms/ques (linking tool: 359.35 ms/ques), relation detection: 862.76 ms/ques (similarity: 702.79 ms/ques), EDG generation: 47.21 ms/ques [INFO] Entity Linking P: 0.7390829694323144, R: 0.6921397379912664, micro F1: 0.707241630276565, macro F1: 0.7148414990390348 [INFO] Relation Linking P: 0.5987081513828238, R: 0.5458515283842795, micro F1: 0.5611717612809326, macro F1: 0.5710593606703669 [INFO] Type Linking P: 0.5578602620087336, R: 0.5578602620087336, micro F1: 0.5578602620087336, macro F1: 0.5578602620087336 @@ -12855,8 +12855,8 @@ Question 916: In which team did Dave Bing and Ron Reed started their basketball [INFO] Golden Answer: [http://dbpedia.org/resource/Detroit_Pistons] [INFO] Predicted Answer: [http://dbpedia.org/resource/Detroit_Pistons] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 917, P: 0.505, R: 0.561, macro F1: 0.514, macro F1*: 0.532 -[INFO] Current time: 2770 ms, average: 2899.56 ms/ques, entity detection: 499.55 ms/ques (linking tool: 406.20 ms/ques), relation detection: 939.14 ms/ques (similarity: 709.51 ms/ques), EDG generation: 46.73 ms/ques +[INFO] Cumulative metrics, sample: 917, P: 0.505, R: 0.561, macro F1: 0.532 +[INFO] Current question consumed time: 2491 ms, average: 2593.07 ms/ques, entity detection: 436.75 ms/ques (linking tool: 359.18 ms/ques), relation detection: 862.73 ms/ques (similarity: 703.14 ms/ques), EDG generation: 47.21 ms/ques [INFO] Entity Linking P: 0.7393675027262814, R: 0.6924754634678298, micro F1: 0.7075608869502001, macro F1: 0.7151536393468221 [INFO] Relation Linking P: 0.5980552526354053, R: 0.5452562704471101, micro F1: 0.56055979643766, macro F1: 0.5704366132759608 [INFO] Type Linking P: 0.5583424209378408, R: 0.5583424209378408, micro F1: 0.5583424209378408, macro F1: 0.5583424209378408 @@ -12869,8 +12869,8 @@ Question 917: List the settlements area whose province is Metropolitan City of V [INFO] Golden Answer: [http://dbpedia.org/resource/Venice, http://dbpedia.org/resource/Chioggia, http://dbpedia.org/resource/Campagna_Lupia, http://dbpedia.org/resource/Cavarzere, http://dbpedia.org/resource/Portogruaro, http://dbpedia.org/resource/Cona,_Veneto] [INFO] Predicted Answer: [http://dbpedia.org/resource/Venice, http://dbpedia.org/resource/Chioggia, http://dbpedia.org/resource/Campagna_Lupia, http://dbpedia.org/resource/Cavarzere, http://dbpedia.org/resource/Portogruaro, http://dbpedia.org/resource/Cona,_Veneto] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 918, P: 0.506, R: 0.562, macro F1: 0.514, macro F1*: 0.532 -[INFO] Current time: 1836 ms, average: 2898.40 ms/ques, entity detection: 499.38 ms/ques (linking tool: 406.07 ms/ques), relation detection: 938.57 ms/ques (similarity: 709.23 ms/ques), EDG generation: 46.70 ms/ques +[INFO] Cumulative metrics, sample: 918, P: 0.506, R: 0.562, macro F1: 0.532 +[INFO] Current question consumed time: 1610 ms, average: 2592.00 ms/ques, entity detection: 436.60 ms/ques (linking tool: 359.06 ms/ques), relation detection: 862.21 ms/ques (similarity: 702.87 ms/ques), EDG generation: 47.22 ms/ques [INFO] Entity Linking P: 0.7396514161220044, R: 0.6928104575163399, micro F1: 0.7078794480755267, macro F1: 0.7154650960510947 [INFO] Relation Linking P: 0.5984931009440813, R: 0.545751633986928, micro F1: 0.561038489469863, macro F1: 0.5709068659858495 [INFO] Type Linking P: 0.5577342047930284, R: 0.5577342047930284, micro F1: 0.5577342047930284, macro F1: 0.5577342047930284 @@ -12883,8 +12883,8 @@ Question 918: What was recorded by Live in Detroit, MI and also palce of birth o [INFO] Golden Answer: [http://dbpedia.org/resource/Michigan] [INFO] Predicted Answer: [http://dbpedia.org/resource/Michigan] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 919, P: 0.506, R: 0.562, macro F1: 0.515, macro F1*: 0.533 -[INFO] Current time: 6915 ms, average: 2902.77 ms/ques, entity detection: 499.87 ms/ques (linking tool: 406.54 ms/ques), relation detection: 940.88 ms/ques (similarity: 710.23 ms/ques), EDG generation: 46.74 ms/ques +[INFO] Cumulative metrics, sample: 919, P: 0.506, R: 0.562, macro F1: 0.533 +[INFO] Current question consumed time: 5913 ms, average: 2595.61 ms/ques, entity detection: 437.03 ms/ques (linking tool: 359.47 ms/ques), relation detection: 864.35 ms/ques (similarity: 703.90 ms/ques), EDG generation: 47.27 ms/ques [INFO] Entity Linking P: 0.7399347116430903, R: 0.6926006528835691, micro F1: 0.7078346028291622, macro F1: 0.7154856725572761 [INFO] Relation Linking P: 0.598929996372869, R: 0.5457018498367792, micro F1: 0.5611534276387387, macro F1: 0.5710783043922881 [INFO] Type Linking P: 0.558215451577802, R: 0.558215451577802, micro F1: 0.558215451577802, macro F1: 0.558215451577802 @@ -12897,8 +12897,8 @@ Question 919: From how many different institutes have the members of Judge Advoc [INFO] Golden Answer: [7] [INFO] Predicted Answer: [] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 920, P: 0.506, R: 0.561, macro F1: 0.514, macro F1*: 0.532 -[INFO] Current time: 2408 ms, average: 2902.23 ms/ques, entity detection: 499.89 ms/ques (linking tool: 406.50 ms/ques), relation detection: 940.41 ms/ques (similarity: 709.82 ms/ques), EDG generation: 46.80 ms/ques +[INFO] Cumulative metrics, sample: 920, P: 0.506, R: 0.561, macro F1: 0.532 +[INFO] Current question consumed time: 1775 ms, average: 2594.72 ms/ques, entity detection: 436.96 ms/ques (linking tool: 359.37 ms/ques), relation detection: 863.77 ms/ques (similarity: 703.39 ms/ques), EDG generation: 47.33 ms/ques [INFO] Entity Linking P: 0.7391304347826086, R: 0.6918478260869565, micro F1: 0.7070652173913045, macro F1: 0.714707970739279 [INFO] Relation Linking P: 0.5982789855072463, R: 0.5451086956521739, micro F1: 0.5605434782608705, macro F1: 0.5704575671049051 [INFO] Type Linking P: 0.5576086956521739, R: 0.5576086956521739, micro F1: 0.5576086956521739, macro F1: 0.5576086956521739 @@ -12911,8 +12911,8 @@ Question 920: Whos a famous relative of anne spielberg? [INFO] Golden Answer: [http://dbpedia.org/resource/Steven_Spielberg] [INFO] Predicted Answer: [http://dbpedia.org/resource/Steven_Spielberg] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 921, P: 0.506, R: 0.562, macro F1: 0.515, macro F1*: 0.533 -[INFO] Current time: 746 ms, average: 2899.89 ms/ques, entity detection: 499.49 ms/ques (linking tool: 406.12 ms/ques), relation detection: 939.44 ms/ques (similarity: 709.09 ms/ques), EDG generation: 46.77 ms/ques +[INFO] Cumulative metrics, sample: 921, P: 0.506, R: 0.562, macro F1: 0.533 +[INFO] Current question consumed time: 566 ms, average: 2592.52 ms/ques, entity detection: 436.57 ms/ques (linking tool: 359.02 ms/ques), relation detection: 862.88 ms/ques (similarity: 702.66 ms/ques), EDG generation: 47.30 ms/ques [INFO] Entity Linking P: 0.739413680781759, R: 0.6921824104234527, micro F1: 0.7073832790445169, macro F1: 0.7150189177070478 [INFO] Relation Linking P: 0.5987151646760767, R: 0.5456026058631922, micro F1: 0.5610206297502723, macro F1: 0.5709262976194738 [INFO] Type Linking P: 0.5580890336590663, R: 0.5580890336590663, micro F1: 0.5580890336590663, macro F1: 0.5580890336590663 @@ -12925,8 +12925,8 @@ Question 921: To which city does Northeast Library belong? [INFO] Golden Answer: [http://dbpedia.org/resource/Minneapolis] [INFO] Predicted Answer: [http://dbpedia.org/resource/Minneapolis] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 922, P: 0.507, R: 0.562, macro F1: 0.515, macro F1*: 0.533 -[INFO] Current time: 986 ms, average: 2897.82 ms/ques, entity detection: 499.17 ms/ques (linking tool: 405.79 ms/ques), relation detection: 938.70 ms/ques (similarity: 708.62 ms/ques), EDG generation: 46.73 ms/ques +[INFO] Cumulative metrics, sample: 922, P: 0.507, R: 0.562, macro F1: 0.533 +[INFO] Current question consumed time: 660 ms, average: 2590.42 ms/ques, entity detection: 436.27 ms/ques (linking tool: 358.72 ms/ques), relation detection: 862.04 ms/ques (similarity: 702.02 ms/ques), EDG generation: 47.26 ms/ques [INFO] Entity Linking P: 0.7396963123644251, R: 0.6925162689804772, micro F1: 0.7077006507592192, macro F1: 0.7153291865879372 [INFO] Relation Linking P: 0.5991503976861894, R: 0.5460954446854663, micro F1: 0.5614967462039054, macro F1: 0.5713940024970336 [INFO] Type Linking P: 0.5574837310195228, R: 0.5574837310195228, micro F1: 0.5574837310195228, macro F1: 0.5574837310195228 @@ -12939,8 +12939,8 @@ Question 922: What are the airline hubs of Cascade Airways? [INFO] Golden Answer: [http://dbpedia.org/resource/Buddhism, http://dbpedia.org/resource/Liberalism, http://dbpedia.org/resource/Royalist] [INFO] Predicted Answer: [Cascade Airways@en] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 923, P: 0.506, R: 0.562, macro F1: 0.515, macro F1*: 0.532 -[INFO] Current time: 828 ms, average: 2895.58 ms/ques, entity detection: 498.86 ms/ques (linking tool: 405.50 ms/ques), relation detection: 937.84 ms/ques (similarity: 707.96 ms/ques), EDG generation: 46.69 ms/ques +[INFO] Cumulative metrics, sample: 923, P: 0.506, R: 0.562, macro F1: 0.532 +[INFO] Current question consumed time: 653 ms, average: 2588.32 ms/ques, entity detection: 435.98 ms/ques (linking tool: 358.44 ms/ques), relation detection: 861.24 ms/ques (similarity: 701.35 ms/ques), EDG generation: 47.22 ms/ques [INFO] Entity Linking P: 0.7388949079089924, R: 0.6917659804983749, micro F1: 0.7069339111592634, macro F1: 0.7145541820520889 [INFO] Relation Linking P: 0.5985012639942217, R: 0.5455037919826652, micro F1: 0.5608884073672815, macro F1: 0.5707749407391819 [INFO] Type Linking P: 0.5579631635969664, R: 0.5579631635969664, micro F1: 0.5579631635969664, macro F1: 0.5579631635969664 @@ -12953,8 +12953,8 @@ Question 923: Who has manufactured vehicles designed originally by NASA? [INFO] Golden Answer: [http://dbpedia.org/resource/Boeing, http://dbpedia.org/resource/General_Motors] [INFO] Predicted Answer: [http://dbpedia.org/resource/Boeing, http://dbpedia.org/resource/General_Motors, http://dbpedia.org/resource/Scaled_Composites, http://dbpedia.org/resource/Lyndon_B._Johnson_Space_Center] [INFO] P: 0.5, R: 1.0, F: 0.6666666666666666 -[INFO] Cumulative metrics, sample: 924, P: 0.506, R: 0.562, macro F1: 0.515, macro F1*: 0.533 -[INFO] Current time: 4423 ms, average: 2897.23 ms/ques, entity detection: 498.94 ms/ques (linking tool: 405.54 ms/ques), relation detection: 938.71 ms/ques (similarity: 708.89 ms/ques), EDG generation: 46.65 ms/ques +[INFO] Cumulative metrics, sample: 924, P: 0.506, R: 0.562, macro F1: 0.533 +[INFO] Current question consumed time: 3714 ms, average: 2589.54 ms/ques, entity detection: 435.96 ms/ques (linking tool: 358.41 ms/ques), relation detection: 862.00 ms/ques (similarity: 702.19 ms/ques), EDG generation: 47.18 ms/ques [INFO] Entity Linking P: 0.7391774891774892, R: 0.6920995670995671, micro F1: 0.7072510822510824, macro F1: 0.7148642787584185 [INFO] Relation Linking P: 0.5983946608946609, R: 0.5454545454545454, micro F1: 0.5608225108225118, macro F1: 0.5706995047056546 [INFO] Type Linking P: 0.5573593073593074, R: 0.5573593073593074, micro F1: 0.5573593073593074, macro F1: 0.5573593073593074 @@ -12967,8 +12967,8 @@ Question 924: What is the order of the Henry Clay and profession of the William [INFO] Golden Answer: [http://dbpedia.org/resource/United_States_Senate] [INFO] Predicted Answer: [http://dbpedia.org/resource/United_States_Senate] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 925, P: 0.507, R: 0.563, macro F1: 0.515, macro F1*: 0.533 -[INFO] Current time: 3369 ms, average: 2897.74 ms/ques, entity detection: 498.91 ms/ques (linking tool: 405.41 ms/ques), relation detection: 939.22 ms/ques (similarity: 708.96 ms/ques), EDG generation: 46.64 ms/ques +[INFO] Cumulative metrics, sample: 925, P: 0.507, R: 0.563, macro F1: 0.533 +[INFO] Current question consumed time: 2722 ms, average: 2589.68 ms/ques, entity detection: 435.87 ms/ques (linking tool: 358.26 ms/ques), relation detection: 862.37 ms/ques (similarity: 702.26 ms/ques), EDG generation: 47.18 ms/ques [INFO] Entity Linking P: 0.7394594594594595, R: 0.6924324324324325, micro F1: 0.7075675675675677, macro F1: 0.7151737014477672 [INFO] Relation Linking P: 0.5988288288288288, R: 0.5459459459459459, micro F1: 0.5612972972972982, macro F1: 0.5711659247190843 [INFO] Type Linking P: 0.5578378378378378, R: 0.5578378378378378, micro F1: 0.5578378378378378, macro F1: 0.5578378378378378 @@ -12981,8 +12981,8 @@ Question 925: What show has theme music composer as Ron Grainer and at the same [INFO] Golden Answer: [http://dbpedia.org/resource/Doctor_Who] [INFO] Predicted Answer: [http://dbpedia.org/resource/The_Eagles_(UK_band)] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 926, P: 0.506, R: 0.562, macro F1: 0.515, macro F1*: 0.533 -[INFO] Current time: 1810 ms, average: 2896.56 ms/ques, entity detection: 498.57 ms/ques (linking tool: 405.07 ms/ques), relation detection: 938.84 ms/ques (similarity: 708.83 ms/ques), EDG generation: 46.72 ms/ques +[INFO] Cumulative metrics, sample: 926, P: 0.506, R: 0.562, macro F1: 0.533 +[INFO] Current question consumed time: 1590 ms, average: 2588.60 ms/ques, entity detection: 435.56 ms/ques (linking tool: 357.95 ms/ques), relation detection: 862.01 ms/ques (similarity: 702.13 ms/ques), EDG generation: 47.26 ms/ques [INFO] Entity Linking P: 0.7397408207343412, R: 0.6922246220302376, micro F1: 0.7075233981281498, macro F1: 0.7151943681609544 [INFO] Relation Linking P: 0.5981821454283657, R: 0.5453563714902808, micro F1: 0.5606911447084242, macro F1: 0.5705491148651761 [INFO] Type Linking P: 0.5572354211663066, R: 0.5572354211663066, micro F1: 0.5572354211663066, macro F1: 0.5572354211663066 @@ -12995,8 +12995,8 @@ Question 926: What is the life stance of the ethnic groups related to Micronesia [INFO] Golden Answer: [http://dbpedia.org/resource/Animism, http://dbpedia.org/resource/Christianity, http://dbpedia.org/resource/Shamanism, http://dbpedia.org/resource/Catholic_Church, http://dbpedia.org/resource/Mahayana, http://dbpedia.org/resource/Protestantism, http://dbpedia.org/resource/Shinto, http://dbpedia.org/resource/Buddhism] [INFO] Predicted Answer: [http://dbpedia.org/resource/Kiribati, http://dbpedia.org/resource/Tuvalu, http://dbpedia.org/resource/Lelu_Island, http://dbpedia.org/resource/Fais_Island, http://dbpedia.org/resource/Satawal, http://dbpedia.org/resource/Palau, http://dbpedia.org/resource/Woleai, http://dbpedia.org/resource/Eauripik] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 927, P: 0.506, R: 0.562, macro F1: 0.514, macro F1*: 0.532 -[INFO] Current time: 1078 ms, average: 2894.60 ms/ques, entity detection: 498.34 ms/ques (linking tool: 404.86 ms/ques), relation detection: 938.15 ms/ques (similarity: 708.25 ms/ques), EDG generation: 46.70 ms/ques +[INFO] Cumulative metrics, sample: 927, P: 0.506, R: 0.562, macro F1: 0.532 +[INFO] Current question consumed time: 932 ms, average: 2586.82 ms/ques, entity detection: 435.35 ms/ques (linking tool: 357.76 ms/ques), relation detection: 861.38 ms/ques (similarity: 701.56 ms/ques), EDG generation: 47.24 ms/ques [INFO] Entity Linking P: 0.7400215749730313, R: 0.6925566343042071, micro F1: 0.7078389068680332, macro F1: 0.7155027878504309 [INFO] Relation Linking P: 0.597536857245595, R: 0.5447680690399137, micro F1: 0.560086299892126, macro F1: 0.5699336357768642 [INFO] Type Linking P: 0.5566343042071198, R: 0.5566343042071198, micro F1: 0.5566343042071198, macro F1: 0.5566343042071198 @@ -13009,8 +13009,8 @@ Question 927: Name the river whose mouth mountain is Southend-on-Sea and mouth p [INFO] Golden Answer: [http://dbpedia.org/resource/River_Thames] [INFO] Predicted Answer: [http://dbpedia.org/resource/River_Stour,_Suffolk, http://dbpedia.org/resource/River_Thames] [INFO] P: 0.5, R: 1.0, F: 0.6666666666666666 -[INFO] Cumulative metrics, sample: 928, P: 0.506, R: 0.562, macro F1: 0.514, macro F1*: 0.532 -[INFO] Current time: 4039 ms, average: 2895.84 ms/ques, entity detection: 498.38 ms/ques (linking tool: 404.88 ms/ques), relation detection: 938.64 ms/ques (similarity: 709.05 ms/ques), EDG generation: 46.70 ms/ques +[INFO] Cumulative metrics, sample: 928, P: 0.506, R: 0.562, macro F1: 0.532 +[INFO] Current question consumed time: 3684 ms, average: 2588.00 ms/ques, entity detection: 435.37 ms/ques (linking tool: 357.78 ms/ques), relation detection: 861.84 ms/ques (similarity: 702.35 ms/ques), EDG generation: 47.24 ms/ques [INFO] Entity Linking P: 0.740301724137931, R: 0.6923491379310345, micro F1: 0.7077945402298851, macro F1: 0.7155229150186095 [INFO] Relation Linking P: 0.5979705459770115, R: 0.5447198275862069, micro F1: 0.5602011494252882, macro F1: 0.5701044136576112 [INFO] Type Linking P: 0.5560344827586207, R: 0.5560344827586207, micro F1: 0.5560344827586207, macro F1: 0.5560344827586207 @@ -13023,8 +13023,8 @@ Question 928: Which venue of indy PopCon is also the death palce of Louis Le Coc [INFO] Golden Answer: [http://dbpedia.org/resource/Indianapolis] [INFO] Predicted Answer: [http://dbpedia.org/resource/Indianapolis, http://dbpedia.org/resource/Indiana_Convention_Center, http://dbpedia.org/resource/Downtown] [INFO] P: 0.3333333333333333, R: 1.0, F: 0.5 -[INFO] Cumulative metrics, sample: 929, P: 0.505, R: 0.562, macro F1: 0.514, macro F1*: 0.532 -[INFO] Current time: 2490 ms, average: 2895.40 ms/ques, entity detection: 498.52 ms/ques (linking tool: 404.85 ms/ques), relation detection: 938.41 ms/ques (similarity: 708.74 ms/ques), EDG generation: 46.75 ms/ques +[INFO] Cumulative metrics, sample: 929, P: 0.505, R: 0.562, macro F1: 0.532 +[INFO] Current question consumed time: 2293 ms, average: 2587.68 ms/ques, entity detection: 435.54 ms/ques (linking tool: 357.81 ms/ques), relation detection: 861.66 ms/ques (similarity: 702.04 ms/ques), EDG generation: 47.30 ms/ques [INFO] Entity Linking P: 0.7405812701829925, R: 0.6921420882669537, micro F1: 0.7077502691065662, macro F1: 0.7155428350528387 [INFO] Relation Linking P: 0.5984033010405454, R: 0.5446716899892358, micro F1: 0.5603157517043424, macro F1: 0.5702746361011051 [INFO] Type Linking P: 0.5565123789020452, R: 0.5565123789020452, micro F1: 0.5565123789020452, macro F1: 0.5565123789020452 @@ -13038,8 +13038,8 @@ Question 929: What genre's softwares are released with a GNU GPL license? Chrome extension@en, Browser extension, Windows Store App, Windows Phone, Android and iOS@en, http://dbpedia.org/resource/Rendering_(computer_graphics), http://dbpedia.org/resource/Disk_image, Application Delivery@en, http://dbpedia.org/resource/Multimedia, Digital music workstation@en, ISO mounter@en, http://dbpedia.org/resource/Time_management, http://dbpedia.org/resource/Task_management, Mobile application framework@en, Debian-Installer loader@en, Profiler@en, http://dbpedia.org/resource/Logic_simulation, http://dbpedia.org/resource/Navigation, Accounting software, POS software, Content management system and Enterprise resource planning@en, http://dbpedia.org/resource/Continuous_integration, http://dbpedia.org/resource/Google_Chrome_extension, http://dbpedia.org/resource/Church_software, WLAN device drivers@en, profiler@en, performance monitor and testing@en, Version control system client@en, Scientific Software, Mathematical software@en, Web-based user-configurable data management software@en, http://dbpedia.org/resource/BitTorrent_tracker, http://dbpedia.org/resource/Internet_bot, Router-OS@en, http://dbpedia.org/resource/Category:Network-related_software, Molecular biology toolkit@en, Insurance Quotation Solution@en, Dictionary Writing Software@en, http://dbpedia.org/resource/Business_process_management, http://dbpedia.org/resource/Workflow, http://dbpedia.org/resource/Content-control_software, http://dbpedia.org/resource/Agent-based_model, intellectual locale switchers@en, Scientific simulation software@en] [INFO] Predicted Answer: [http://dbpedia.org/resource/Library_(computing)] [INFO] P: 1.0, R: 0.0011198208286674132, F: 0.0022371364653243847 -[INFO] Cumulative metrics, sample: 930, P: 0.506, R: 0.562, macro F1: 0.514, macro F1*: 0.532 -[INFO] Current time: 2151 ms, average: 2894.60 ms/ques, entity detection: 498.93 ms/ques (linking tool: 405.31 ms/ques), relation detection: 937.81 ms/ques (similarity: 708.34 ms/ques), EDG generation: 46.72 ms/ques +[INFO] Cumulative metrics, sample: 930, P: 0.506, R: 0.562, macro F1: 0.532 +[INFO] Current question consumed time: 1906 ms, average: 2586.95 ms/ques, entity detection: 435.91 ms/ques (linking tool: 358.22 ms/ques), relation detection: 861.09 ms/ques (similarity: 701.64 ms/ques), EDG generation: 47.27 ms/ques [INFO] Entity Linking P: 0.7397849462365591, R: 0.6913978494623656, micro F1: 0.706989247311828, macro F1: 0.7147734341549323 [INFO] Relation Linking P: 0.5988351254480286, R: 0.5446236559139785, micro F1: 0.5604301075268825, macro F1: 0.5704443056928326 [INFO] Type Linking P: 0.556989247311828, R: 0.556989247311828, micro F1: 0.556989247311828, macro F1: 0.556989247311828 @@ -13052,8 +13052,8 @@ Question 930: Is Christchurch the largest city of South Island? [INFO] Golden Answer: [true] [INFO] Predicted Answer: [true] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 931, P: 0.506, R: 0.562, macro F1: 0.514, macro F1*: 0.533 -[INFO] Current time: 2503 ms, average: 2894.18 ms/ques, entity detection: 499.21 ms/ques (linking tool: 405.46 ms/ques), relation detection: 937.68 ms/ques (similarity: 708.05 ms/ques), EDG generation: 46.72 ms/ques +[INFO] Cumulative metrics, sample: 931, P: 0.506, R: 0.562, macro F1: 0.533 +[INFO] Current question consumed time: 2077 ms, average: 2586.40 ms/ques, entity detection: 436.10 ms/ques (linking tool: 358.32 ms/ques), relation detection: 860.97 ms/ques (similarity: 701.36 ms/ques), EDG generation: 47.23 ms/ques [INFO] Entity Linking P: 0.7400644468313641, R: 0.6917293233082706, micro F1: 0.7073039742212675, macro F1: 0.7150810258955717 [INFO] Relation Linking P: 0.599266022198353, R: 0.5451127819548872, micro F1: 0.5609022556390986, macro F1: 0.5709081071862292 [INFO] Type Linking P: 0.5574650912996778, R: 0.5574650912996778, micro F1: 0.5574650912996778, macro F1: 0.5574650912996778 @@ -13066,8 +13066,8 @@ Question 931: Who is the leader of Allgemeine SS ? [INFO] Golden Answer: [http://dbpedia.org/resource/Heinrich_Himmler] [INFO] Predicted Answer: [http://dbpedia.org/resource/Heinrich_Himmler] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 932, P: 0.507, R: 0.563, macro F1: 0.515, macro F1*: 0.533 -[INFO] Current time: 889 ms, average: 2892.03 ms/ques, entity detection: 498.82 ms/ques (linking tool: 405.11 ms/ques), relation detection: 937.05 ms/ques (similarity: 707.50 ms/ques), EDG generation: 46.68 ms/ques +[INFO] Cumulative metrics, sample: 932, P: 0.507, R: 0.563, macro F1: 0.533 +[INFO] Current question consumed time: 750 ms, average: 2584.43 ms/ques, entity detection: 435.76 ms/ques (linking tool: 358.02 ms/ques), relation detection: 860.40 ms/ques (similarity: 700.82 ms/ques), EDG generation: 47.18 ms/ques [INFO] Entity Linking P: 0.740343347639485, R: 0.6920600858369099, micro F1: 0.7076180257510729, macro F1: 0.7153879538988281 [INFO] Relation Linking P: 0.5996959942775393, R: 0.5456008583690987, micro F1: 0.5613733905579407, macro F1: 0.5713709043768515 [INFO] Type Linking P: 0.5579399141630901, R: 0.5579399141630901, micro F1: 0.5579399141630901, macro F1: 0.5579399141630901 @@ -13080,8 +13080,8 @@ Question 932: Giorgos Kaminis is the mayor of what? [INFO] Golden Answer: [http://dbpedia.org/resource/Athens] [INFO] Predicted Answer: [http://dbpedia.org/resource/Athens] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 933, P: 0.507, R: 0.563, macro F1: 0.515, macro F1*: 0.534 -[INFO] Current time: 891 ms, average: 2889.88 ms/ques, entity detection: 498.52 ms/ques (linking tool: 404.80 ms/ques), relation detection: 936.31 ms/ques (similarity: 706.89 ms/ques), EDG generation: 46.65 ms/ques +[INFO] Cumulative metrics, sample: 933, P: 0.507, R: 0.563, macro F1: 0.534 +[INFO] Current question consumed time: 753 ms, average: 2582.47 ms/ques, entity detection: 435.50 ms/ques (linking tool: 357.73 ms/ques), relation detection: 859.71 ms/ques (similarity: 700.21 ms/ques), EDG generation: 47.15 ms/ques [INFO] Entity Linking P: 0.7406216505894962, R: 0.692390139335477, micro F1: 0.7079314040728831, macro F1: 0.7156942203153547 [INFO] Relation Linking P: 0.6001250446588067, R: 0.5460878885316184, micro F1: 0.5618435155412655, macro F1: 0.5718327005445248 [INFO] Type Linking P: 0.5584137191854234, R: 0.5584137191854234, micro F1: 0.5584137191854234, macro F1: 0.5584137191854234 @@ -13094,8 +13094,8 @@ Question 933: Where is the headquarters of Metro Transit (Minnesota) ? [INFO] Golden Answer: [http://dbpedia.org/resource/Minneapolis, http://dbpedia.org/resource/Minnesota] [INFO] Predicted Answer: [http://dbpedia.org/resource/Minneapolis, http://dbpedia.org/resource/Minnesota] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 934, P: 0.508, R: 0.564, macro F1: 0.516, macro F1*: 0.534 -[INFO] Current time: 2013 ms, average: 2888.94 ms/ques, entity detection: 498.60 ms/ques (linking tool: 404.93 ms/ques), relation detection: 935.69 ms/ques (similarity: 706.37 ms/ques), EDG generation: 46.62 ms/ques +[INFO] Cumulative metrics, sample: 934, P: 0.508, R: 0.564, macro F1: 0.534 +[INFO] Current question consumed time: 1798 ms, average: 2581.63 ms/ques, entity detection: 435.60 ms/ques (linking tool: 357.86 ms/ques), relation detection: 859.16 ms/ques (similarity: 699.71 ms/ques), EDG generation: 47.11 ms/ques [INFO] Entity Linking P: 0.7408993576017131, R: 0.6927194860813705, micro F1: 0.7082441113490364, macro F1: 0.715999827286495 [INFO] Relation Linking P: 0.6005531763026409, R: 0.5465738758029979, micro F1: 0.5623126338329772, macro F1: 0.5722934989546518 [INFO] Type Linking P: 0.5588865096359743, R: 0.5588865096359743, micro F1: 0.5588865096359743, macro F1: 0.5588865096359743 @@ -13108,8 +13108,8 @@ Question 934: Give me the count of all the names of characters created by Frank [INFO] Golden Answer: [18] [INFO] Predicted Answer: [10, 27] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 935, P: 0.507, R: 0.563, macro F1: 0.515, macro F1*: 0.534 -[INFO] Current time: 2903 ms, average: 2888.96 ms/ques, entity detection: 498.33 ms/ques (linking tool: 404.66 ms/ques), relation detection: 935.61 ms/ques (similarity: 706.98 ms/ques), EDG generation: 46.61 ms/ques +[INFO] Cumulative metrics, sample: 935, P: 0.507, R: 0.563, macro F1: 0.534 +[INFO] Current question consumed time: 2585 ms, average: 2581.63 ms/ques, entity detection: 435.31 ms/ques (linking tool: 357.59 ms/ques), relation detection: 859.07 ms/ques (similarity: 700.35 ms/ques), EDG generation: 47.11 ms/ques [INFO] Entity Linking P: 0.7411764705882353, R: 0.6930481283422459, micro F1: 0.7085561497326203, macro F1: 0.7163047769443348 [INFO] Relation Linking P: 0.5999108734402852, R: 0.5459893048128343, micro F1: 0.5617112299465249, macro F1: 0.5716814203461441 [INFO] Type Linking P: 0.5593582887700534, R: 0.5593582887700534, micro F1: 0.5593582887700534, macro F1: 0.5593582887700534 @@ -13122,8 +13122,8 @@ Question 935: Which fictional character portrayed in Terminator 2: Judgment Day [INFO] Golden Answer: [http://dbpedia.org/resource/John_Connor] [INFO] Predicted Answer: [http://dbpedia.org/resource/Sarah_Connor_(Terminator), http://dbpedia.org/resource/Cameron_(Terminator), http://dbpedia.org/resource/John_Connor] [INFO] P: 0.3333333333333333, R: 1.0, F: 0.5 -[INFO] Cumulative metrics, sample: 936, P: 0.507, R: 0.564, macro F1: 0.515, macro F1*: 0.534 -[INFO] Current time: 2084 ms, average: 2888.10 ms/ques, entity detection: 498.35 ms/ques (linking tool: 404.57 ms/ques), relation detection: 934.87 ms/ques (similarity: 706.48 ms/ques), EDG generation: 46.60 ms/ques +[INFO] Cumulative metrics, sample: 936, P: 0.507, R: 0.564, macro F1: 0.534 +[INFO] Current question consumed time: 1725 ms, average: 2580.72 ms/ques, entity detection: 435.30 ms/ques (linking tool: 357.49 ms/ques), relation detection: 858.40 ms/ques (similarity: 699.86 ms/ques), EDG generation: 47.10 ms/ques [INFO] Entity Linking P: 0.7414529914529915, R: 0.6928418803418803, micro F1: 0.708511396011396, macro F1: 0.7163236721895939 [INFO] Relation Linking P: 0.600338319088319, R: 0.5459401709401709, micro F1: 0.5618233618233626, macro F1: 0.5718484773047879 [INFO] Type Linking P: 0.5587606837606838, R: 0.5587606837606838, micro F1: 0.5587606837606838, macro F1: 0.5587606837606838 @@ -13136,8 +13136,8 @@ Question 936: Where was the narrator of Spain on the Road Again born? [INFO] Golden Answer: [http://dbpedia.org/resource/Seattle, http://dbpedia.org/resource/United_States, http://dbpedia.org/resource/Washington_(state)] [INFO] Predicted Answer: [http://dbpedia.org/resource/Armando_Salas, http://dbpedia.org/resource/Carlos_Saura, http://dbpedia.org/resource/Felipe_VI_of_Spain, http://dbpedia.org/resource/Gaspar_Sanz, http://dbpedia.org/resource/Ignacio_Velázquez_Rivera, http://dbpedia.org/resource/Joaquín_Rodrigo, http://dbpedia.org/resource/José_Sánchez_Mota, http://dbpedia.org/resource/Pablo_Morillo, http://dbpedia.org/resource/Paco_de_Lucía, http://dbpedia.org/resource/Pedro_de_Alvarado, http://dbpedia.org/resource/Antonio_Ordóñez, http://dbpedia.org/resource/Carlos_Checa, http://dbpedia.org/resource/Carlos_Montoya, http://dbpedia.org/resource/Constantino_Romero, http://dbpedia.org/resource/Florentino_Pérez, http://dbpedia.org/resource/Gregorio_Lavilla, http://dbpedia.org/resource/Joan_Laporta, http://dbpedia.org/resource/John_Macias, http://dbpedia.org/resource/Jorge_Garbajosa, http://dbpedia.org/resource/Juan_Ramón_López_Caro, http://dbpedia.org/resource/Lydia_Zimmermann, http://dbpedia.org/resource/Mariano_Goybet, http://dbpedia.org/resource/Nacho_Vidal, http://dbpedia.org/resource/Nieves_Navarro, http://dbpedia.org/resource/Santiago_Ezquerro, http://dbpedia.org/resource/Xavier_Cugat, http://dbpedia.org/resource/Ángel_Guirado, http://dbpedia.org/resource/Agustín_Guzmán, http://dbpedia.org/resource/Anxo_Quintana, http://dbpedia.org/resource/Cristina_Pato, http://dbpedia.org/resource/David_Cal, http://dbpedia.org/resource/Dionisio_Alcalá_Galiano, http://dbpedia.org/resource/Elena_Gómez, http://dbpedia.org/resource/Enrique_Collar, http://dbpedia.org/resource/Iván_Raña, http://dbpedia.org/resource/Luis_Fernández, http://dbpedia.org/resource/Luis_Royo, http://dbpedia.org/resource/Severo_Ochoa, http://dbpedia.org/resource/Suso_de_Toro, http://dbpedia.org/resource/Ángel_Cabrera_(academic), http://dbpedia.org/resource/Buenaventura_Durruti, http://dbpedia.org/resource/José_Díaz_(politician), http://dbpedia.org/resource/Judah_Halevi, http://dbpedia.org/resource/Julián_Simón, http://dbpedia.org/resource/Luis_E._Echávarri, http://dbpedia.org/resource/Luis_de_Guindos, http://dbpedia.org/resource/Miguel_Llobet, http://dbpedia.org/resource/Paz_Vega, http://dbpedia.org/resource/Ramon_Casas_i_Carbó, http://dbpedia.org/resource/Remedios_Varo, http://dbpedia.org/resource/Rodrigo_Rato, http://dbpedia.org/resource/Adolfo_Domínguez, http://dbpedia.org/resource/Alfredo_Landa, http://dbpedia.org/resource/Ana_Belén, http://dbpedia.org/resource/Asier_Illarramendi, http://dbpedia.org/resource/Belén_Rueda, http://dbpedia.org/resource/Beñat_Albizuri, http://dbpedia.org/resource/Carmen_de_Icaza, http://dbpedia.org/resource/Daniel_Vázquez_Evuy, http://dbpedia.org/resource/Diego_Mazquiarán, http://dbpedia.org/resource/Francisco_Ayala_(novelist), http://dbpedia.org/resource/Isaac_Gálvez, http://dbpedia.org/resource/Javier_Portillo, http://dbpedia.org/resource/José_Luis_López_Vázquez, http://dbpedia.org/resource/José_Manuel_Rodriguez_Delgado, http://dbpedia.org/resource/José_de_la_Borda, http://dbpedia.org/resource/Juan_José_Ibarretxe, http://dbpedia.org/resource/Juan_María_Fernández_y_Krohn, http://dbpedia.org/resource/Juanito_Oiarzabal, http://dbpedia.org/resource/Manfred_Sommer, http://dbpedia.org/resource/Miguel_Pérez_(footballer), http://dbpedia.org/resource/Pablo_Alborán, http://dbpedia.org/resource/Paul_Naschy, http://dbpedia.org/resource/Pepe_Romero, http://dbpedia.org/resource/Rafael_Gil, http://dbpedia.org/resource/Rafael_Moneo, http://dbpedia.org/resource/Ramón_Blanco_Rodríguez, http://dbpedia.org/resource/Richard_Blanco, http://dbpedia.org/resource/Salva_Ballesta, http://dbpedia.org/resource/Víctor_Muñoz, http://dbpedia.org/resource/Xavier_Tondo, http://dbpedia.org/resource/Yon_González, http://dbpedia.org/resource/Domingo_de_Santo_Tomás, http://dbpedia.org/resource/Julio_Cervera_Baviera, http://dbpedia.org/resource/Manuel_María_de_Salcedo, http://dbpedia.org/resource/Nani_Roma, http://dbpedia.org/resource/Abel_Resino, http://dbpedia.org/resource/Alejandro_Díez_Macho, http://dbpedia.org/resource/Eladio_Jiménez, http://dbpedia.org/resource/Francisco_Cepeda_(cyclist), http://dbpedia.org/resource/Francisco_Mancebo, http://dbpedia.org/resource/Igor_González_de_Galdeano, http://dbpedia.org/resource/Jon_Odriozola, http://dbpedia.org/resource/Jorge_Semprún, http://dbpedia.org/resource/Luis_Pérez_Rodríguez, http://dbpedia.org/resource/Manuel_Beltrán, http://dbpedia.org/resource/Mathieu_Orfila, http://dbpedia.org/resource/Mikel_Pradera, http://dbpedia.org/resource/Roberto_Laiseka, http://dbpedia.org/resource/Tristán_de_Luna_y_Arellano, http://dbpedia.org/resource/Vicente_Trueba, http://dbpedia.org/resource/Javier_Urruticoechea, http://dbpedia.org/resource/Juan_Bautista_Basset_y_Ramos, http://dbpedia.org/resource/Juan_de_Herrera, http://dbpedia.org/resource/Fernando_Muñoz, http://dbpedia.org/resource/Joan_Cañellas, http://dbpedia.org/resource/José_María_Maravall_Herrero, http://dbpedia.org/resource/Juan_de_Pareja, http://dbpedia.org/resource/Juanita_Reina, http://dbpedia.org/resource/Paquita_Rico, http://dbpedia.org/resource/Pedro_Fages, http://dbpedia.org/resource/Ricardo_Serna, http://dbpedia.org/resource/Teresa_Sánchez_López, http://dbpedia.org/resource/Valero_Rivera_Folch, http://dbpedia.org/resource/Verónica_Sánchez, http://dbpedia.org/resource/Vital_Alsar, http://dbpedia.org/resource/Carles_Puigdemont, http://dbpedia.org/resource/Cristina_Cifuentes, http://dbpedia.org/resource/Iñigo_Urkullu, http://dbpedia.org/resource/Javier_Fernández_Fernández, http://dbpedia.org/resource/Miguel_Ángel_Revilla, http://dbpedia.org/resource/Uxue_Barkos, http://dbpedia.org/resource/Ximo_Puig, http://dbpedia.org/resource/Arturo_Barea, http://dbpedia.org/resource/Carolina_Coronado, http://dbpedia.org/resource/Fidel_Pagés, http://dbpedia.org/resource/José_Joaquín_de_Arrillaga, http://dbpedia.org/resource/Luis_Jayme, http://dbpedia.org/resource/Albert_Monteys, http://dbpedia.org/resource/Alberto_Fabra, http://dbpedia.org/resource/Ana_Miralles, http://dbpedia.org/resource/Antonio_Segura, http://dbpedia.org/resource/Carlos_López-Cantera, http://dbpedia.org/resource/Fernando_Caruncho, http://dbpedia.org/resource/Fernando_de_Santiago_y_Díaz, http://dbpedia.org/resource/Francisco_Páez_de_la_Cadena, http://dbpedia.org/resource/Joaquín_Cera, http://dbpedia.org/resource/Luis_García_Mozos, http://dbpedia.org/resource/María_del_Carmen_Martínez-Bordiú_y_Franco, http://dbpedia.org/resource/Princess_Marie_Isabelle_of_Orléans, http://dbpedia.org/resource/Scott_Kolins, http://dbpedia.org/resource/Concha_Méndez, http://dbpedia.org/resource/María_Teresa_León, http://dbpedia.org/resource/Antonio_Vázquez_Romero, http://dbpedia.org/resource/Jesús_Posada, http://dbpedia.org/resource/Adam_Raga, http://dbpedia.org/resource/Albert_Sánchez_Piñol, http://dbpedia.org/resource/Amaia_Montero, http://dbpedia.org/resource/Antonio_Valdés_y_Fernández_Bazán, http://dbpedia.org/resource/Beth_(singer), http://dbpedia.org/resource/Claudia_Bassols, http://dbpedia.org/resource/Concha_Buika, http://dbpedia.org/resource/Joan_Gili, http://dbpedia.org/resource/Joan_Roca_i_Fontané, http://dbpedia.org/resource/Joan_Vinyoli, http://dbpedia.org/resource/Josep_Tapiró_Baró, http://dbpedia.org/resource/Josep_Vicenç_Foix, http://dbpedia.org/resource/Kílian_Jornet_Burgada, http://dbpedia.org/resource/Laureano_de_Torres_y_Ayala, http://dbpedia.org/resource/Luis_Antonio_García_Navarro, http://dbpedia.org/resource/Mireia_Belmonte, http://dbpedia.org/resource/Oriol_Elcacho, http://dbpedia.org/resource/Oriol_Servià, http://dbpedia.org/resource/Pere_Calders, http://dbpedia.org/resource/Quim_Monzó, http://dbpedia.org/resource/Roger_Grimau, http://dbpedia.org/resource/Segundo_Llorente, http://dbpedia.org/resource/Xavier_Vilalta, http://dbpedia.org/resource/José_María_García-Aranda, http://dbpedia.org/resource/David_Lozano, http://dbpedia.org/resource/Javier_Aguirresarobe, http://dbpedia.org/resource/José_Joaquín_de_Arredondo, http://dbpedia.org/resource/Baltasar_de_Zúñiga, http://dbpedia.org/resource/María_González_Veracruz, http://dbpedia.org/resource/Elvira_Quintillá, http://dbpedia.org/resource/Manuel_Redondo, http://dbpedia.org/resource/María_Rivas_(actress), http://dbpedia.org/resource/Álvaro_de_Luna_(actor), http://dbpedia.org/resource/Mark_Miremont, http://dbpedia.org/resource/Andrés_Gandarias, http://dbpedia.org/resource/Gérard_Hernandez, http://dbpedia.org/resource/Julián_Berrendero, http://dbpedia.org/resource/Laudelino_Cubino, http://dbpedia.org/resource/Damià_Abella, http://dbpedia.org/resource/Emilio_Nsue, http://dbpedia.org/resource/Francisco_Molina, http://dbpedia.org/resource/Iván_Pérez_Muñoz, http://dbpedia.org/resource/Jordi_Magem_Badals, http://dbpedia.org/resource/Aitor_Osa, http://dbpedia.org/resource/Benjamín_Noval, http://dbpedia.org/resource/Iker_Camaño, http://dbpedia.org/resource/Iker_Flores, http://dbpedia.org/resource/José_Vicente_García, http://dbpedia.org/resource/Santiago_Pérez_(cyclist), http://dbpedia.org/resource/Santos_González, http://dbpedia.org/resource/Pedro_Arico_Suárez, http://dbpedia.org/resource/Lucia_Palacios, http://dbpedia.org/resource/Alfonso_Cela, http://dbpedia.org/resource/Emilio_Muñoz, http://dbpedia.org/resource/Francisco_Rivera_Ordóñez, http://dbpedia.org/resource/José_Ortega_Cano, http://dbpedia.org/resource/Julio_Aparicio_Díaz, http://dbpedia.org/resource/Julián_López_Escobar, http://dbpedia.org/resource/Tuni_(footballer), http://dbpedia.org/resource/Antonio_Mercero, http://dbpedia.org/resource/David_Davis_(handballer), http://dbpedia.org/resource/Enric_González, http://dbpedia.org/resource/Fermin_Muguruza, http://dbpedia.org/resource/Pablo_Couñago, http://dbpedia.org/resource/Roberto_Carlos_(Spanish_footballer), http://dbpedia.org/resource/Antonio_Sebastián_de_Toledo,_2nd_Marquis_of_Mancera, http://dbpedia.org/resource/Juan_Suárez_Botas, http://dbpedia.org/resource/Nacho_Canut, http://dbpedia.org/resource/Antonio_Benítez, http://dbpedia.org/resource/Joaquín_López-Dóriga, http://dbpedia.org/resource/Juan_Ruiz_Anchía, http://dbpedia.org/resource/María_Luisa_Bergaz_Conesa, http://dbpedia.org/resource/María_Pujalte, http://dbpedia.org/resource/Vera_Pawlowsky-Glahn, http://dbpedia.org/resource/Fernando_Llorente_Mañas, http://dbpedia.org/resource/José_Ángel_Mañas, http://dbpedia.org/resource/Carlos_Pau_y_Español, http://dbpedia.org/resource/Fabián_Roncero, http://dbpedia.org/resource/Joan_Cornellà, http://dbpedia.org/resource/Sergio_López_Miró, http://dbpedia.org/resource/Pablo_Larrazábal, http://dbpedia.org/resource/Robert_MacLean, http://dbpedia.org/resource/Ángel_Edo, http://dbpedia.org/resource/Adrián_Alonso_Pereira, http://dbpedia.org/resource/Dani_Quintana, http://dbpedia.org/resource/Fele_Martínez, http://dbpedia.org/resource/Gerard_Badía, http://dbpedia.org/resource/Antonio_del_Real, http://dbpedia.org/resource/Jade_Boho, http://dbpedia.org/resource/Jorge_Salán, http://dbpedia.org/resource/Mariano_Barroso, http://dbpedia.org/resource/Raúl_Fabiani, http://dbpedia.org/resource/Ricardo_Franco, http://dbpedia.org/resource/Virginia_Buika, http://dbpedia.org/resource/Francisco_García_Romero, http://dbpedia.org/resource/Ibán_Pérez, http://dbpedia.org/resource/Josef_Lontscharitsch, http://dbpedia.org/resource/José_María_Alvira, http://dbpedia.org/resource/José_Pérez_Francés, http://dbpedia.org/resource/Marcos_Vidal, http://dbpedia.org/resource/Robert_Pérez_Palou, http://dbpedia.org/resource/Saray_García, http://dbpedia.org/resource/Sebastián_Rodríguez_Veloso, http://dbpedia.org/resource/Óscar_Pérez_Bovela, http://dbpedia.org/resource/Jaime_de_Piniés, http://dbpedia.org/resource/Xavier_Pascual_Vives, http://dbpedia.org/resource/Alvarez_of_Córdoba, http://dbpedia.org/resource/Sergio_Hernández_(racing_driver), http://dbpedia.org/resource/Antonio_Trashorras, http://dbpedia.org/resource/Clara_Lago, http://dbpedia.org/resource/Jordi_Dauder, http://dbpedia.org/resource/Poli_Díaz, http://dbpedia.org/resource/Unax_Ugalde, http://dbpedia.org/resource/Víctor_García_(Spanish_director), http://dbpedia.org/resource/Nicolás_del_Campo, http://dbpedia.org/resource/Ray_Rowe_(American_football), http://dbpedia.org/resource/Joan_Maria_Vendrell_Martínez, http://dbpedia.org/resource/Jordi_Morales, http://dbpedia.org/resource/Manuel_Hernandez_(soccer), http://dbpedia.org/resource/Alejandro_Zaera-Polo, http://dbpedia.org/resource/Toñito, http://dbpedia.org/resource/Gonzalo_López-Gallego, http://dbpedia.org/resource/Alberto_Leanizbarrutia, http://dbpedia.org/resource/Manuel_Olaguer_Feliú, http://dbpedia.org/resource/Olivia_Molina_(actress), http://dbpedia.org/resource/Bárbara_Elorrieta, http://dbpedia.org/resource/Enrique_Arce, http://dbpedia.org/resource/Luis_Prendes, http://dbpedia.org/resource/Manuel_Gaspar_Haro, http://dbpedia.org/resource/Bruno_Carotti, http://dbpedia.org/resource/José_Antonio_Gentico, http://dbpedia.org/resource/Manuel_Luna, http://dbpedia.org/resource/Alberto_Tous, http://dbpedia.org/resource/José_Antonio_Conde_(tennis), http://dbpedia.org/resource/Almudena_Muñoz, http://dbpedia.org/resource/Amaia_Andrés, http://dbpedia.org/resource/Amat_Escalante, http://dbpedia.org/resource/Ana_María_Tostado, http://dbpedia.org/resource/Andrés_Crespo, http://dbpedia.org/resource/Asunción_Domenech, http://dbpedia.org/resource/Benjamín_Vicedo, http://dbpedia.org/resource/Carmen_Miranda_(volleyball), http://dbpedia.org/resource/Chiquito_de_la_Calzada, http://dbpedia.org/resource/César_González_(fencer), http://dbpedia.org/resource/David_Villabona, http://dbpedia.org/resource/Estela_Domínguez, http://dbpedia.org/resource/Esther_Koplowitz,_Marquise_of_Cubas, http://dbpedia.org/resource/Francisco_Hervás_(volleyball), http://dbpedia.org/resource/Francisco_Sánchez_(volleyball), http://dbpedia.org/resource/Gabriel_Vidal, http://dbpedia.org/resource/Héctor_López_(volleyball), http://dbpedia.org/resource/Inmaculada_González, http://dbpedia.org/resource/Inmaculada_Torres, http://dbpedia.org/resource/Jesús_Centeno, http://dbpedia.org/resource/Jesús_Garrido, http://dbpedia.org/resource/Jesús_Sánchez_(volleyball), http://dbpedia.org/resource/José_Luis_Álvarez, http://dbpedia.org/resource/Laura_de_la_Torre, http://dbpedia.org/resource/Marco_Antonio_Rioja, http://dbpedia.org/resource/Marta_Gens, http://dbpedia.org/resource/María_del_Mar_Rey, http://dbpedia.org/resource/Miguel_Hernández_(footballer), http://dbpedia.org/resource/Miguel_Ángel_Maroto, http://dbpedia.org/resource/Miguel_Ángel_Prieto, http://dbpedia.org/resource/Montserat_Esquerdo, http://dbpedia.org/resource/Olga_Martín, http://dbpedia.org/resource/Ramiro_Bravo, http://dbpedia.org/resource/Raúl_Peinador, http://dbpedia.org/resource/Rita_Oraá, http://dbpedia.org/resource/Rocío_Ruiz, http://dbpedia.org/resource/Rosa_María_Castillejo, http://dbpedia.org/resource/Virginia_Cardona, http://dbpedia.org/resource/Ángel_Alonso, http://dbpedia.org/resource/Manuel_Antonio, http://dbpedia.org/resource/Miguel_Ángel_Fernández_Ordóñez, http://dbpedia.org/resource/Carlos_Ruesga, http://dbpedia.org/resource/Cristian_Ugalde, http://dbpedia.org/resource/Daniel_Sarmiento_Melián, http://dbpedia.org/resource/Eduardo_Gurbindo, http://dbpedia.org/resource/Gonzalo_Pérez_de_Vargas, http://dbpedia.org/resource/Mikel_Aguirrezabalaga, http://dbpedia.org/resource/Pascual_Marquina_Narro, http://dbpedia.org/resource/Raúl_Entrerríos, http://dbpedia.org/resource/Valero_Rivera_López, http://dbpedia.org/resource/Víctor_Tomás, http://dbpedia.org/resource/Ángeles_Parejo, http://dbpedia.org/resource/Federico_Alonso, http://dbpedia.org/resource/Cristie_Schoen, http://dbpedia.org/resource/Jamaal_al-Din_M._Zarabozo, http://dbpedia.org/resource/Iván_Velasco, http://dbpedia.org/resource/Didac_Pérez, http://dbpedia.org/resource/Maverick_Viñales, http://dbpedia.org/resource/Sergio_Gadea, http://dbpedia.org/resource/Fulgencio_Coll_Bucher, http://dbpedia.org/resource/Conchita_Montes, http://dbpedia.org/resource/Lina_Morgan, http://dbpedia.org/resource/Aitor_López_Rekarte, http://dbpedia.org/resource/Antoñito, http://dbpedia.org/resource/Ciriaco_Errasti, http://dbpedia.org/resource/Emilio_José_Viqueira, http://dbpedia.org/resource/Lluis_Codina, http://dbpedia.org/resource/Roberto_Lombraña, http://dbpedia.org/resource/María_Cayetana_de_Silva,_13th_Duchess_of_Alba, http://dbpedia.org/resource/Diego_Barber, http://dbpedia.org/resource/Luis_Balague, http://dbpedia.org/resource/Rodrigo_de_la_Fuente, http://dbpedia.org/resource/Salvador_Cardona, http://dbpedia.org/resource/Tomás_Antonio_Sánchez, http://dbpedia.org/resource/María_Andrea_Casamayor, http://dbpedia.org/resource/Arturo_Fernández_Rodríguez, http://dbpedia.org/resource/Francisco_Aguilar_y_Leal, http://dbpedia.org/resource/Luis_Rubiales, http://dbpedia.org/resource/Ruth_Ndoumbe, http://dbpedia.org/resource/Xisco_Nadal, http://dbpedia.org/resource/Miguel_Ángel_Gómez_Martínez, http://dbpedia.org/resource/Joseph_Maria_Gordon, http://dbpedia.org/resource/José_Manuel_Martínez_Toral, http://dbpedia.org/resource/Mercedes_Brignone, http://dbpedia.org/resource/Manuel_Martínez_(fencer), http://dbpedia.org/resource/Manuela_Carmena, http://dbpedia.org/resource/Mariano_Díaz, http://dbpedia.org/resource/Roger_Casamajor, http://dbpedia.org/resource/Imanol_Erviti, http://dbpedia.org/resource/Javier_Dorado, http://dbpedia.org/resource/Ferran_Laviña, http://dbpedia.org/resource/Josep_Maria_Izquierdo, http://dbpedia.org/resource/Sito_Alonso, http://dbpedia.org/resource/Tom_Hernández, http://dbpedia.org/resource/David_Belenguer, http://dbpedia.org/resource/Gustavo_César, http://dbpedia.org/resource/Manuel_Jiménez_Abalo, http://dbpedia.org/resource/Pepe_Rubianes, http://dbpedia.org/resource/Francisco_Gómez-Jordana_Sousa, http://dbpedia.org/resource/José_Galisteo, http://dbpedia.org/resource/Andrew_Fernando_Holmes, http://dbpedia.org/resource/José_Luis_Carrasco, http://dbpedia.org/resource/Carlos_Barredo, http://dbpedia.org/resource/Rubén_Lobato, http://dbpedia.org/resource/Alfons_Alzamora, http://dbpedia.org/resource/Mariano_Bombarda, http://dbpedia.org/resource/Oriol_Junyent, http://dbpedia.org/resource/Carlos_Bellvís, http://dbpedia.org/resource/Jorge_Molino, http://dbpedia.org/resource/Leslie_Dewart, http://dbpedia.org/resource/Pablo_Santos_(tennis), http://dbpedia.org/resource/Antonio_Vera, http://dbpedia.org/resource/Dani_Bouzas, http://dbpedia.org/resource/Diego_Segura, http://dbpedia.org/resource/Fernando_Martín_Carreras, http://dbpedia.org/resource/Florinda_Chico, http://dbpedia.org/resource/Francisco_Javier_Castaño, http://dbpedia.org/resource/Iván_Candela, http://dbpedia.org/resource/Joaquín_Moso, http://dbpedia.org/resource/José_Francisco_Mañuz, http://dbpedia.org/resource/José_Javier_Rubio, http://dbpedia.org/resource/Kily_Álvarez, http://dbpedia.org/resource/Sergio_Sestelo, http://dbpedia.org/resource/Álex_Rubio, http://dbpedia.org/resource/Óscar_López_Martínez, http://dbpedia.org/resource/Adriana_Reverón, http://dbpedia.org/resource/Andrea_Huisgen, http://dbpedia.org/resource/Carla_García, http://dbpedia.org/resource/Claudia_Moro, http://dbpedia.org/resource/David_Karanka, http://dbpedia.org/resource/Desirée_Cordero_Ferrer, http://dbpedia.org/resource/Elisabeth_Reyes, http://dbpedia.org/resource/Estíbaliz_Pereira, http://dbpedia.org/resource/Natalia_Zabala, http://dbpedia.org/resource/Patricia_Yurena_Rodríguez, http://dbpedia.org/resource/Óscar_Faura, http://dbpedia.org/resource/Albert_Rivera, http://dbpedia.org/resource/Alberto_Losada, http://dbpedia.org/resource/Alberto_Marcos_Rey, http://dbpedia.org/resource/Alberto_Miguel, http://dbpedia.org/resource/Guillermo_Eizaguirre, http://dbpedia.org/resource/Rafael_Lesmes, http://dbpedia.org/resource/Antonio_María_Cascajares_y_Azara, http://dbpedia.org/resource/Esteban_Casagolda, http://dbpedia.org/resource/Mikel_Amantegui, http://dbpedia.org/resource/Mikel_Gaztañaga, http://dbpedia.org/resource/Mikel_Roteta, http://dbpedia.org/resource/Mikel_Álvaro, http://dbpedia.org/resource/Silvia_Marty, http://dbpedia.org/resource/Carlos_Albert, http://dbpedia.org/resource/Jordi_Ribera, http://dbpedia.org/resource/Emiliano_Álvarez, http://dbpedia.org/resource/Moncho_López, http://dbpedia.org/resource/Joel_González, http://dbpedia.org/resource/Mario_de_Sárraga, http://dbpedia.org/resource/Raúl_García_de_Mateos, http://dbpedia.org/resource/Óscar_García-Casarrubios, http://dbpedia.org/resource/Antonio_López_(footballer,_born_1980), http://dbpedia.org/resource/Antonio_Tomás, http://dbpedia.org/resource/Aritz_Solabarrieta, http://dbpedia.org/resource/Endika_Bordas, http://dbpedia.org/resource/Javi_Guerrero, http://dbpedia.org/resource/Javier_Calleja, http://dbpedia.org/resource/José_Antonio_Culebras, http://dbpedia.org/resource/Juan_Calatayud, http://dbpedia.org/resource/Juan_Manuel_Rosado, http://dbpedia.org/resource/Manuel_Blanco_Rodríguez, http://dbpedia.org/resource/Pablo_Casar_Bustillo, http://dbpedia.org/resource/Rubén_García_Arnal, http://dbpedia.org/resource/Sergio_G._Sánchez, http://dbpedia.org/resource/Toni_Moral, http://dbpedia.org/resource/Álex_Pérez, http://dbpedia.org/resource/Efrén_Vázquez, http://dbpedia.org/resource/Iván_Iglesias, http://dbpedia.org/resource/Jacobo_Campos, http://dbpedia.org/resource/Manuel_Canabal, http://dbpedia.org/resource/Sergio_Pachón, http://dbpedia.org/resource/David_García_Dapena, http://dbpedia.org/resource/Josep_Franch, http://dbpedia.org/resource/José_Simeón, http://dbpedia.org/resource/Adrián_Palomares, http://dbpedia.org/resource/Alejandro_Marque, http://dbpedia.org/resource/Arkaitz_Durán, http://dbpedia.org/resource/Diego_Rubio_(cyclist), http://dbpedia.org/resource/Araceli_Navarro, http://dbpedia.org/resource/Begoña_Fernández, http://dbpedia.org/resource/Carmen_Martín, http://dbpedia.org/resource/Joaquim_Maria_Puyal, http://dbpedia.org/resource/Macarena_Aguilar, http://dbpedia.org/resource/Nely_Carla_Alberto, http://dbpedia.org/resource/Nuria_Benzal, http://dbpedia.org/resource/Kirian_Rodríguez_Ledesma, http://dbpedia.org/resource/Iñaki_Bollaín, http://dbpedia.org/resource/Marcos_Gondra_Krug, http://dbpedia.org/resource/Óscar_García_Guerrero, http://dbpedia.org/resource/Carlos_Castro_Caputo, http://dbpedia.org/resource/Felipe_Vaqueriza, http://dbpedia.org/resource/Miguel_Ángel_Carrilero, http://dbpedia.org/resource/Francisco_Llorente_Gento, http://dbpedia.org/resource/Jaime_Jornet, http://dbpedia.org/resource/Jesús_Enrique_Velasco, http://dbpedia.org/resource/José_Luis_Ablanedo, http://dbpedia.org/resource/Juan_José_Pereira_Zambrano, http://dbpedia.org/resource/Sergio_Hinestrosa, http://dbpedia.org/resource/Unai_Alba, http://dbpedia.org/resource/Àlex_Pascual, http://dbpedia.org/resource/Javier_Artero, http://dbpedia.org/resource/José_Manuel_Roca_Cases, http://dbpedia.org/resource/Felisa_Mary, http://dbpedia.org/resource/Alfonso_Gutiérrez, http://dbpedia.org/resource/Biel_Medina, http://dbpedia.org/resource/Eduardo_Conget, http://dbpedia.org/resource/Fernando_Delgado, http://dbpedia.org/resource/Javier_Herreros, http://dbpedia.org/resource/Javier_Oliete, http://dbpedia.org/resource/José_Vivó, http://dbpedia.org/resource/Blanca_de_Silos, http://dbpedia.org/resource/Carmen_Viance, http://dbpedia.org/resource/Isabel_de_Pomés, http://dbpedia.org/resource/Javier_Modrego, http://dbpedia.org/resource/Luis_Hurtado, http://dbpedia.org/resource/Mercedes_Vecino, http://dbpedia.org/resource/Rosita_Yarza, http://dbpedia.org/resource/Guillermo_Pérez_Moreno, http://dbpedia.org/resource/Iván_Malón, http://dbpedia.org/resource/Walter_Fernández, http://dbpedia.org/resource/Ángel_Atienza, http://dbpedia.org/resource/Iñaki_Muñoz, http://dbpedia.org/resource/Carmelo_Yuste_Yuste, http://dbpedia.org/resource/José_Herrada, http://dbpedia.org/resource/Kiko_Ratón, http://dbpedia.org/resource/Cindy_Lima, http://dbpedia.org/resource/Kike_Mateo, http://dbpedia.org/resource/Ana_Miranda_Paz, http://dbpedia.org/resource/Carmen_Romero_López, http://dbpedia.org/resource/Enrique_Guerrero_Salom, http://dbpedia.org/resource/Josefa_Andrés_Barea, http://dbpedia.org/resource/Ricardo_Cortés_Lastra, http://dbpedia.org/resource/Sergio_Gutiérrez_Prieto, http://dbpedia.org/resource/José_Antonio_Plaza, http://dbpedia.org/resource/Emilio_Gutiérrez_González, http://dbpedia.org/resource/Alejo_Sauras, http://dbpedia.org/resource/Daniel_Mangrané, http://dbpedia.org/resource/Isabel_Garcés, http://dbpedia.org/resource/Jesús_Garay_(director), http://dbpedia.org/resource/Mario_Berriatúa, http://dbpedia.org/resource/Marisa_de_Leza, http://dbpedia.org/resource/Iván_Peñaranda, http://dbpedia.org/resource/Clara_Espar, http://dbpedia.org/resource/Judith_Forca, http://dbpedia.org/resource/Víctor_Gomis, http://dbpedia.org/resource/Rubén_Jurado, http://dbpedia.org/resource/Miguel_de_Miguel, http://dbpedia.org/resource/Pablo_Barrachina_Estevan, http://dbpedia.org/resource/Cristina_Castaño, http://dbpedia.org/resource/Jaume_Rovira_(cyclist), http://dbpedia.org/resource/Clemente_Fernández_López, http://dbpedia.org/resource/Federico_Uruburu, http://dbpedia.org/resource/Francisco_Calvet, http://dbpedia.org/resource/Elisabet_Sadó_Garriga, http://dbpedia.org/resource/Jony_López, http://dbpedia.org/resource/Juanma_(footballer,_born_1977), http://dbpedia.org/resource/Manuel_Gato, http://dbpedia.org/resource/Clemente_Gràcia, http://dbpedia.org/resource/Antonio_Abadía, http://dbpedia.org/resource/Juan_José_de_los_Ángeles, http://dbpedia.org/resource/José_Ángel_Gascón, http://dbpedia.org/resource/Sarai_Gascón_Moreno, http://dbpedia.org/resource/Javi_Sánchez, http://dbpedia.org/resource/Eleuterio_Maisonnave_y_Cutayar, http://dbpedia.org/resource/Emilio_Alcalá-Galiano,_4th_Count_of_Casa_Valencia, http://dbpedia.org/resource/José_Félix_de_Lequerica_y_Erquiza, http://dbpedia.org/resource/Zuriñe_Rodríguez, http://dbpedia.org/resource/Diego_Andrés_Bermúdez, http://dbpedia.org/resource/Celso_Albelo, http://dbpedia.org/resource/Ruth_Vega_Fernandez, http://dbpedia.org/resource/Roberto_Cairo, http://dbpedia.org/resource/Elixabete_Capa, http://dbpedia.org/resource/Marta_Moreno, http://dbpedia.org/resource/Patricia_Pinedo, http://dbpedia.org/resource/Julio_Vallejo_Ruiloba, http://dbpedia.org/resource/Manuel_Herrero_Maestre, http://dbpedia.org/resource/Francisco_Jesús_Ferrón_Ruiz, http://dbpedia.org/resource/Manuel_Sáez, http://dbpedia.org/resource/Àngel_Fabregat, http://dbpedia.org/resource/Esther_Peris, http://dbpedia.org/resource/Irene_Villa, http://dbpedia.org/resource/Nathalie_Carpanedo, http://dbpedia.org/resource/Teresa_Silva, http://dbpedia.org/resource/Julian_Bado, http://dbpedia.org/resource/Sergio_Campano_Franco, http://dbpedia.org/resource/Bartolomé_González_de_Villaverde, http://dbpedia.org/resource/Blas_de_Peralta, http://dbpedia.org/resource/Domingo_Ortiz_de_Rozas_y_Rodillo, http://dbpedia.org/resource/Manuel_Garnica_Roldán, http://dbpedia.org/resource/Jaime_Vilamajó, http://dbpedia.org/resource/Josep_Clarós, http://dbpedia.org/resource/Juan_Jusdado, http://dbpedia.org/resource/Manuel_Guijarro_Doménech, http://dbpedia.org/resource/Joseph_de_Agulló_y_Pinos, http://dbpedia.org/resource/Diego_Rodriguez_de_Valdés, http://dbpedia.org/resource/Gonzalo_Casco, http://dbpedia.org/resource/Alonso_de_Escobar_y_Cáceres, http://dbpedia.org/resource/Ana_Galán, http://dbpedia.org/resource/Julieta_Valero, http://dbpedia.org/resource/Martín_Suárez_de_Toledo, http://dbpedia.org/resource/Toti_Martínez_de_Lezea, http://dbpedia.org/resource/Alizzz, http://dbpedia.org/resource/Marisa_Gonzalez, http://dbpedia.org/resource/Miguel_Salis, http://dbpedia.org/resource/Eduardo_Arranz-Bravo, http://dbpedia.org/resource/Mario_Gimeno, http://dbpedia.org/resource/Amparo_Cuevas, http://dbpedia.org/resource/Ángel_Sertucha, http://dbpedia.org/resource/Frank_Cuesta, http://dbpedia.org/resource/Manuel_Aller, http://dbpedia.org/resource/Alonso_de_Ojeda, http://dbpedia.org/resource/Anne_of_Austria, http://dbpedia.org/resource/Antoni_Clavé, http://dbpedia.org/resource/Antonio_Pica, http://dbpedia.org/resource/Bernardo_de_Gálvez, http://dbpedia.org/resource/Candida_Maria_of_Jesus, http://dbpedia.org/resource/Eduardo_Noriega_(Spanish_actor), http://dbpedia.org/resource/Edurne_Pasaban, http://dbpedia.org/resource/Emma_Penella, http://dbpedia.org/resource/Enrique_Iglesias, http://dbpedia.org/resource/Gil_Parrondo, http://dbpedia.org/resource/Gonzalo_Jiménez_de_Quesada, http://dbpedia.org/resource/Hazel_Ann_Mendoza, http://dbpedia.org/resource/Jesús_Huerta_de_Soto, http://dbpedia.org/resource/Josep_Maria_Benet_i_Jornet, http://dbpedia.org/resource/Kane_(wrestler), http://dbpedia.org/resource/Kiko_(Spanish_footballer), http://dbpedia.org/resource/Lope_de_Vega, http://dbpedia.org/resource/Lucía_Sánchez_Saornil, http://dbpedia.org/resource/Luis_Buñuel, http://dbpedia.org/resource/Manuel_Valls, http://dbpedia.org/resource/Marc_Oliveras, http://dbpedia.org/resource/Miguel_Illescas, http://dbpedia.org/resource/Miguel_Poblet, http://dbpedia.org/resource/Pedro_Almodóvar, http://dbpedia.org/resource/Pedro_Delgado, http://dbpedia.org/resource/Raül_López, http://dbpedia.org/resource/Rosa_Maria_Sardà, http://dbpedia.org/resource/Ruth_Beitia, http://dbpedia.org/resource/Saint_Dominic, http://dbpedia.org/resource/Salvador_de_Sá, http://dbpedia.org/resource/Vicente_Aleixandre, http://dbpedia.org/resource/Álex_Abrines, http://dbpedia.org/resource/Álex_González_(actor), http://dbpedia.org/resource/Álex_Palou, http://dbpedia.org/resource/Alejandra_Andreu, http://dbpedia.org/resource/Antonio_Moreno, http://dbpedia.org/resource/Fermin, http://dbpedia.org/resource/Francisco_Fernández_Ochoa, http://dbpedia.org/resource/Francisco_de_Paula_Martínez_de_la_Rosa_y_Berdejo, http://dbpedia.org/resource/Iván_Campo, http://dbpedia.org/resource/José_Manuel_Castañón, http://dbpedia.org/resource/José_María_Cano, http://dbpedia.org/resource/Julio_Iglesias,_Jr., http://dbpedia.org/resource/Luis_Aragonés, http://dbpedia.org/resource/Marc_Márquez, http://dbpedia.org/resource/Mark_Russinovich, http://dbpedia.org/resource/Najwa_Nimri, http://dbpedia.org/resource/Pedro_Sánchez_(Spanish_politician), http://dbpedia.org/resource/Pío_Baroja, http://dbpedia.org/resource/Roberto_Heras, http://dbpedia.org/resource/Rosalía_de_Castro, http://dbpedia.org/resource/Salvador_Espriu, http://dbpedia.org/resource/Samuel_Sánchez, http://dbpedia.org/resource/Óscar_Freire, http://dbpedia.org/resource/Agustí_Villaronga, http://dbpedia.org/resource/Aketza_Peña, http://dbpedia.org/resource/Ana_Obregón, http://dbpedia.org/resource/Francisco_Fernández_de_la_Cueva,_8th_Duke_of_Alburquerque, http://dbpedia.org/resource/Francisco_José_Pacheco, http://dbpedia.org/resource/Félix_Rodríguez_de_la_Fuente, http://dbpedia.org/resource/Gaizka_Mendieta, http://dbpedia.org/resource/Gonzalo_Anes, http://dbpedia.org/resource/Igor_Astarloa, http://dbpedia.org/resource/Jaime_Alguersuari, http://dbpedia.org/resource/Joan_Horrach, http://dbpedia.org/resource/José_Araquistáin, http://dbpedia.org/resource/Juan_José_Imbroda, http://dbpedia.org/resource/Max_Cortés, http://dbpedia.org/resource/Miguel_Delibes, http://dbpedia.org/resource/Miguel_Morayta, http://dbpedia.org/resource/Monica_Vera, http://dbpedia.org/resource/Míchel_(footballer), http://dbpedia.org/resource/Natalia_Sánchez_(actress), http://dbpedia.org/resource/Pablo_de_Sarasate, http://dbpedia.org/resource/Paquirri, http://dbpedia.org/resource/Salvador_of_Horta, http://dbpedia.org/resource/Santiago_Herrero_Amor, http://dbpedia.org/resource/Wally_Szczerbiak, http://dbpedia.org/resource/Fernando_Trueba, http://dbpedia.org/resource/Jose_de_Creeft, http://dbpedia.org/resource/José_Villalonga_Llorente, http://dbpedia.org/resource/Kiko_Casilla, http://dbpedia.org/resource/Pablo_Serrano, http://dbpedia.org/resource/Segundo_de_Chomón, http://dbpedia.org/resource/Soraya_Sáenz_de_Santamaría, http://dbpedia.org/resource/Antonio_León_Ortega, http://dbpedia.org/resource/Bartolomé_Bermejo, http://dbpedia.org/resource/Blanca_Suárez, http://dbpedia.org/resource/Carmelo_Larrea, http://dbpedia.org/resource/Crisant_Bosch, http://dbpedia.org/resource/Elena_Romero, http://dbpedia.org/resource/Emma_Cohen, http://dbpedia.org/resource/Francisco_Fernández_Ordóñez, http://dbpedia.org/resource/Gaspar_Núñez_de_Arce, http://dbpedia.org/resource/Isidre_Nonell, http://dbpedia.org/resource/Ismael_Merlo, http://dbpedia.org/resource/Joan_Fuster, http://dbpedia.org/resource/José_Canalejas, http://dbpedia.org/resource/José_Fernando_de_Abascal_y_Sousa, http://dbpedia.org/resource/José_Gómez_Ortega, http://dbpedia.org/resource/José_María_Cristiá, http://dbpedia.org/resource/Juan_de_Orduña, http://dbpedia.org/resource/Manuel_Iradier, http://dbpedia.org/resource/Manuel_Summers, http://dbpedia.org/resource/Manuel_de_la_Cámara_y_Libermoore, http://dbpedia.org/resource/Marina_Baura, http://dbpedia.org/resource/Martín_Marculeta, http://dbpedia.org/resource/Miguel_Cabrera_Cabrera, http://dbpedia.org/resource/Mireia_Lalaguna, http://dbpedia.org/resource/Pedro_Cortina_y_Mauri, http://dbpedia.org/resource/Ramón_de_la_Fuente_Leal, http://dbpedia.org/resource/Álex_Márquez, http://dbpedia.org/resource/Alonso_de_Orozco_Mena, http://dbpedia.org/resource/Alonso_de_Sotomayor, http://dbpedia.org/resource/Ana_de_Mendoza,_Princess_of_Eboli, http://dbpedia.org/resource/Antonio_de_Oquendo, http://dbpedia.org/resource/Augusto_Algueró, http://dbpedia.org/resource/Carlos_Fitz-James_Stuart,_4th_Duke_of_Liria_and_Jérica, http://dbpedia.org/resource/Cristobal_of_Saint_Catherine, http://dbpedia.org/resource/Diego_Carrillo_de_Mendoza,_1st_Marquis_of_Gelves, http://dbpedia.org/resource/Francisco_Cuervo_y_Valdés, http://dbpedia.org/resource/Francisco_Ignacio_Alcina, http://dbpedia.org/resource/Ignacio_Ellacuría, http://dbpedia.org/resource/Infante_Enrique,_Duke_of_Seville, http://dbpedia.org/resource/Jacobo_Fitz-James_Stuart,_17th_Duke_of_Alba, http://dbpedia.org/resource/Joaquín_Canaves, http://dbpedia.org/resource/Josefa_de_Óbidos, http://dbpedia.org/resource/Juan_Carreño_de_Miranda, http://dbpedia.org/resource/Mary_of_Jesus_of_Ágreda, http://dbpedia.org/resource/Pedro_Romero, http://dbpedia.org/resource/Tomás_Marín_de_Poveda,_1st_Marquis_of_Cañada_Hermosa, http://dbpedia.org/resource/Tomás_de_Torrejón_y_Velasco, http://dbpedia.org/resource/Tomás_de_la_Cerda,_3rd_Marquis_of_la_Laguna, http://dbpedia.org/resource/Úrsula_Micaela_Morata, http://dbpedia.org/resource/Ágatha_Ruiz_de_la_Prada, http://dbpedia.org/resource/José_Parra_Martínez, http://dbpedia.org/resource/Manuel_Antonio_Flores, http://dbpedia.org/resource/Patrick_Roscoe, http://dbpedia.org/resource/Andrés_Nin, http://dbpedia.org/resource/Esteban_Fernández_de_Castro, http://dbpedia.org/resource/Francisco_Javier_Illán_Vivas, http://dbpedia.org/resource/Martín_García_Óñez_de_Loyola, http://dbpedia.org/resource/Pedro_Núñez_de_Guzmán, http://dbpedia.org/resource/Don_José_Vidal, http://dbpedia.org/resource/Fabri_Salcedo, http://dbpedia.org/resource/Antonio_Ricardos, http://dbpedia.org/resource/Antonio_de_la_Cruz, http://dbpedia.org/resource/Carla_Antonelli, http://dbpedia.org/resource/Francisco_Javier_Venegas, http://dbpedia.org/resource/Franz_Joseph,_Marquis_de_Lusignan, http://dbpedia.org/resource/Juan_Magan, http://dbpedia.org/resource/Manuel_la_Peña, http://dbpedia.org/resource/José_Olaguer_Feliú, http://dbpedia.org/resource/Juan_Fernando_López_Aguilar, http://dbpedia.org/resource/Leticia_Romero, http://dbpedia.org/resource/Nicolás_Estévanez, http://dbpedia.org/resource/Sven_Giegold, http://dbpedia.org/resource/Antoni_Ribas, http://dbpedia.org/resource/Arly_Jover, http://dbpedia.org/resource/Fulgencio_Argüelles, http://dbpedia.org/resource/Justo_Pérez_de_Urbel, http://dbpedia.org/resource/Margarita_Sierra, http://dbpedia.org/resource/Victor_Balaguer_(singer), http://dbpedia.org/resource/Carmen_Balcells, http://dbpedia.org/resource/Rafael_Rullán, http://dbpedia.org/resource/Patricia_Urquiola, http://dbpedia.org/resource/Cristina_Galbó, http://dbpedia.org/resource/Antonio_Mateu_Lahoz, http://dbpedia.org/resource/Luis_Ricardo_Falero, http://dbpedia.org/resource/José_Luis_Rubiera, http://dbpedia.org/resource/Juan_Albert_Viloca, http://dbpedia.org/resource/Manuel_Mota_(fashion_designer), http://dbpedia.org/resource/Pablo_Iglesias_Turrión, http://dbpedia.org/resource/Constança_Capdeville, http://dbpedia.org/resource/Consuelo_Díez, http://dbpedia.org/resource/Luisa_Casagemas, http://dbpedia.org/resource/Maria_Teresa_Pelegrí_i_Marimón, http://dbpedia.org/resource/María_Luisa_Ozaita, http://dbpedia.org/resource/María_Teresa_Prieto, http://dbpedia.org/resource/Narcisa_Freixas, http://dbpedia.org/resource/Simone_Lovell, http://dbpedia.org/resource/Soledad_Bengoecha_de_Cármena, http://dbpedia.org/resource/Teresa_Borràs_i_Fornell, http://dbpedia.org/resource/Zulema_de_la_Cruz, http://dbpedia.org/resource/Fernando_Luna, http://dbpedia.org/resource/José_Ramón_Bauzà, http://dbpedia.org/resource/Pedro_Chirino, http://dbpedia.org/resource/Salvador_Freixedo, http://dbpedia.org/resource/Julio_Llorente, http://dbpedia.org/resource/Michèle_Tabarot, http://dbpedia.org/resource/Andoni_Ortuzar, http://dbpedia.org/resource/Beltrán_de_la_Cueva,_1st_Duke_of_Alburquerque, http://dbpedia.org/resource/Emilio_Eiroa, http://dbpedia.org/resource/Gerardo_Vera, http://dbpedia.org/resource/Josu_Jon_Imaz, http://dbpedia.org/resource/José_Antonio_Ardanza, http://dbpedia.org/resource/Said_al-Andalusi, http://dbpedia.org/resource/Adam_Karol_Czartoryski, http://dbpedia.org/resource/José_Mari, http://dbpedia.org/resource/Francisco_de_Paula_Marín, http://dbpedia.org/resource/Michelle_Jenner, http://dbpedia.org/resource/Emiliano_Rodríguez, http://dbpedia.org/resource/Francisco_Ruiz_de_Castro, http://dbpedia.org/resource/José_Antonio_Escuredo, http://dbpedia.org/resource/Sergi_Escobar, http://dbpedia.org/resource/Alberto_Cifuentes_Martínez, http://dbpedia.org/resource/César_Cruchaga, http://dbpedia.org/resource/Dani_Estrada, http://dbpedia.org/resource/Domingo_Larrainzar, http://dbpedia.org/resource/Enrique_Fernando_Ortiz_Moruno, http://dbpedia.org/resource/Fabriciano_González, http://dbpedia.org/resource/Francisco_Puñal, http://dbpedia.org/resource/Gregorio_Benito, http://dbpedia.org/resource/Iñigo_Larrainzar, http://dbpedia.org/resource/Javi_Venta, http://dbpedia.org/resource/Javier_Casquero, http://dbpedia.org/resource/Javier_Castañeda, http://dbpedia.org/resource/Jonathan_Sesma, http://dbpedia.org/resource/Josetxo, http://dbpedia.org/resource/Josu_Uribe, http://dbpedia.org/resource/José_Izquierdo_(footballer,_born_1980), http://dbpedia.org/resource/José_Mejías, http://dbpedia.org/resource/Manuel_Echezarreta, http://dbpedia.org/resource/Marcelino_Pérez, http://dbpedia.org/resource/Martín_Domínguez, http://dbpedia.org/resource/Miguel_Albiol, http://dbpedia.org/resource/Miguel_Ángel_Sánchez_Muñoz, http://dbpedia.org/resource/Paco_Jémez, http://dbpedia.org/resource/Pepín_(footballer,_born_1959), http://dbpedia.org/resource/Quique_Álvarez, http://dbpedia.org/resource/Raúl_López_Gutiérrez, http://dbpedia.org/resource/Roberto_Peragón, http://dbpedia.org/resource/Salvador_Mejías, http://dbpedia.org/resource/Sara_Monforte, http://dbpedia.org/resource/Ángel_Martín_González, http://dbpedia.org/resource/Úrsula_Corberó, http://dbpedia.org/resource/Jordi_Murphy, http://dbpedia.org/resource/Ramón_Martínez_(fencer), http://dbpedia.org/resource/Mireia_Miró_Varela, http://dbpedia.org/resource/Diego_Guzmán_de_Silva, http://dbpedia.org/resource/José_Antonio_Zaldúa, http://dbpedia.org/resource/Pedro_Pi, http://dbpedia.org/resource/Beñat_San_José, http://dbpedia.org/resource/Félix_Suárez, http://dbpedia.org/resource/Nuria_Bermúdez, http://dbpedia.org/resource/Jaime_Asensio_de_la_Fuente, http://dbpedia.org/resource/Hernando_Martel, http://dbpedia.org/resource/Leopoldo_Centeno, http://dbpedia.org/resource/Luis_Pérez_Pascual, http://dbpedia.org/resource/Roberto_Losada, http://dbpedia.org/resource/Manuel_Mesa, http://dbpedia.org/resource/Vicente_López_y_Portaña, http://dbpedia.org/resource/Ernest_Corominas, http://dbpedia.org/resource/Félix_Ruiz, http://dbpedia.org/resource/Jesús_Esperanza, http://dbpedia.org/resource/Joaquín_Alonso, http://dbpedia.org/resource/Manuel_Zúñiga, http://dbpedia.org/resource/Miguel_Roca, http://dbpedia.org/resource/Urbano_Ortega, http://dbpedia.org/resource/Cristina_Bes_Ginesta, http://dbpedia.org/resource/Jordi_Bes_Ginesta, http://dbpedia.org/resource/José_María_Jiménez, http://dbpedia.org/resource/Pépito_Pavon, http://dbpedia.org/resource/Ramiro_Figueiras_Amarelle, http://dbpedia.org/resource/Alicia_Sánchez-Camacho, http://dbpedia.org/resource/Amadeu_Vives_i_Roig, http://dbpedia.org/resource/Cristian_Malmagro, http://dbpedia.org/resource/Gedeón_Guardiola, http://dbpedia.org/resource/Juan_de_Sanabria, http://dbpedia.org/resource/Pablo_Puyol, http://dbpedia.org/resource/Ángel_del_Pozo, http://dbpedia.org/resource/Amets_Txurruka, http://dbpedia.org/resource/Iñigo_González_de_Heredia, http://dbpedia.org/resource/Javier_Moreno_(cyclist), http://dbpedia.org/resource/Jon_Izagirre, http://dbpedia.org/resource/Jonathan_Castroviejo, http://dbpedia.org/resource/Josu_Larrazabal, http://dbpedia.org/resource/Juan_José_Lobato, http://dbpedia.org/resource/Mikel_Nieve, http://dbpedia.org/resource/Pablo_Urtasun, http://dbpedia.org/resource/Rubén_Pérez, http://dbpedia.org/resource/Gerardo_Herrero, http://dbpedia.org/resource/Itxaro_Mentxaka, http://dbpedia.org/resource/Adolfo_Moyano_Burgos, http://dbpedia.org/resource/Alberto_Romea, http://dbpedia.org/resource/Cala_(footballer,_born_1990), http://dbpedia.org/resource/Lolita_Sevilla, http://dbpedia.org/resource/Manolo_Morán, http://dbpedia.org/resource/Josep_Prades_i_Gallent, http://dbpedia.org/resource/Ibon_Areso, http://dbpedia.org/resource/Diego_Ordóñez, http://dbpedia.org/resource/Luis_Fernández_Gutiérrez, http://dbpedia.org/resource/Alex_Barnett, http://dbpedia.org/resource/Adela_Cantalapiedra, http://dbpedia.org/resource/Ana_Ibáñez_Llorente, http://dbpedia.org/resource/Felipe_Mellizo, http://dbpedia.org/resource/Jesús_Álvarez_(journalist), http://dbpedia.org/resource/Julia_Varela, http://dbpedia.org/resource/Manuel_Toharia, http://dbpedia.org/resource/Rodolfo_Chikilicuatre, http://dbpedia.org/resource/Ángela_Rodicio, http://dbpedia.org/resource/Sergio_Valdeolmillos, http://dbpedia.org/resource/Fernando_Zunzunegui, http://dbpedia.org/resource/Jay_Santos, http://dbpedia.org/resource/Manuel_Bueno, http://dbpedia.org/resource/Federico_Galera_Díez, http://dbpedia.org/resource/Guillermo_Roldán, http://dbpedia.org/resource/José_Álvarez_de_Toledo,_Duke_of_Alba, http://dbpedia.org/resource/Marta_Rivera_de_la_Cruz, http://dbpedia.org/resource/Gabriel_Moya, http://dbpedia.org/resource/Ángel_Vivar_Dorado, http://dbpedia.org/resource/Pedro_Afán_de_Ribera, http://dbpedia.org/resource/Santiago_Formoso, http://dbpedia.org/resource/Curro_Montoya, http://dbpedia.org/resource/César_Láinez, http://dbpedia.org/resource/Juan_Redondo, http://dbpedia.org/resource/Antonio_Trevín, http://dbpedia.org/resource/Juan_Luis_Rodríguez-Vigil, http://dbpedia.org/resource/Rafael_Fernández_Álvarez, http://dbpedia.org/resource/Raúl_Cabanas, http://dbpedia.org/resource/Txemi_Urtasun, http://dbpedia.org/resource/Francisco_Javier_Jusué, http://dbpedia.org/resource/José_Higinio_Gómez_González, http://dbpedia.org/resource/Luis_Venegas, http://dbpedia.org/resource/Moncho_Fernández, http://dbpedia.org/resource/Raúl_Cancio, http://dbpedia.org/resource/Vicente_Parra, http://dbpedia.org/resource/Álex_Lombardero, http://dbpedia.org/resource/Francisco_Pérez_de_Prado_y_Cuesta, http://dbpedia.org/resource/Esteve_Rabat, http://dbpedia.org/resource/Infanta_Maria_Cristina_of_Spain_(1833–1902), http://dbpedia.org/resource/Iván_Maestro, http://dbpedia.org/resource/Iván_Silva, http://dbpedia.org/resource/José_David_de_Gea, http://dbpedia.org/resource/Diego_Mainz, http://dbpedia.org/resource/José_Mingorance, http://dbpedia.org/resource/Juan_Cruz_Ochoa, http://dbpedia.org/resource/Juan_Uslé, http://dbpedia.org/resource/Ricardo_Fargas_Roger, http://dbpedia.org/resource/Maria_Antonia_Paris, http://dbpedia.org/resource/Francisco_Maspons_y_Labrós, http://dbpedia.org/resource/José_Moré_Bonet, http://dbpedia.org/resource/Victor_Alonso, http://dbpedia.org/resource/Carlos_Pérez_García, http://dbpedia.org/resource/Koke_(footballer,_born_1983), http://dbpedia.org/resource/Daniel_Zueras, http://dbpedia.org/resource/Antonio_de_Benavides, http://dbpedia.org/resource/Hernando_de_Miranda, http://dbpedia.org/resource/Juan_Treviño_de_Guillamas, http://dbpedia.org/resource/Pablo_de_Hita_y_Salazar, http://dbpedia.org/resource/Pedro_de_Aranda_y_Avellaneda, http://dbpedia.org/resource/Rodrigo_del_Junco, http://dbpedia.org/resource/Albert_Tomas, http://dbpedia.org/resource/Diego_Díaz_Garrido, http://dbpedia.org/resource/Gregorio_Fonseca, http://dbpedia.org/resource/Javier_Falagán, http://dbpedia.org/resource/José_Lamiel, http://dbpedia.org/resource/José_Luis_Santamaría, http://dbpedia.org/resource/José_Manuel_Galdames, http://dbpedia.org/resource/Juan_Antonio_Chesa, http://dbpedia.org/resource/Pablo_Maqueda, http://dbpedia.org/resource/Santi_Cuesta, http://dbpedia.org/resource/Xabier_Eskurza, http://dbpedia.org/resource/Óscar_Tabuenka, http://dbpedia.org/resource/Albert_Cabestany, http://dbpedia.org/resource/Francisco_José_Pérez, http://dbpedia.org/resource/Alberto_Fernández_Sainz, http://dbpedia.org/resource/Daniel_Espejo_Guillén, http://dbpedia.org/resource/Delio_Fernández, http://dbpedia.org/resource/Gustavo_Domínguez, http://dbpedia.org/resource/Jesús_Blanco_Villar, http://dbpedia.org/resource/Ignacio_Pinazo_Camarlench, http://dbpedia.org/resource/Diana_Lorys, http://dbpedia.org/resource/Domingo_Ortega, http://dbpedia.org/resource/José_de_Yanguas,_11th_Viscount_of_Santa_Clara_de_Avedillo, http://dbpedia.org/resource/Juan_José_Lerena_y_Barry, http://dbpedia.org/resource/Marc_Vidal_(chef), http://dbpedia.org/resource/Carlos_Areces, http://dbpedia.org/resource/Elsa_Pinilla_Osuna, http://dbpedia.org/resource/Joaquín_Reyes_(actor), http://dbpedia.org/resource/Albert_Marrama, http://dbpedia.org/resource/Jacobo_Ynclán, http://dbpedia.org/resource/Víctor_Unamuno, http://dbpedia.org/resource/Sara_Mérida, http://dbpedia.org/resource/Joaquín_Piña_Batllevell, http://dbpedia.org/resource/Paul_Abasolo, http://dbpedia.org/resource/Vicent_Andrés_Estellés, http://dbpedia.org/resource/Diego_Flores, http://dbpedia.org/resource/Ángela_Pumariega, http://dbpedia.org/resource/Ana_Petra_Pérez_Florido, http://dbpedia.org/resource/Josefa_Naval_Girbés, http://dbpedia.org/resource/Manuel_Míguez_González, http://dbpedia.org/resource/María_Josefa_Sancho_de_Guerra, http://dbpedia.org/resource/Paula_Montal_Fornés, http://dbpedia.org/resource/Alberto_Saavedra, http://dbpedia.org/resource/Jesús_Ramón_Martínez_de_Ezquerecocha_Suso, http://dbpedia.org/resource/Román_Montañez, http://dbpedia.org/resource/José_Ignacio_Aizpurúa, http://dbpedia.org/resource/Asier_Salcedo, http://dbpedia.org/resource/David_Cabarcos, http://dbpedia.org/resource/Juan_Ramón_Ruano, http://dbpedia.org/resource/Ximo_Enguix, http://dbpedia.org/resource/Óscar_Fernández_(manager), http://dbpedia.org/resource/José_Manuel_Monsalve]... [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 937, P: 0.507, R: 0.563, macro F1: 0.515, macro F1*: 0.533 -[INFO] Current time: 9681 ms, average: 2895.35 ms/ques, entity detection: 499.29 ms/ques (linking tool: 405.54 ms/ques), relation detection: 938.10 ms/ques (similarity: 707.45 ms/ques), EDG generation: 46.58 ms/ques +[INFO] Cumulative metrics, sample: 937, P: 0.507, R: 0.563, macro F1: 0.533 +[INFO] Current question consumed time: 8270 ms, average: 2586.79 ms/ques, entity detection: 436.15 ms/ques (linking tool: 358.38 ms/ques), relation detection: 861.14 ms/ques (similarity: 700.84 ms/ques), EDG generation: 47.08 ms/ques [INFO] Entity Linking P: 0.7406616862326574, R: 0.692102454642476, micro F1: 0.707755247242974, macro F1: 0.7155591858798931 [INFO] Relation Linking P: 0.6007648523657061, R: 0.5458911419423693, micro F1: 0.5619352543578805, macro F1: 0.5720149947755688 [INFO] Type Linking P: 0.5592315901814301, R: 0.5592315901814301, micro F1: 0.5592315901814301, macro F1: 0.5592315901814301 @@ -13150,8 +13150,8 @@ Question 937: Does AMC Gremlin have the Volkswagen engine? [INFO] Golden Answer: [true] [INFO] Predicted Answer: [true] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 938, P: 0.507, R: 0.563, macro F1: 0.515, macro F1*: 0.534 -[INFO] Current time: 1096 ms, average: 2893.43 ms/ques, entity detection: 499.08 ms/ques (linking tool: 405.33 ms/ques), relation detection: 937.28 ms/ques (similarity: 706.80 ms/ques), EDG generation: 46.54 ms/ques +[INFO] Cumulative metrics, sample: 938, P: 0.507, R: 0.563, macro F1: 0.534 +[INFO] Current question consumed time: 931 ms, average: 2585.02 ms/ques, entity detection: 435.98 ms/ques (linking tool: 358.18 ms/ques), relation detection: 860.37 ms/ques (similarity: 700.19 ms/ques), EDG generation: 47.04 ms/ques [INFO] Entity Linking P: 0.7398720682302772, R: 0.69136460554371, micro F1: 0.707000710732054, macro F1: 0.7147963296049681 [INFO] Relation Linking P: 0.6001243781094527, R: 0.5453091684434968, micro F1: 0.5613361762615501, macro F1: 0.5714051706873219 [INFO] Type Linking P: 0.5597014925373134, R: 0.5597014925373134, micro F1: 0.5597014925373134, macro F1: 0.5597014925373134 @@ -13164,8 +13164,8 @@ Question 938: Name the movie with narrator as Trevor Peacock and music composer [INFO] Golden Answer: [http://dbpedia.org/resource/Fred_Claus] [INFO] Predicted Answer: [http://dbpedia.org/resource/Fred_Claus] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 939, P: 0.508, R: 0.564, macro F1: 0.516, macro F1*: 0.534 -[INFO] Current time: 1965 ms, average: 2892.44 ms/ques, entity detection: 499.04 ms/ques (linking tool: 405.25 ms/ques), relation detection: 936.81 ms/ques (similarity: 706.62 ms/ques), EDG generation: 46.54 ms/ques +[INFO] Cumulative metrics, sample: 939, P: 0.508, R: 0.564, macro F1: 0.534 +[INFO] Current question consumed time: 1692 ms, average: 2584.07 ms/ques, entity detection: 435.91 ms/ques (linking tool: 358.09 ms/ques), relation detection: 859.92 ms/ques (similarity: 700.02 ms/ques), EDG generation: 47.05 ms/ques [INFO] Entity Linking P: 0.7401490947816827, R: 0.6916932907348243, micro F1: 0.707312744053958, macro F1: 0.7151012823513617 [INFO] Relation Linking P: 0.6000177493787717, R: 0.5452609158679447, micro F1: 0.5612708555200575, macro F1: 0.5713303451659316 [INFO] Type Linking P: 0.5591054313099042, R: 0.5591054313099042, micro F1: 0.5591054313099042, macro F1: 0.5591054313099042 @@ -13178,8 +13178,8 @@ Question 939: Where did the scientist study whose doctoral thesis was supervised [INFO] Golden Answer: [http://dbpedia.org/resource/Vanderbilt_University, http://dbpedia.org/resource/University_of_Colorado_Boulder] [INFO] Predicted Answer: [http://dbpedia.org/resource/Tom_Maniatis] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 940, P: 0.507, R: 0.563, macro F1: 0.515, macro F1*: 0.534 -[INFO] Current time: 868 ms, average: 2890.29 ms/ques, entity detection: 498.73 ms/ques (linking tool: 404.92 ms/ques), relation detection: 936.00 ms/ques (similarity: 706.05 ms/ques), EDG generation: 46.52 ms/ques +[INFO] Cumulative metrics, sample: 940, P: 0.507, R: 0.563, macro F1: 0.534 +[INFO] Current question consumed time: 725 ms, average: 2582.09 ms/ques, entity detection: 435.60 ms/ques (linking tool: 357.79 ms/ques), relation detection: 859.19 ms/ques (similarity: 699.46 ms/ques), EDG generation: 47.04 ms/ques [INFO] Entity Linking P: 0.7404255319148936, R: 0.6920212765957446, micro F1: 0.7076241134751773, macro F1: 0.715405582637413 [INFO] Relation Linking P: 0.6004432624113475, R: 0.5452127659574468, micro F1: 0.5613829787234049, macro F1: 0.5714967211684264 [INFO] Type Linking P: 0.5585106382978723, R: 0.5585106382978723, micro F1: 0.5585106382978723, macro F1: 0.5585106382978723 @@ -13192,8 +13192,8 @@ Question 940: Where can one find some quick bread in the world? [INFO] Golden Answer: [http://dbpedia.org/resource/New_York_City, http://dbpedia.org/resource/North_America, http://dbpedia.org/resource/Shetland] [INFO] Predicted Answer: [http://dbpedia.org/resource/United_States] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 941, P: 0.507, R: 0.563, macro F1: 0.515, macro F1*: 0.533 -[INFO] Current time: 676 ms, average: 2887.93 ms/ques, entity detection: 498.32 ms/ques (linking tool: 404.55 ms/ques), relation detection: 935.11 ms/ques (similarity: 705.36 ms/ques), EDG generation: 46.49 ms/ques +[INFO] Cumulative metrics, sample: 941, P: 0.507, R: 0.563, macro F1: 0.533 +[INFO] Current question consumed time: 539 ms, average: 2579.92 ms/ques, entity detection: 435.24 ms/ques (linking tool: 357.45 ms/ques), relation detection: 858.37 ms/ques (similarity: 698.78 ms/ques), EDG generation: 47.00 ms/ques [INFO] Entity Linking P: 0.740701381509033, R: 0.6923485653560042, micro F1: 0.7079348211122919, macro F1: 0.71570923255934 [INFO] Relation Linking P: 0.5998051718030464, R: 0.5446333687566419, micro F1: 0.5607863974495224, macro F1: 0.5708893920279712 [INFO] Type Linking P: 0.5589798087141339, R: 0.5589798087141339, micro F1: 0.5589798087141339, macro F1: 0.5589798087141339 @@ -13206,8 +13206,8 @@ Question 941: What is the sea whose cities are Gdask and Oulu? [INFO] Golden Answer: [http://dbpedia.org/resource/Baltic_Sea] [INFO] Predicted Answer: [http://dbpedia.org/resource/Oulun_Kärpät, http://dbpedia.org/resource/Oulun_Lyseon_Lukio, http://dbpedia.org/resource/University_of_Oulu, http://dbpedia.org/resource/Frans_Michael_Franzén_(sculpture), http://dbpedia.org/resource/Oulu_City_Theatre, http://dbpedia.org/resource/Oulu_International_School, http://dbpedia.org/resource/Oulu_University_of_Applied_Sciences, http://dbpedia.org/resource/Finnish_national_road_4, http://dbpedia.org/resource/Oulu_Airport, http://dbpedia.org/resource/Oulun_Suomalaisen_Yhteiskoulun_Lukio, http://dbpedia.org/resource/Diaconia_University_of_Applied_Sciences, http://dbpedia.org/resource/Oulu_Market_Hall] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 942, P: 0.506, R: 0.562, macro F1: 0.514, macro F1*: 0.533 -[INFO] Current time: 4046 ms, average: 2889.16 ms/ques, entity detection: 498.13 ms/ques (linking tool: 404.34 ms/ques), relation detection: 935.54 ms/ques (similarity: 707.00 ms/ques), EDG generation: 46.46 ms/ques +[INFO] Cumulative metrics, sample: 942, P: 0.506, R: 0.562, macro F1: 0.533 +[INFO] Current question consumed time: 3619 ms, average: 2581.03 ms/ques, entity detection: 435.04 ms/ques (linking tool: 357.25 ms/ques), relation detection: 858.74 ms/ques (similarity: 700.41 ms/ques), EDG generation: 46.98 ms/ques [INFO] Entity Linking P: 0.7409766454352441, R: 0.692144373673036, micro F1: 0.7078910120311394, macro F1: 0.7157285523315248 [INFO] Relation Linking P: 0.5991684359518754, R: 0.5440552016985138, micro F1: 0.5601910828025484, macro F1: 0.5702833523336739 [INFO] Type Linking P: 0.5583864118895966, R: 0.5583864118895966, micro F1: 0.5583864118895966, macro F1: 0.5583864118895966 @@ -13220,8 +13220,8 @@ Question 942: What rivers originate from Australian Alps? [INFO] Golden Answer: [http://dbpedia.org/resource/Murray_River] [INFO] Predicted Answer: [http://dbpedia.org/resource/Snowy_River, http://dbpedia.org/resource/Adelong_Creek, http://dbpedia.org/resource/Happy_Jacks_Creek, http://dbpedia.org/resource/Rock_Flat_Creek] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 943, P: 0.506, R: 0.562, macro F1: 0.514, macro F1*: 0.532 -[INFO] Current time: 1267 ms, average: 2887.44 ms/ques, entity detection: 497.79 ms/ques (linking tool: 404.03 ms/ques), relation detection: 934.90 ms/ques (similarity: 706.60 ms/ques), EDG generation: 46.42 ms/ques +[INFO] Cumulative metrics, sample: 943, P: 0.506, R: 0.562, macro F1: 0.532 +[INFO] Current question consumed time: 1043 ms, average: 2579.40 ms/ques, entity detection: 434.73 ms/ques (linking tool: 356.97 ms/ques), relation detection: 858.13 ms/ques (similarity: 700.02 ms/ques), EDG generation: 46.94 ms/ques [INFO] Entity Linking P: 0.7412513255567338, R: 0.6924708377518558, micro F1: 0.7082007776599505, macro F1: 0.7160312360777324 [INFO] Relation Linking P: 0.5985330505478967, R: 0.5434782608695652, micro F1: 0.5595970307529169, macro F1: 0.5696785979833731 [INFO] Type Linking P: 0.5577942735949099, R: 0.5577942735949099, micro F1: 0.5577942735949099, macro F1: 0.5577942735949099 @@ -13234,8 +13234,8 @@ Question 943: Which venues are located at places governed by John Roberts? [INFO] Golden Answer: [http://dbpedia.org/resource/Tempe_Center_for_the_Arts, http://dbpedia.org/resource/Marquee_Theatre, http://dbpedia.org/resource/Hynes_Convention_Center, http://dbpedia.org/resource/Celebrity_Theatre, http://dbpedia.org/resource/Comerica_Theatre, http://dbpedia.org/resource/Phoenix_Symphony_Hall] [INFO] Predicted Answer: [http://dbpedia.org/resource/Titan_Tower_(Fisher_Towers), http://dbpedia.org/resource/Whiteface_Mountain, http://dbpedia.org/resource/Devils_Tower, http://dbpedia.org/resource/Mount_Minsi, http://dbpedia.org/resource/Mount_Wilson_(Nevada), http://dbpedia.org/resource/Moose_Mountain_(Alaska), http://dbpedia.org/resource/Chickies_Ridge, http://dbpedia.org/resource/Mount_Sunapee, http://dbpedia.org/resource/Haycock_Mountain, http://dbpedia.org/resource/Quartz_Mountain, http://dbpedia.org/resource/Valles_Caldera, http://dbpedia.org/resource/Middle_Cathedral_Rock, http://dbpedia.org/resource/Tahquitz_Peak, http://dbpedia.org/resource/Cerro_San_Luis_Obispo, http://dbpedia.org/resource/Lost_Arrow_Spire, http://dbpedia.org/resource/Cerro_Pedernal, http://dbpedia.org/resource/El_Capitan, http://dbpedia.org/resource/Cannon_Mountain_(New_Hampshire), http://dbpedia.org/resource/Lover's_Leap_(Tahoe,_California), http://dbpedia.org/resource/Charlotte_Dome] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 944, P: 0.505, R: 0.561, macro F1: 0.513, macro F1*: 0.532 -[INFO] Current time: 33129 ms, average: 2919.48 ms/ques, entity detection: 497.61 ms/ques (linking tool: 403.77 ms/ques), relation detection: 951.66 ms/ques (similarity: 711.97 ms/ques), EDG generation: 46.39 ms/ques +[INFO] Cumulative metrics, sample: 944, P: 0.505, R: 0.561, macro F1: 0.532 +[INFO] Current question consumed time: 27574 ms, average: 2605.87 ms/ques, entity detection: 434.53 ms/ques (linking tool: 356.73 ms/ques), relation detection: 871.37 ms/ques (similarity: 705.33 ms/ques), EDG generation: 46.91 ms/ques [INFO] Entity Linking P: 0.7415254237288136, R: 0.6927966101694916, micro F1: 0.7085098870056497, macro F1: 0.7163332749167564 [INFO] Relation Linking P: 0.597899011299435, R: 0.5429025423728814, micro F1: 0.5590042372881362, macro F1: 0.569075124892289 [INFO] Type Linking P: 0.5572033898305084, R: 0.5572033898305084, micro F1: 0.5572033898305084, macro F1: 0.5572033898305084 @@ -13248,8 +13248,8 @@ Question 944: What is the venue of San Diego Comic-Con International ? [INFO] Golden Answer: [http://dbpedia.org/resource/San_Diego, http://dbpedia.org/resource/San_Diego_Convention_Center, http://dbpedia.org/resource/Downtown] [INFO] Predicted Answer: [http://dbpedia.org/resource/Carne_asada_fries, http://dbpedia.org/resource/Sharp_Memorial_Hospital, http://dbpedia.org/resource/UC_San_Diego_Health, http://dbpedia.org/resource/Scripps_Mercy_Hospital] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 945, P: 0.505, R: 0.560, macro F1: 0.513, macro F1*: 0.531 -[INFO] Current time: 5267 ms, average: 2921.96 ms/ques, entity detection: 497.90 ms/ques (linking tool: 404.05 ms/ques), relation detection: 953.09 ms/ques (similarity: 712.46 ms/ques), EDG generation: 46.36 ms/ques +[INFO] Cumulative metrics, sample: 945, P: 0.505, R: 0.560, macro F1: 0.531 +[INFO] Current question consumed time: 4636 ms, average: 2608.02 ms/ques, entity detection: 434.79 ms/ques (linking tool: 356.99 ms/ques), relation detection: 872.69 ms/ques (similarity: 705.82 ms/ques), EDG generation: 46.88 ms/ques [INFO] Entity Linking P: 0.7417989417989418, R: 0.6931216931216931, micro F1: 0.7088183421516754, macro F1: 0.7166346709119571 [INFO] Relation Linking P: 0.5977954144620811, R: 0.5433862433862434, micro F1: 0.5591181657848331, macro F1: 0.5692937707928821 [INFO] Type Linking P: 0.5576719576719577, R: 0.5576719576719577, micro F1: 0.5576719576719577, macro F1: 0.5576719576719577 @@ -13262,8 +13262,8 @@ Question 945: Which office holder owes its allegiance to a state whose capital t [INFO] Golden Answer: [http://dbpedia.org/resource/James_Madison] [INFO] Predicted Answer: [http://dbpedia.org/resource/George_B._Mowad, http://dbpedia.org/resource/Aldo_Tatangelo, http://dbpedia.org/resource/George_W._Hardy,_Jr., http://dbpedia.org/resource/Kenneth_E._BeLieu, http://dbpedia.org/resource/Industrial_Workers_of_the_World, http://dbpedia.org/resource/Floyd_Smith_(Louisiana_politician), http://dbpedia.org/resource/Fred_Baden, http://dbpedia.org/resource/J.E._%22Pat%22_Patterson, http://dbpedia.org/resource/Robert_T._Tobin, http://dbpedia.org/resource/Sam_Caldwell, http://dbpedia.org/resource/Thaddeus_Beal, http://dbpedia.org/resource/International_Longshore_and_Warehouse_Union, http://dbpedia.org/resource/United_Automobile_Workers, http://dbpedia.org/resource/Joseph_A._Doyle, http://dbpedia.org/resource/NANDA, http://dbpedia.org/resource/P._Elmo_Futrell,_Jr.] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 946, P: 0.504, R: 0.560, macro F1: 0.512, macro F1*: 0.530 -[INFO] Current time: 29941 ms, average: 2950.52 ms/ques, entity detection: 498.23 ms/ques (linking tool: 404.28 ms/ques), relation detection: 968.03 ms/ques (similarity: 714.78 ms/ques), EDG generation: 46.37 ms/ques +[INFO] Cumulative metrics, sample: 946, P: 0.504, R: 0.560, macro F1: 0.530 +[INFO] Current question consumed time: 26145 ms, average: 2632.90 ms/ques, entity detection: 435.09 ms/ques (linking tool: 357.23 ms/ques), relation detection: 885.53 ms/ques (similarity: 708.14 ms/ques), EDG generation: 46.88 ms/ques [INFO] Entity Linking P: 0.741014799154334, R: 0.6923890063424947, micro F1: 0.7080690627202254, macro F1: 0.7158771289765321 [INFO] Relation Linking P: 0.5971634954193094, R: 0.5428118393234672, micro F1: 0.5585271317829463, macro F1: 0.5686919803374986 [INFO] Type Linking P: 0.5570824524312896, R: 0.5570824524312896, micro F1: 0.5570824524312896, macro F1: 0.5570824524312896 @@ -13276,8 +13276,8 @@ Question 946: Which professional fighter, who is also a politician and was elect [INFO] Golden Answer: [http://dbpedia.org/resource/Manny_Pacquiao] [INFO] Predicted Answer: [http://dbpedia.org/resource/Corazon_Aquino] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 947, P: 0.503, R: 0.559, macro F1: 0.512, macro F1*: 0.530 -[INFO] Current time: 5600 ms, average: 2953.32 ms/ques, entity detection: 498.60 ms/ques (linking tool: 404.69 ms/ques), relation detection: 968.49 ms/ques (similarity: 716.29 ms/ques), EDG generation: 46.45 ms/ques +[INFO] Cumulative metrics, sample: 947, P: 0.503, R: 0.559, macro F1: 0.530 +[INFO] Current question consumed time: 5080 ms, average: 2635.49 ms/ques, entity detection: 435.44 ms/ques (linking tool: 357.61 ms/ques), relation detection: 885.97 ms/ques (similarity: 709.67 ms/ques), EDG generation: 46.95 ms/ques [INFO] Entity Linking P: 0.7402323125659979, R: 0.6916578669482577, micro F1: 0.707321365716297, macro F1: 0.7151211869184789 [INFO] Relation Linking P: 0.5965329109468497, R: 0.5422386483632524, micro F1: 0.5579373460049285, macro F1: 0.5680914608228866 [INFO] Type Linking P: 0.5564941921858501, R: 0.5564941921858501, micro F1: 0.5564941921858501, macro F1: 0.5564941921858501 @@ -13290,8 +13290,8 @@ Question 947: List the comic characters created by Joe Shuster ? [INFO] Golden Answer: [http://dbpedia.org/resource/Lara_(comics), http://dbpedia.org/resource/Lois_Lane, http://dbpedia.org/resource/Perry_White, http://dbpedia.org/resource/Superboy_(Kal-El), http://dbpedia.org/resource/Doctor_Occult, http://dbpedia.org/resource/Mister_Mxyzptlk, http://dbpedia.org/resource/Rose_Psychic, http://dbpedia.org/resource/Superman, http://dbpedia.org/resource/Delores_Winters, http://dbpedia.org/resource/Clark_Kent, http://dbpedia.org/resource/Lex_Luthor, http://dbpedia.org/resource/George_Taylor_(DC_Comics)] [INFO] Predicted Answer: [http://dbpedia.org/resource/Lara_(comics), http://dbpedia.org/resource/Lois_Lane, http://dbpedia.org/resource/Perry_White, http://dbpedia.org/resource/Superboy_(Kal-El), http://dbpedia.org/resource/Doctor_Occult, http://dbpedia.org/resource/Mister_Mxyzptlk, http://dbpedia.org/resource/Rose_Psychic, http://dbpedia.org/resource/Superman, http://dbpedia.org/resource/Delores_Winters, http://dbpedia.org/resource/Clark_Kent, http://dbpedia.org/resource/Lex_Luthor, http://dbpedia.org/resource/George_Taylor_(DC_Comics)] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 948, P: 0.504, R: 0.560, macro F1: 0.512, macro F1*: 0.530 -[INFO] Current time: 2283 ms, average: 2952.61 ms/ques, entity detection: 498.27 ms/ques (linking tool: 404.38 ms/ques), relation detection: 967.88 ms/ques (similarity: 715.96 ms/ques), EDG generation: 46.42 ms/ques +[INFO] Cumulative metrics, sample: 948, P: 0.504, R: 0.560, macro F1: 0.530 +[INFO] Current question consumed time: 1093 ms, average: 2633.86 ms/ques, entity detection: 435.14 ms/ques (linking tool: 357.33 ms/ques), relation detection: 885.40 ms/ques (similarity: 709.35 ms/ques), EDG generation: 46.91 ms/ques [INFO] Entity Linking P: 0.740506329113924, R: 0.6919831223628692, micro F1: 0.7076300984528832, macro F1: 0.7154229041218471 [INFO] Relation Linking P: 0.5969585091420534, R: 0.5427215189873418, micro F1: 0.5584036568213789, macro F1: 0.5685494539827286 [INFO] Type Linking P: 0.5559071729957806, R: 0.5559071729957806, micro F1: 0.5559071729957806, macro F1: 0.5559071729957806 @@ -13304,8 +13304,8 @@ Question 948: Which kind of conventions are held in Rosemont, Illinois? [INFO] Golden Answer: [Anime, Japanese Culture, Video Gaming@en, http://dbpedia.org/resource/Furry_fandom, http://dbpedia.org/resource/Doctor_Who, http://dbpedia.org/resource/Science_fiction_on_television, Multi-genre@en, Horror@en] [INFO] Predicted Answer: [http://dbpedia.org/resource/Santa_Anita_Handicap, http://dbpedia.org/resource/Withers_Stakes, http://dbpedia.org/resource/San_Antonio_Handicap, http://dbpedia.org/resource/Narragansett_Special, Eastern Shore Handicap@en] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 949, P: 0.503, R: 0.559, macro F1: 0.512, macro F1*: 0.530 -[INFO] Current time: 3684 ms, average: 2953.39 ms/ques, entity detection: 498.73 ms/ques (linking tool: 404.74 ms/ques), relation detection: 967.40 ms/ques (similarity: 715.77 ms/ques), EDG generation: 46.38 ms/ques +[INFO] Cumulative metrics, sample: 949, P: 0.503, R: 0.559, macro F1: 0.530 +[INFO] Current question consumed time: 3389 ms, average: 2634.65 ms/ques, entity detection: 435.59 ms/ques (linking tool: 357.70 ms/ques), relation detection: 884.98 ms/ques (similarity: 709.16 ms/ques), EDG generation: 46.88 ms/ques [INFO] Entity Linking P: 0.7397260273972602, R: 0.6912539515279241, micro F1: 0.706884439761152, macro F1: 0.7146690338329937 [INFO] Relation Linking P: 0.5963294696171408, R: 0.5421496311907271, micro F1: 0.5578152441166145, macro F1: 0.567950350237752 [INFO] Type Linking P: 0.5553213909378293, R: 0.5553213909378293, micro F1: 0.5553213909378293, macro F1: 0.5553213909378293 @@ -13318,8 +13318,8 @@ Question 949: What city is located on the mouth of Duwamish River? [INFO] Golden Answer: [http://dbpedia.org/resource/Seattle] [INFO] Predicted Answer: [http://dbpedia.org/resource/Auburn,_Washington, http://dbpedia.org/resource/Kent,_Washington, http://dbpedia.org/resource/Tukwila,_Washington] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 950, P: 0.503, R: 0.558, macro F1: 0.511, macro F1*: 0.529 -[INFO] Current time: 2648 ms, average: 2953.06 ms/ques, entity detection: 498.39 ms/ques (linking tool: 404.42 ms/ques), relation detection: 967.28 ms/ques (similarity: 715.83 ms/ques), EDG generation: 46.42 ms/ques +[INFO] Cumulative metrics, sample: 950, P: 0.503, R: 0.558, macro F1: 0.529 +[INFO] Current question consumed time: 2305 ms, average: 2634.31 ms/ques, entity detection: 435.27 ms/ques (linking tool: 357.41 ms/ques), relation detection: 884.86 ms/ques (similarity: 709.22 ms/ques), EDG generation: 46.92 ms/ques [INFO] Entity Linking P: 0.74, R: 0.6915789473684211, micro F1: 0.7071929824561403, macro F1: 0.714970588235294 [INFO] Relation Linking P: 0.5957017543859648, R: 0.541578947368421, micro F1: 0.5572280701754392, macro F1: 0.5673525077638176 [INFO] Type Linking P: 0.5547368421052632, R: 0.5547368421052632, micro F1: 0.5547368421052632, macro F1: 0.5547368421052632 @@ -13332,8 +13332,8 @@ Question 950: Which governer of Winston Bryant is the president of Joseph Stigli [INFO] Golden Answer: [http://dbpedia.org/resource/Bill_Clinton] [INFO] Predicted Answer: [http://dbpedia.org/resource/Bill_Clinton] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 951, P: 0.503, R: 0.559, macro F1: 0.512, macro F1*: 0.530 -[INFO] Current time: 1867 ms, average: 2951.92 ms/ques, entity detection: 498.34 ms/ques (linking tool: 404.32 ms/ques), relation detection: 966.88 ms/ques (similarity: 715.45 ms/ques), EDG generation: 46.40 ms/ques +[INFO] Cumulative metrics, sample: 951, P: 0.503, R: 0.559, macro F1: 0.530 +[INFO] Current question consumed time: 1594 ms, average: 2633.21 ms/ques, entity detection: 435.17 ms/ques (linking tool: 357.27 ms/ques), relation detection: 884.52 ms/ques (similarity: 708.85 ms/ques), EDG generation: 46.90 ms/ques [INFO] Entity Linking P: 0.7402733964248159, R: 0.6919032597266036, micro F1: 0.7075008762705923, macro F1: 0.7152715049156079 [INFO] Relation Linking P: 0.5961268839817735, R: 0.5420609884332281, micro F1: 0.5576936558009119, macro F1: 0.5678098243608054 [INFO] Type Linking P: 0.555205047318612, R: 0.555205047318612, micro F1: 0.555205047318612, macro F1: 0.555205047318612 @@ -13346,8 +13346,8 @@ Question 951: How many people have led agencies in German occupied Europe? [INFO] Golden Answer: [18] [INFO] Predicted Answer: [22] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 952, P: 0.503, R: 0.558, macro F1: 0.511, macro F1*: 0.529 -[INFO] Current time: 2219 ms, average: 2951.15 ms/ques, entity detection: 497.92 ms/ques (linking tool: 403.94 ms/ques), relation detection: 966.18 ms/ques (similarity: 716.07 ms/ques), EDG generation: 46.36 ms/ques +[INFO] Cumulative metrics, sample: 952, P: 0.503, R: 0.558, macro F1: 0.529 +[INFO] Current question consumed time: 1073 ms, average: 2631.57 ms/ques, entity detection: 434.81 ms/ques (linking tool: 356.94 ms/ques), relation detection: 883.89 ms/ques (similarity: 708.43 ms/ques), EDG generation: 46.87 ms/ques [INFO] Entity Linking P: 0.740546218487395, R: 0.6922268907563025, micro F1: 0.7078081232492996, macro F1: 0.7155717858991103 [INFO] Relation Linking P: 0.5965511204481793, R: 0.542016806722689, micro F1: 0.5578081232493003, macro F1: 0.5679779407726803 [INFO] Type Linking P: 0.555672268907563, R: 0.555672268907563, micro F1: 0.555672268907563, macro F1: 0.555672268907563 @@ -13360,8 +13360,8 @@ Question 952: Where did the plays written by Robert Schenkkan take place? [INFO] Golden Answer: [http://dbpedia.org/resource/Oregon_Shakespeare_Festival, http://dbpedia.org/resource/Seattle, http://dbpedia.org/resource/Intiman_Theatre_Festival] [INFO] Predicted Answer: [http://dbpedia.org/resource/Oregon_Shakespeare_Festival, http://dbpedia.org/resource/Seattle, http://dbpedia.org/resource/Intiman_Theatre_Festival] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 953, P: 0.503, R: 0.559, macro F1: 0.511, macro F1*: 0.530 -[INFO] Current time: 1215 ms, average: 2949.33 ms/ques, entity detection: 497.59 ms/ques (linking tool: 403.62 ms/ques), relation detection: 965.59 ms/ques (similarity: 715.69 ms/ques), EDG generation: 46.33 ms/ques +[INFO] Cumulative metrics, sample: 953, P: 0.503, R: 0.559, macro F1: 0.530 +[INFO] Current question consumed time: 1097 ms, average: 2629.96 ms/ques, entity detection: 434.52 ms/ques (linking tool: 356.65 ms/ques), relation detection: 883.36 ms/ques (similarity: 708.06 ms/ques), EDG generation: 46.84 ms/ques [INFO] Entity Linking P: 0.7408184679958028, R: 0.6925498426023085, micro F1: 0.7081147254284714, macro F1: 0.7158714332023862 [INFO] Relation Linking P: 0.596974466596712, R: 0.5424973767051416, micro F1: 0.5582721231199725, macro F1: 0.5684336721304585 [INFO] Type Linking P: 0.5550891920251836, R: 0.5550891920251836, micro F1: 0.5550891920251836, macro F1: 0.5550891920251836 @@ -13374,8 +13374,8 @@ Question 953: Under which president did the predecessor of Simon Ferro work? [INFO] Golden Answer: [http://dbpedia.org/resource/Bill_Clinton] [INFO] Predicted Answer: [http://dbpedia.org/resource/Adam_Clarke, http://dbpedia.org/resource/Al_Rosen, http://dbpedia.org/resource/Alberto_Lleras_Camargo, http://dbpedia.org/resource/Alexandros_Zaimis, http://dbpedia.org/resource/Alfonso_López_Michelsen, http://dbpedia.org/resource/Aslan_Maskhadov, http://dbpedia.org/resource/Atal_Bihari_Vajpayee, http://dbpedia.org/resource/Barack_Obama_in_comics, http://dbpedia.org/resource/Barney_(dog), http://dbpedia.org/resource/Battle_of_Fallen_Timbers, http://dbpedia.org/resource/Belisario_Betancur, http://dbpedia.org/resource/Benjamin_Mkapa, http://dbpedia.org/resource/Bill_DeWitt, http://dbpedia.org/resource/Boyd_K._Packer, http://dbpedia.org/resource/Brandon_Tartikoff, http://dbpedia.org/resource/Carl_Gustaf_Emil_Mannerheim, http://dbpedia.org/resource/Carlos_Lleras_Restrepo, http://dbpedia.org/resource/Carmen_Policy, http://dbpedia.org/resource/Caroline_Kennedy, http://dbpedia.org/resource/Carthage_Palace, http://dbpedia.org/resource/Charles_L._McNary, http://dbpedia.org/resource/Charles_Phelps_Taft, http://dbpedia.org/resource/Charles_W._Fairbanks, http://dbpedia.org/resource/Charles_Weeghman, http://dbpedia.org/resource/Chen_Gongbo, http://dbpedia.org/resource/Chun_Doo-hwan, http://dbpedia.org/resource/Commandant, http://dbpedia.org/resource/Coolidge,_Arizona, http://dbpedia.org/resource/Coolidge_Dam, http://dbpedia.org/resource/Coolidge_Homestead, http://dbpedia.org/resource/Coolidge_effect, http://dbpedia.org/resource/Cristina_Fernández_de_Kirchner, http://dbpedia.org/resource/Cyril_Burt, http://dbpedia.org/resource/Cyrus_Adler, http://dbpedia.org/resource/Cyrus_Vance, http://dbpedia.org/resource/César_Gaviria, http://dbpedia.org/resource/Daniel_Ortega, http://dbpedia.org/resource/Danny_Morrison_(sports_executive), http://dbpedia.org/resource/David_O._McKay, http://dbpedia.org/resource/Donald_Broadbent, http://dbpedia.org/resource/Donald_Smith,_1st_Baron_Strathcona_and_Mount_Royal, http://dbpedia.org/resource/Dorothy_Bush_Koch, http://dbpedia.org/resource/Dorothy_Hodgkin, http://dbpedia.org/resource/Duff_Pattullo, http://dbpedia.org/resource/E._H._Moore, http://dbpedia.org/resource/Ed_Asner, http://dbpedia.org/resource/Ed_McCaskey, http://dbpedia.org/resource/Edwin_E._Moise, http://dbpedia.org/resource/Emmeline_B._Wells, http://dbpedia.org/resource/Eric_Temple_Bell, http://dbpedia.org/resource/Fall_of_Suharto, http://dbpedia.org/resource/Frank_B._Kellogg, http://dbpedia.org/resource/Franklin_Buchanan, http://dbpedia.org/resource/Frederic_Bartlett, http://dbpedia.org/resource/Gabe_Paul, http://dbpedia.org/resource/Gaullism, http://dbpedia.org/resource/George-Étienne_Cartier, http://dbpedia.org/resource/George_David_Birkhoff, http://dbpedia.org/resource/George_Halas,_Jr., http://dbpedia.org/resource/George_Stephen,_1st_Baron_Mount_Stephen, http://dbpedia.org/resource/Gordon_A._Craig, http://dbpedia.org/resource/Grace_Coolidge, http://dbpedia.org/resource/Hamada_Madi, http://dbpedia.org/resource/Harold_B._Lee, http://dbpedia.org/resource/Head_of_the_Chechen_Republic, http://dbpedia.org/resource/Heber_J._Grant, http://dbpedia.org/resource/Henry_John_Stephen_Smith, http://dbpedia.org/resource/Ignacio_Velázquez_Rivera, http://dbpedia.org/resource/Imelda_Marcos, http://dbpedia.org/resource/Ion_Iliescu, http://dbpedia.org/resource/Ismar_Schorsch, http://dbpedia.org/resource/James_A._Garfield, http://dbpedia.org/resource/James_Cagney, http://dbpedia.org/resource/Jerry_McMorris, http://dbpedia.org/resource/Jerzy_Neyman, http://dbpedia.org/resource/Joe_Banner, http://dbpedia.org/resource/John_Henry_Coatsworth, http://dbpedia.org/resource/John_Perry_(engineer), http://dbpedia.org/resource/John_William_Mackail, http://dbpedia.org/resource/Joseph-Adolphe_Chapleau, http://dbpedia.org/resource/Joseph_Fielding_Smith, http://dbpedia.org/resource/José_Manuel_Barroso, http://dbpedia.org/resource/José_María_Melo, http://dbpedia.org/resource/Juan_Bautista_Sacasa, http://dbpedia.org/resource/Juan_Jesús_Vivas, http://dbpedia.org/resource/Julio_César_Turbay_Ayala, http://dbpedia.org/resource/Kadyrovtsy, http://dbpedia.org/resource/Ken_Harrelson, http://dbpedia.org/resource/Kevin_Brennan_(politician), http://dbpedia.org/resource/Kim_Young-sam, http://dbpedia.org/resource/Kung_Te-cheng, http://dbpedia.org/resource/Larry_Dolan, http://dbpedia.org/resource/Lexx, http://dbpedia.org/resource/List_of_Presidents_of_Colombia, http://dbpedia.org/resource/List_of_heads_of_state_of_Spain, http://dbpedia.org/resource/Martín_Torrijos, http://dbpedia.org/resource/Mary_Robinson, http://dbpedia.org/resource/Maurice_Wilkes, http://dbpedia.org/resource/Michael_McCaskey, http://dbpedia.org/resource/Mike_Brown_(Cincinnati_Bengals_owner), http://dbpedia.org/resource/Mirek_Topolánek, http://dbpedia.org/resource/Misael_Pastrana_Borrero, http://dbpedia.org/resource/Myles_Cooper, http://dbpedia.org/resource/New_Life_Movement, http://dbpedia.org/resource/Nick_Mileti, http://dbpedia.org/resource/Nicola_Canali, http://dbpedia.org/resource/Nwafor_Orizu, http://dbpedia.org/resource/Ohio_City,_Cleveland, http://dbpedia.org/resource/Paul_Weiss_(philosopher), http://dbpedia.org/resource/Pierre_Juneau, http://dbpedia.org/resource/Plymouth,_Vermont, http://dbpedia.org/resource/President_of_Indonesia, http://dbpedia.org/resource/President_of_the_European_Commission, http://dbpedia.org/resource/President_of_the_Massachusetts_Senate, http://dbpedia.org/resource/Presidential_Unit_Citation_(United_States), http://dbpedia.org/resource/Quintin_Hogg,_Baron_Hailsham_of_St_Marylebone, http://dbpedia.org/resource/Rafael_Reyes, http://dbpedia.org/resource/Ramzan_Kadyrov, http://dbpedia.org/resource/Randy_Lerner, http://dbpedia.org/resource/Rapidan_Camp, http://dbpedia.org/resource/Richard_B._Angus, http://dbpedia.org/resource/Robert_Cummings_Neville, http://dbpedia.org/resource/Robert_Henry_Thurston, http://dbpedia.org/resource/Robert_May,_Baron_May_of_Oxford, http://dbpedia.org/resource/Robert_Schuman, http://dbpedia.org/resource/Salim_Ahmed_Salim, http://dbpedia.org/resource/Stephen_Stigler, http://dbpedia.org/resource/Suharto, http://dbpedia.org/resource/Sun_Fo, http://dbpedia.org/resource/Tang_Shaoyi, http://dbpedia.org/resource/Tarja_Halonen, http://dbpedia.org/resource/Ted_Phillips, http://dbpedia.org/resource/The_Autobiography_of_Calvin_Coolidge, http://dbpedia.org/resource/Theophilus_Painter, http://dbpedia.org/resource/Thomas_R._Marshall, http://dbpedia.org/resource/Timothy_Dwight_IV, http://dbpedia.org/resource/Tony_Clement, http://dbpedia.org/resource/United_Russia, http://dbpedia.org/resource/United_States_presidential_election,_1912, http://dbpedia.org/resource/Vaira_Vīķe-Freiberga, http://dbpedia.org/resource/Virgilio_Barco_Vargas, http://dbpedia.org/resource/Václav_Klaus, http://dbpedia.org/resource/Walter_Hallstein, http://dbpedia.org/resource/Walter_Pidgeon, http://dbpedia.org/resource/Wellington_Koo, http://dbpedia.org/resource/William_C._deMille, http://dbpedia.org/resource/William_Graham_Sumner, http://dbpedia.org/resource/William_Parsons,_3rd_Earl_of_Rosse, http://dbpedia.org/resource/Zhou_Ziqi, http://dbpedia.org/resource/Zviad_Gamsakhurdia, http://dbpedia.org/resource/Abdalá_Bucaram, http://dbpedia.org/resource/Abdel_Hakim_Amer, http://dbpedia.org/resource/Abdurrahman_Wahid, http://dbpedia.org/resource/Acting_President_of_the_United_States, http://dbpedia.org/resource/Acting_Vice_President_of_the_United_States, http://dbpedia.org/resource/Adam_Morton, http://dbpedia.org/resource/Adib_Shishakli, http://dbpedia.org/resource/Adolfo_Ruiz_Cortines, http://dbpedia.org/resource/Alberto_Dahik, http://dbpedia.org/resource/Alec_Douglas-Home, http://dbpedia.org/resource/Alfredo_Palacio, http://dbpedia.org/resource/Alfredo_Poveda, http://dbpedia.org/resource/Andris_Bērziņš_(Latvian_President), http://dbpedia.org/resource/Annette_Lu, http://dbpedia.org/resource/Anthony_Howard_(journalist), http://dbpedia.org/resource/Antonio_Flores_Jijón, http://dbpedia.org/resource/Arnold_Rüütel, http://dbpedia.org/resource/Arnoldo_Alemán, http://dbpedia.org/resource/Artur_da_Costa_e_Silva, http://dbpedia.org/resource/Asmara_President's_Office, http://dbpedia.org/resource/Assassination_of_Anwar_Sadat, http://dbpedia.org/resource/B._J._Vorster, http://dbpedia.org/resource/Charles_Brackett, http://dbpedia.org/resource/Charles_Léon_Hammes, http://dbpedia.org/resource/Charles_R._Larson, http://dbpedia.org/resource/Christiane_Herzog, http://dbpedia.org/resource/Columbian_mammoth, http://dbpedia.org/resource/D'Arcy_Wentworth_Thompson, http://dbpedia.org/resource/David_Truman, http://dbpedia.org/resource/Dennis_Archer, http://dbpedia.org/resource/Derringer, http://dbpedia.org/resource/Desmond_Hoyte, http://dbpedia.org/resource/Diego_Noboa, http://dbpedia.org/resource/Dobrica_Ćosić, http://dbpedia.org/resource/Domingo_Nieto, http://dbpedia.org/resource/Ebenezer_Cobb_Morley, http://dbpedia.org/resource/Ecuadorian_general_election,_2006, http://dbpedia.org/resource/Edmond_Privat, http://dbpedia.org/resource/Edward_Bernard_Raczyński, http://dbpedia.org/resource/Ellen_Johnson_Sirleaf, http://dbpedia.org/resource/Eloy_Alfaro, http://dbpedia.org/resource/Enrique_Bolaños, http://dbpedia.org/resource/Ernest_Cushing_Richardson, http://dbpedia.org/resource/Errol_Harris, http://dbpedia.org/resource/Execution_of_Saddam_Hussein, http://dbpedia.org/resource/Fabián_Alarcón, http://dbpedia.org/resource/Florentino_Pérez, http://dbpedia.org/resource/Francis_Lieber, http://dbpedia.org/resource/Francisco_Macías_Nguema, http://dbpedia.org/resource/Francisco_Urcuyo, http://dbpedia.org/resource/Freda_Meissner-Blau, http://dbpedia.org/resource/Frederick_William_Franz, http://dbpedia.org/resource/Gabriel_García_Moreno, http://dbpedia.org/resource/Galo_Plaza, http://dbpedia.org/resource/George_Clinton_(vice_president), http://dbpedia.org/resource/Government_Palace_(Peru), http://dbpedia.org/resource/Guadalupe_Victoria, http://dbpedia.org/resource/Guillermo_Rodríguez_(politician), http://dbpedia.org/resource/Hassan_Sheikh_Mohamud, http://dbpedia.org/resource/Henry_Pomeroy_Davison, http://dbpedia.org/resource/Herbert_York, http://dbpedia.org/resource/Humphrey_Tonkin, http://dbpedia.org/resource/Hussein_Farrah_Aidid, http://dbpedia.org/resource/Jacques_Viger_(1787–1858), http://dbpedia.org/resource/Jaime_Morales_Carazo, http://dbpedia.org/resource/Jaime_Roldós_Aguilera, http://dbpedia.org/resource/James_T._Aubrey, http://dbpedia.org/resource/Jamil_Mahuad, http://dbpedia.org/resource/Jan_Peter_Balkenende, http://dbpedia.org/resource/Jean-Claude_Juncker, http://dbpedia.org/resource/Jean-Claude_Trichet, http://dbpedia.org/resource/Jean_Doré, http://dbpedia.org/resource/Jean_Lesage, http://dbpedia.org/resource/Jean_Rey_(politician), http://dbpedia.org/resource/Jimmy_Hoffa, http://dbpedia.org/resource/Joan_Laporta, http://dbpedia.org/resource/Joe_Ghiz, http://dbpedia.org/resource/John_C._Wells, http://dbpedia.org/resource/John_F._Kennedy_Center_for_the_Performing_Arts, http://dbpedia.org/resource/John_Sandfield_Macdonald, http://dbpedia.org/resource/Jonathan_Dancy, http://dbpedia.org/resource/José_Joaquín_de_Olmedo, http://dbpedia.org/resource/José_María_Velasco_Ibarra, http://dbpedia.org/resource/Juan_José_Flores, http://dbpedia.org/resource/Julia_Grant, http://dbpedia.org/resource/Karl_Malden, http://dbpedia.org/resource/Karl_Renner, http://dbpedia.org/resource/Konstantin_Päts, http://dbpedia.org/resource/Lee_Myung-bak, http://dbpedia.org/resource/Lennart_Meri, http://dbpedia.org/resource/Lien_Chan, http://dbpedia.org/resource/List_of_Presidents_of_Egypt, http://dbpedia.org/resource/List_of_members_of_the_European_Court_of_Justice, http://dbpedia.org/resource/Livingston_Farrand, http://dbpedia.org/resource/Lucio_Gutiérrez, http://dbpedia.org/resource/Lucía_Hiriart, http://dbpedia.org/resource/Léopold_Sédar_Senghor, http://dbpedia.org/resource/Ma_Ying-jeou, http://dbpedia.org/resource/Manuel_de_Ascásubi, http://dbpedia.org/resource/Marc_Ravalomanana, http://dbpedia.org/resource/Martin_Schulz, http://dbpedia.org/resource/Marty_Meehan, http://dbpedia.org/resource/Meles_Zenawi, http://dbpedia.org/resource/Michael_Heseltine, http://dbpedia.org/resource/Mike_Ilitch, http://dbpedia.org/resource/Milton_J._Ferguson, http://dbpedia.org/resource/Mohamed_Hussein_Tantawi, http://dbpedia.org/resource/Mohamed_Khouna_Ould_Haidalla, http://dbpedia.org/resource/Mohammad_Reza_Aref, http://dbpedia.org/resource/Negasso_Gidada, http://dbpedia.org/resource/Omar_Suleiman, http://dbpedia.org/resource/Omar_Torrijos, http://dbpedia.org/resource/Osvaldo_Hurtado, http://dbpedia.org/resource/Park_Geun-hye, http://dbpedia.org/resource/Patrick_Bateson, http://dbpedia.org/resource/Patty_Duke, http://dbpedia.org/resource/Paul-Henri_Spaak, http://dbpedia.org/resource/Pedro_José_de_Arteta, http://dbpedia.org/resource/Pratibha_Patil, http://dbpedia.org/resource/President_of_El_Salvador, http://dbpedia.org/resource/President_of_Turkmenistan, http://dbpedia.org/resource/President_of_the_European_Central_Bank, http://dbpedia.org/resource/Presidential_palace, http://dbpedia.org/resource/Ramón_Calderón, http://dbpedia.org/resource/Ramón_Castilla, http://dbpedia.org/resource/Raúl_Alfonsín, http://dbpedia.org/resource/Religious_affiliations_of_Presidents_of_the_United_States, http://dbpedia.org/resource/Richard_Nixon_Foundation, http://dbpedia.org/resource/Robert_Lucas,_Jr., http://dbpedia.org/resource/Rodrigo_Borja_Cevallos, http://dbpedia.org/resource/Salvador_Sánchez_Cerén, http://dbpedia.org/resource/Saparmurat_Niyazov, http://dbpedia.org/resource/Sergio_Ramírez, http://dbpedia.org/resource/Sharad_Pawar, http://dbpedia.org/resource/Shukri_al-Quwatli, http://dbpedia.org/resource/Sidónio_Pais, http://dbpedia.org/resource/Sixto_Durán_Ballén, http://dbpedia.org/resource/Sjafruddin_Prawiranegara, http://dbpedia.org/resource/Song_Jin-woo_(journalist), http://dbpedia.org/resource/Sonia_Gandhi, http://dbpedia.org/resource/Steve_MacLean_(astronaut), http://dbpedia.org/resource/Sun_Yat-sen_University, http://dbpedia.org/resource/Taba_Summit, http://dbpedia.org/resource/Talcott_Parsons, http://dbpedia.org/resource/Teodoro_Obiang_Nguema_Mbasogo, http://dbpedia.org/resource/Timothy_Williamson, http://dbpedia.org/resource/Trygve_Haavelmo, http://dbpedia.org/resource/Tsakhiagiin_Elbegdorj, http://dbpedia.org/resource/Ugo_Mifsud_Bonnici, http://dbpedia.org/resource/Vice_President_of_Ecuador, http://dbpedia.org/resource/Vice_President_of_Iran, http://dbpedia.org/resource/Vicente_Boluda, http://dbpedia.org/resource/Vicente_Ramón_Roca, http://dbpedia.org/resource/Vicente_Rocafuerte, http://dbpedia.org/resource/Victoriano_Huerta, http://dbpedia.org/resource/Violence_Against_Women_Act, http://dbpedia.org/resource/William_Aberhart, http://dbpedia.org/resource/William_R._King, http://dbpedia.org/resource/Óscar_Humberto_Mejía_Victores, http://dbpedia.org/resource/107th_New_York_State_Legislature, http://dbpedia.org/resource/Aboubacar_Somparé, http://dbpedia.org/resource/Absolute_Power_(film), http://dbpedia.org/resource/Adlai_Stevenson_I, http://dbpedia.org/resource/Adolphus_William_Ward, http://dbpedia.org/resource/Ahmed_Hassan_al-Bakr, http://dbpedia.org/resource/Ahmed_Sékou_Touré, http://dbpedia.org/resource/Alan_G._Merten, http://dbpedia.org/resource/Alberto_Núñez_Feijóo, http://dbpedia.org/resource/Alejandro_M._Sinibaldi, http://dbpedia.org/resource/Alfonso_Portillo, http://dbpedia.org/resource/Alpha_Condé, http://dbpedia.org/resource/Alton_B._Parker, http://dbpedia.org/resource/Aristides_Pereira, http://dbpedia.org/resource/Arthur_Somers-Cocks,_6th_Baron_Somers, http://dbpedia.org/resource/Askar_Akayev, http://dbpedia.org/resource/Benjamin_Rush_Rhees, http://dbpedia.org/resource/Bernard_Bosanquet_(philosopher), http://dbpedia.org/resource/Bob_Harlan, http://dbpedia.org/resource/Carlos_Salazar_Castro, http://dbpedia.org/resource/Cellou_Dalein_Diallo, http://dbpedia.org/resource/Charles_Nicholson, http://dbpedia.org/resource/Cheddi_Jagan, http://dbpedia.org/resource/David_Cox_(statistician), http://dbpedia.org/resource/David_R._Francis, http://dbpedia.org/resource/Dirk_Ballendorf, http://dbpedia.org/resource/Doroteo_Vasconcelos, http://dbpedia.org/resource/Edward_Akufo-Addo, http://dbpedia.org/resource/Edward_Gent, http://dbpedia.org/resource/Edwin_Bidwell_Wilson, http://dbpedia.org/resource/Eshaq_Jahangiri, http://dbpedia.org/resource/Esther_Cleveland, http://dbpedia.org/resource/Everett_Hughes_(sociologist), http://dbpedia.org/resource/Family_of_Barack_Obama, http://dbpedia.org/resource/Fernando_Romeo_Lucas_García, http://dbpedia.org/resource/Frances_Folsom_Cleveland_Preston, http://dbpedia.org/resource/Francis_Lynde_Stetson, http://dbpedia.org/resource/Francisco_Ferrera, http://dbpedia.org/resource/Francisco_Javier_Arana, http://dbpedia.org/resource/Francisco_Morazán, http://dbpedia.org/resource/Francisco_Villagrán_Kramer, http://dbpedia.org/resource/Frederick_Abel, http://dbpedia.org/resource/Garfield_Barwick, http://dbpedia.org/resource/George_Blumenthal, http://dbpedia.org/resource/George_Papandreou, http://dbpedia.org/resource/Georgios_Christakis-Zografos, http://dbpedia.org/resource/Gerardo_Barrios, http://dbpedia.org/resource/Grant's_Tomb, http://dbpedia.org/resource/H._V._Evatt, http://dbpedia.org/resource/Harri_Holkeri, http://dbpedia.org/resource/Henrique_Rosa, http://dbpedia.org/resource/Jacob_Ziv, http://dbpedia.org/resource/Janet_Jagan, http://dbpedia.org/resource/Jimmy_&_Rosalynn_Carter_Work_Project, http://dbpedia.org/resource/John_Atta_Mills, http://dbpedia.org/resource/John_Leslie,_1st_Duke_of_Rothes, http://dbpedia.org/resource/John_Niemeyer_Findlay, http://dbpedia.org/resource/John_Phillips_(geologist), http://dbpedia.org/resource/José_Francisco_Barrundia, http://dbpedia.org/resource/José_María_Reina_Barrios, http://dbpedia.org/resource/José_Santos_Guardiola, http://dbpedia.org/resource/José_Trinidad_Cabañas, http://dbpedia.org/resource/Juan_Lindo, http://dbpedia.org/resource/Julio_César_Méndez_Montenegro, http://dbpedia.org/resource/Konstantinos_Tsatsos, http://dbpedia.org/resource/Kumba_Ialá, http://dbpedia.org/resource/Lansana_Conté, http://dbpedia.org/resource/Levi_P._Morton, http://dbpedia.org/resource/Lionel_Brett,_4th_Viscount_Esher, http://dbpedia.org/resource/List_of_Presidents_of_Guinea-Bissau, http://dbpedia.org/resource/List_of_Presidents_of_Iran, http://dbpedia.org/resource/List_of_heads_of_state_of_Guinea, http://dbpedia.org/resource/Lou_Henry_Hoover, http://dbpedia.org/resource/Louis_Lansana_Beavogui, http://dbpedia.org/resource/Luís_Cabral, http://dbpedia.org/resource/Lynn_Thorndike, http://dbpedia.org/resource/Malam_Bacai_Sanhá, http://dbpedia.org/resource/Mamadu_Ture_Kuruma, http://dbpedia.org/resource/Manuel_Barillas, http://dbpedia.org/resource/Manuel_Fraga_Iribarne, http://dbpedia.org/resource/Manuel_José_Arce, http://dbpedia.org/resource/Marilyn_Warren, http://dbpedia.org/resource/Mark_Murphy_(safety,_born_1955), http://dbpedia.org/resource/Mesut_Yılmaz, http://dbpedia.org/resource/Miguel_Ydígoras_Fuentes, http://dbpedia.org/resource/Milorad_Dodik, http://dbpedia.org/resource/Moussa_Dadis_Camara, http://dbpedia.org/resource/Nicholas_Rescher, http://dbpedia.org/resource/Nii_Amaa_Ollennu, http://dbpedia.org/resource/Otto_Pérez_Molina, http://dbpedia.org/resource/Pedro_Molina_Mazariegos, http://dbpedia.org/resource/Pedro_de_Aycinena_y_Piñol, http://dbpedia.org/resource/President_of_the_Australian_Senate, http://dbpedia.org/resource/President_of_the_New_South_Wales_Legislative_Council, http://dbpedia.org/resource/President_of_the_Xunta_of_Galicia, http://dbpedia.org/resource/Prince_Henry,_Duke_of_Gloucester, http://dbpedia.org/resource/Putin_must_go, http://dbpedia.org/resource/Raimundo_Pereira, http://dbpedia.org/resource/Ralph_P._Boas,_Jr., http://dbpedia.org/resource/Ramiro_de_León_Carpio, http://dbpedia.org/resource/Raymond_Poincaré, http://dbpedia.org/resource/Rose_Cleveland, http://dbpedia.org/resource/Roxana_Baldetti, http://dbpedia.org/resource/Russian_presidential_election,_2008, http://dbpedia.org/resource/Sabine_Bergmann-Pohl, http://dbpedia.org/resource/San_Anton_Palace, http://dbpedia.org/resource/Sir_Banister_Fletcher, http://dbpedia.org/resource/Stanley_Rosen, http://dbpedia.org/resource/Sun_Yat-sen_Memorial_Hall_(Guangzhou), http://dbpedia.org/resource/Sékouba_Konaté, http://dbpedia.org/resource/Thomas_A._Hendricks, http://dbpedia.org/resource/Thomas_Brassey,_1st_Earl_Brassey, http://dbpedia.org/resource/Thomas_Brisbane, http://dbpedia.org/resource/United_States_presidential_election,_1904, http://dbpedia.org/resource/Vicente_Cerna_y_Cerna, http://dbpedia.org/resource/Victorin_Lurel, http://dbpedia.org/resource/Waverly_Place, http://dbpedia.org/resource/Westland_Mansion, http://dbpedia.org/resource/Wheeler_Hazard_Peckham, http://dbpedia.org/resource/Wilhelm_Pieck, http://dbpedia.org/resource/William_B._Hornblower, http://dbpedia.org/resource/William_Douglas,_1st_Duke_of_Queensberry, http://dbpedia.org/resource/William_Lyne_Wilson, http://dbpedia.org/resource/Władysław_Anders, http://dbpedia.org/resource/Xanana_Gusmão, http://dbpedia.org/resource/Ángel_Aníbal_Guevara, http://dbpedia.org/resource/Ángel_Cabrera_(academic), http://dbpedia.org/resource/Óscar_Berger, http://dbpedia.org/resource/Abdul_Rahman_Ya'kub, http://dbpedia.org/resource/Abraham_Adrian_Albert, http://dbpedia.org/resource/Adams_political_family, http://dbpedia.org/resource/Adonijah_Welch, http://dbpedia.org/resource/Alfonso_Prat-Gay, http://dbpedia.org/resource/Alfred_Fowler, http://dbpedia.org/resource/Alfred_Milner,_1st_Viscount_Milner, http://dbpedia.org/resource/Antonio_Cassese, http://dbpedia.org/resource/Astrid_Nøklebye_Heiberg, http://dbpedia.org/resource/Atifete_Jahjaga, http://dbpedia.org/resource/Basil_O'Connor, http://dbpedia.org/resource/Boris_Tadić, http://dbpedia.org/resource/Borisav_Jović, http://dbpedia.org/resource/Branko_Kostić, http://dbpedia.org/resource/Charles_J._Guiteau, http://dbpedia.org/resource/Christina_Rau, http://dbpedia.org/resource/David_Eugene_Smith, http://dbpedia.org/resource/Devakanta_Barua, http://dbpedia.org/resource/Emil_Sandström, http://dbpedia.org/resource/Enrique_de_la_Mata, http://dbpedia.org/resource/Erich_Mielke, http://dbpedia.org/resource/Fausto_Pocar, http://dbpedia.org/resource/Ferenc_Münnich, http://dbpedia.org/resource/Gabrielle_Kirk_McDonald, http://dbpedia.org/resource/Gustav_Heinemann, http://dbpedia.org/resource/Halabja_chemical_attack, http://dbpedia.org/resource/Harold_Tafler_Shapiro, http://dbpedia.org/resource/Harry_Huskey, http://dbpedia.org/resource/Henry_Charles_Lea, http://dbpedia.org/resource/Hisashi_Owada, http://dbpedia.org/resource/Ingram_Olkin, http://dbpedia.org/resource/J._A._Ratcliffe, http://dbpedia.org/resource/James_J._Whalen, http://dbpedia.org/resource/James_Ward_(psychologist), http://dbpedia.org/resource/Javad_Fakoori, http://dbpedia.org/resource/Jean_de_Muralt, http://dbpedia.org/resource/Jesuit_Ivy, http://dbpedia.org/resource/John_Barton_Payne, http://dbpedia.org/resource/John_Hinckley,_Jr., http://dbpedia.org/resource/John_MacAulay, http://dbpedia.org/resource/John_Quincy_Adams_Birthplace, http://dbpedia.org/resource/John_T._Ford, http://dbpedia.org/resource/John_Wesley_Young, http://dbpedia.org/resource/José_Barroso_Chávez, http://dbpedia.org/resource/José_Gaspar_Rodríguez_de_Francia, http://dbpedia.org/resource/Juan_Antonio_Ríos, http://dbpedia.org/resource/Juan_Somavía, http://dbpedia.org/resource/List_of_Presidents_and_Vice-Presidents_of_the_Assembly_of_States_Parties_of_the_International_Criminal_Court, http://dbpedia.org/resource/List_of_Presidents_of_Republika_Srpska, http://dbpedia.org/resource/Mario_Enrique_Villarroel_Lander, http://dbpedia.org/resource/Marshall_Harvey_Stone, http://dbpedia.org/resource/Marston_Morse, http://dbpedia.org/resource/Maryam_Rajavi, http://dbpedia.org/resource/Massoud_Rajavi, http://dbpedia.org/resource/Miles_Aylmer_Fulton_Barnett, http://dbpedia.org/resource/Miron_Nicolescu, http://dbpedia.org/resource/Momir_Bulatović, http://dbpedia.org/resource/National_Intelligence_Coordinating_Agency, http://dbpedia.org/resource/Norman_Davis_(diplomat), http://dbpedia.org/resource/Paolo_Thaon_di_Revel, http://dbpedia.org/resource/Patrick_Lipton_Robinson, http://dbpedia.org/resource/Paul_Chambers_(industrialist), http://dbpedia.org/resource/Peggy_R._Williams, http://dbpedia.org/resource/Peter_Tomka, http://dbpedia.org/resource/Philippine_Drug_Enforcement_Agency, http://dbpedia.org/resource/Popular_Army_(Iraq), http://dbpedia.org/resource/Presidential_Issue, http://dbpedia.org/resource/Presidents_of_the_International_Olympic_Committee, http://dbpedia.org/resource/Radwaniyah_Palace, http://dbpedia.org/resource/Randolph_Quirk, http://dbpedia.org/resource/Rosh_yeshiva, http://dbpedia.org/resource/Ruud_Lubbers, http://dbpedia.org/resource/SS_President_Coolidge, http://dbpedia.org/resource/Saddam_Hussein_and_al-Qaeda_link_allegations, http://dbpedia.org/resource/Samuel_Flagg_Bemis, http://dbpedia.org/resource/Sigvard_Eklund, http://dbpedia.org/resource/Stephen_M._Schwebel, http://dbpedia.org/resource/Steven_Leath, http://dbpedia.org/resource/The_Adams_Chronicles, http://dbpedia.org/resource/Theodor_Meron, http://dbpedia.org/resource/Thomas_Bach, http://dbpedia.org/resource/Trial_of_Saddam_Hussein, http://dbpedia.org/resource/Uhuru_Kenyatta, http://dbpedia.org/resource/Ulysses_S._Grant_Home, http://dbpedia.org/resource/Vice_President_of_Bulgaria, http://dbpedia.org/resource/Vice_President_of_the_Philippines, http://dbpedia.org/resource/W._Sterling_Cole, http://dbpedia.org/resource/Walter_Dill_Scott, http://dbpedia.org/resource/Will_Lawther, http://dbpedia.org/resource/William_Daniels, http://dbpedia.org/resource/William_Short_(American_ambassador), http://dbpedia.org/resource/William_Smith_O'Brien, http://dbpedia.org/resource/Yitzchok_Hutner, http://dbpedia.org/resource/Zakir_Husain_(politician), http://dbpedia.org/resource/Abraham_Van_Buren, http://dbpedia.org/resource/Afonso_Costa, http://dbpedia.org/resource/Albert_Sabin, http://dbpedia.org/resource/Andrés_Bonifacio, http://dbpedia.org/resource/Ba_U, http://dbpedia.org/resource/Benjamin_Baker_(engineer), http://dbpedia.org/resource/Benjamin_Wadsworth, http://dbpedia.org/resource/Bernard_Benjamin, http://dbpedia.org/resource/Bhogaraju_Pattabhi_Sitaramayya, http://dbpedia.org/resource/Braulio_Carrillo_Colina, http://dbpedia.org/resource/Bushism, http://dbpedia.org/resource/C._F._W._Walther, http://dbpedia.org/resource/Charlotte_Dupuy, http://dbpedia.org/resource/Chris_Ballew, http://dbpedia.org/resource/Curtis_Gans, http://dbpedia.org/resource/David_L._Gunn, http://dbpedia.org/resource/Denys_Roberts, http://dbpedia.org/resource/Dingle_Foot, http://dbpedia.org/resource/Dore_Gold, http://dbpedia.org/resource/Duan_Qirui, http://dbpedia.org/resource/Edward_Forbes, http://dbpedia.org/resource/Elizabeth_Storrs_Mead, http://dbpedia.org/resource/Fadil_Hoxha, http://dbpedia.org/resource/Fay_Kanin, http://dbpedia.org/resource/Francisco_Xavier_do_Amaral, http://dbpedia.org/resource/Fred_W._Friendly, http://dbpedia.org/resource/Fulgencio_Yegros, http://dbpedia.org/resource/Félix_Agramont_Cota, http://dbpedia.org/resource/George_L._Harrison, http://dbpedia.org/resource/Gesine_Schwan, http://dbpedia.org/resource/Global_Marshall_Plan, http://dbpedia.org/resource/Goodluck_Jonathan, http://dbpedia.org/resource/Gyles_Brandreth, http://dbpedia.org/resource/Hannah_Van_Buren, http://dbpedia.org/resource/Harris_Wofford, http://dbpedia.org/resource/Harry_Altham, http://dbpedia.org/resource/Henry_Clifton_Sorby, http://dbpedia.org/resource/Henry_R._Towne, http://dbpedia.org/resource/Homer_L._Shantz, http://dbpedia.org/resource/Howard_W._Hunter, http://dbpedia.org/resource/Hugh_Foot,_Baron_Caradon, http://dbpedia.org/resource/Ion_Jalea, http://dbpedia.org/resource/Jagmohan_Dalmiya, http://dbpedia.org/resource/Jaime_Lusinchi, http://dbpedia.org/resource/Jaime_Ornelas_Camacho, http://dbpedia.org/resource/James_Samuel_Coleman, http://dbpedia.org/resource/Jean_Casimir-Perier, http://dbpedia.org/resource/Jiang_Kanghu, http://dbpedia.org/resource/John_Foot,_Baron_Foot, http://dbpedia.org/resource/John_Herman_Randall,_Jr., http://dbpedia.org/resource/John_Tusa, http://dbpedia.org/resource/John_Van_Buren, http://dbpedia.org/resource/Jonathan_Michie, http://dbpedia.org/resource/José_Batlle_y_Ordóñez, http://dbpedia.org/resource/Juan_José_Ibarretxe, http://dbpedia.org/resource/Leslie_Hore-Belisha,_1st_Baron_Hore-Belisha, http://dbpedia.org/resource/Lipman_Bers, http://dbpedia.org/resource/List_of_unofficial_Presidents_of_the_Philippines, http://dbpedia.org/resource/Léon_Bourgeois, http://dbpedia.org/resource/Macario_Sakay, http://dbpedia.org/resource/Malkiel_Kotler, http://dbpedia.org/resource/Manuel_Teixeira_Gomes, http://dbpedia.org/resource/Mao_suit, http://dbpedia.org/resource/Mariano_Paredes_(President_of_Mexico), http://dbpedia.org/resource/Michael_Crick, http://dbpedia.org/resource/Mihail_Kogălniceanu, http://dbpedia.org/resource/Miss_Beazley_(dog), http://dbpedia.org/resource/Nicolas_Browne-Wilkinson,_Baron_Browne-Wilkinson, http://dbpedia.org/resource/Nitin_Gadkari, http://dbpedia.org/resource/Norman_St_John-Stevas, http://dbpedia.org/resource/Nyan_Tun, http://dbpedia.org/resource/Paddy_Buggy, http://dbpedia.org/resource/Peter_Bazalgette, http://dbpedia.org/resource/Peter_Brooke,_Baron_Brooke_of_Sutton_Mandeville, http://dbpedia.org/resource/Peter_Kinder, http://dbpedia.org/resource/Reformed_Dutch_Church_(Kinderhook,_New_York), http://dbpedia.org/resource/Richard_Brauer, http://dbpedia.org/resource/Robert_P._Patterson, http://dbpedia.org/resource/Robert_Stephen_John_Sparks, http://dbpedia.org/resource/Ruben_Kun, http://dbpedia.org/resource/Rudi_Šeligo, http://dbpedia.org/resource/Sai_Mauk_Kham, http://dbpedia.org/resource/Sam_Nujoma, http://dbpedia.org/resource/Santiago_de_Vera, http://dbpedia.org/resource/Svetlana_Medvedeva, http://dbpedia.org/resource/Ted_Dexter, http://dbpedia.org/resource/Ted_Kaufman, http://dbpedia.org/resource/The_Dead_Zone_(novel), http://dbpedia.org/resource/Truman_Reservoir, http://dbpedia.org/resource/U.S._Route_190, http://dbpedia.org/resource/Ulysses_S._Grant_IV, http://dbpedia.org/resource/Van_Buren_County,_Michigan, http://dbpedia.org/resource/W._Graham_Claytor,_Jr., http://dbpedia.org/resource/Wasim_Sajjad, http://dbpedia.org/resource/William_A._MacCorkle, http://dbpedia.org/resource/William_Henry_Barlow, http://dbpedia.org/resource/Ásgeir_Ásgeirsson, http://dbpedia.org/resource/Ċensu_Tabone, http://dbpedia.org/resource/Alojz_Gradnik, http://dbpedia.org/resource/Alphonse_Massamba-Débat, http://dbpedia.org/resource/Ambrose_Burke, http://dbpedia.org/resource/André_Milongo, http://dbpedia.org/resource/Birthplace_of_Ronald_Reagan, http://dbpedia.org/resource/California's_12th_congressional_district_election,_1946, http://dbpedia.org/resource/Carl_Meinberg, http://dbpedia.org/resource/Center_for_the_National_Interest, http://dbpedia.org/resource/Charles_Pelot_Summerall, http://dbpedia.org/resource/Credibility_gap, http://dbpedia.org/resource/Death_and_funeral_of_Richard_Nixon, http://dbpedia.org/resource/Dmitry_Medvedev's_Cabinet, http://dbpedia.org/resource/Edmund_Schulthess, http://dbpedia.org/resource/Edward_Burleson, http://dbpedia.org/resource/Edward_Nixon, http://dbpedia.org/resource/Foreign_policy_of_Vladimir_Putin, http://dbpedia.org/resource/Frederick_Ferré, http://dbpedia.org/resource/Gaullist_Party, http://dbpedia.org/resource/George_Chandler, http://dbpedia.org/resource/George_Edwards_(aviation), http://dbpedia.org/resource/George_Junkin, http://dbpedia.org/resource/George_Martin_Lees, http://dbpedia.org/resource/Gerald_Cash, http://dbpedia.org/resource/Ghazi_Mashal_Ajil_al-Yawer, http://dbpedia.org/resource/Gheorghe_Gheorghiu-Dej, http://dbpedia.org/resource/Giulio_Carlo_Argan, http://dbpedia.org/resource/Godefroy_de_Blonay, http://dbpedia.org/resource/Gore_Effect, http://dbpedia.org/resource/Grover_E._Murray, http://dbpedia.org/resource/Henry_Brewerton, http://dbpedia.org/resource/Ilmari_Salminen, http://dbpedia.org/resource/Jack_Reagan, http://dbpedia.org/resource/Jerry_Voorhis, http://dbpedia.org/resource/Joachim_Yhombi-Opango, http://dbpedia.org/resource/John_Kroger, http://dbpedia.org/resource/John_Richard_Hyde, http://dbpedia.org/resource/Kenneth_Lewis_Anderson, http://dbpedia.org/resource/La_Casa_Pacifica, http://dbpedia.org/resource/Lee_and_Grant_at_Appomattox, http://dbpedia.org/resource/Lincoln_Catafalque, http://dbpedia.org/resource/List_of_Presidents_of_Romania, http://dbpedia.org/resource/List_of_Presidents_of_Washington_and_Lee_University, http://dbpedia.org/resource/List_of_United_States_presidential_elections_by_popular_vote_margin, http://dbpedia.org/resource/Lorenzo_de_Zavala, http://dbpedia.org/resource/Mahlamba_Ndlopfu, http://dbpedia.org/resource/Martial_law_in_Russia, http://dbpedia.org/resource/Martin_Hollis_(philosopher), http://dbpedia.org/resource/Medvedev_modernisation_programme, http://dbpedia.org/resource/Morning_in_America, http://dbpedia.org/resource/Murray_Chotiner, http://dbpedia.org/resource/Nassir_Abdulaziz_Al-Nasser, http://dbpedia.org/resource/Neil_Reagan, http://dbpedia.org/resource/Nelle_Wilson_Reagan, http://dbpedia.org/resource/Nicolae_Văcăroiu, http://dbpedia.org/resource/Norman_Hackerman, http://dbpedia.org/resource/Other_Losses, http://dbpedia.org/resource/Pasteur_Bizimungu, http://dbpedia.org/resource/Petre_Roman, http://dbpedia.org/resource/Pitirim_Sorokin, http://dbpedia.org/resource/Political_positions_of_Ronald_Reagan, http://dbpedia.org/resource/President_of_the_Senate_of_Romania, http://dbpedia.org/resource/Reagan_Democrat, http://dbpedia.org/resource/Reginald_Skelton, http://dbpedia.org/resource/Richard_Nixon's_last_press_conference, http://dbpedia.org/resource/Riga_Castle, http://dbpedia.org/resource/Ronald_Reagan_Building_and_International_Trade_Center, http://dbpedia.org/resource/Ronald_Reagan_bibliography, http://dbpedia.org/resource/Ruslan_Khasbulatov, http://dbpedia.org/resource/S._Malcolm_Gillis, http://dbpedia.org/resource/Sebastian_Menke, http://dbpedia.org/resource/Socks_(cat), http://dbpedia.org/resource/Solko_van_den_Bergh, http://dbpedia.org/resource/Strelna, http://dbpedia.org/resource/There_you_go_again, http://dbpedia.org/resource/USS_Abraham_Lincoln_(CVN-72), http://dbpedia.org/resource/United_States_Senate_election_in_California,_1950, http://dbpedia.org/resource/Valeriu_Zgonea, http://dbpedia.org/resource/Victor_Cavendish,_9th_Duke_of_Devonshire, http://dbpedia.org/resource/William_H._Nichols, http://dbpedia.org/resource/105th_New_York_State_Legislature, http://dbpedia.org/resource/106th_New_York_State_Legislature, http://dbpedia.org/resource/Aaron_S._Watkins, http://dbpedia.org/resource/Albert_Günther, http://dbpedia.org/resource/Bamie_Roosevelt, http://dbpedia.org/resource/Chub_Feeney, http://dbpedia.org/resource/Clifton_Daniel, http://dbpedia.org/resource/David_Keene, http://dbpedia.org/resource/David_T._Disney, http://dbpedia.org/resource/Ed_Kealty, http://dbpedia.org/resource/Einar_Hille, http://dbpedia.org/resource/Electoral_history_of_Theodore_Roosevelt, http://dbpedia.org/resource/Elliott_Bulloch_Roosevelt, http://dbpedia.org/resource/Finnish_presidential_election,_2000, http://dbpedia.org/resource/Finnish_presidential_election,_2006, http://dbpedia.org/resource/First_inauguration_of_Theodore_Roosevelt, http://dbpedia.org/resource/Frederick_Selous, http://dbpedia.org/resource/George_B._Cortelyou, http://dbpedia.org/resource/Gillian_Slovo, http://dbpedia.org/resource/James_Geikie, http://dbpedia.org/resource/James_Meadows_Rendel_(engineer), http://dbpedia.org/resource/John_Flammang_Schrank, http://dbpedia.org/resource/Leonard_Gillman, http://dbpedia.org/resource/Lincoln_Memorial_University, http://dbpedia.org/resource/Michael_Farris, http://dbpedia.org/resource/Mäntyniemi, http://dbpedia.org/resource/Paul_Beeston, http://dbpedia.org/resource/Pentti_Arajärvi, http://dbpedia.org/resource/Peter_Bavasi, http://dbpedia.org/resource/Political_positions_of_Theodore_Roosevelt, http://dbpedia.org/resource/Presidential_transition_of_Barack_Obama, http://dbpedia.org/resource/Purnaiah, http://dbpedia.org/resource/Robby_Jackson, http://dbpedia.org/resource/Robert_Darnton, http://dbpedia.org/resource/Robert_Lucas_(governor), http://dbpedia.org/resource/Roosevelt_River, http://dbpedia.org/resource/Roosevelt–Rondon_Scientific_Expedition, http://dbpedia.org/resource/Sagamore_Hill_(house), http://dbpedia.org/resource/Theodore_Roosevelt_Association, http://dbpedia.org/resource/Theodore_Roosevelt_Birthplace_National_Historic_Site, http://dbpedia.org/resource/Theodore_Roosevelt_Cyclopedia, http://dbpedia.org/resource/Theodore_Roosevelt_Island, http://dbpedia.org/resource/Tom_Schieffer, http://dbpedia.org/resource/United_States_presidential_pets, http://dbpedia.org/resource/Walter_Francis_White, http://dbpedia.org/resource/Abraham_Pierson, http://dbpedia.org/resource/Alfred_Whitney_Griswold, http://dbpedia.org/resource/Anders_Nordström, http://dbpedia.org/resource/Arthur_Dunkel, http://dbpedia.org/resource/Aubrey_J._Kempner, http://dbpedia.org/resource/Brock_Chisholm, http://dbpedia.org/resource/Charles_Phelps_Taft_II, http://dbpedia.org/resource/Cyril_Clarke, http://dbpedia.org/resource/David_F._Houston, http://dbpedia.org/resource/David_W._Slater, http://dbpedia.org/resource/Duncan_McArthur, http://dbpedia.org/resource/Edgar_Lungu, http://dbpedia.org/resource/Edna_Madzongwe, http://dbpedia.org/resource/Edward_Dorr_Griffin, http://dbpedia.org/resource/Eleanor_Wilson_McAdoo, http://dbpedia.org/resource/Eric_Wyndham_White, http://dbpedia.org/resource/George_W._Wickersham, http://dbpedia.org/resource/Guy_Scott, http://dbpedia.org/resource/H._Ian_Macdonald, http://dbpedia.org/resource/Halfdan_T._Mahler, http://dbpedia.org/resource/Harry_Arthurs, http://dbpedia.org/resource/Helen_Herron_Taft, http://dbpedia.org/resource/Helen_Taft_Manning, http://dbpedia.org/resource/Heman_Humphrey, http://dbpedia.org/resource/Herbert_Brownell,_Jr., http://dbpedia.org/resource/Ivan_Stambolić, http://dbpedia.org/resource/Jacob_M._Dickinson, http://dbpedia.org/resource/James_Fairman_Fielder, http://dbpedia.org/resource/James_Monroe_(congressman), http://dbpedia.org/resource/Jessie_Woodrow_Wilson_Sayre, http://dbpedia.org/resource/John_T._Hayward, http://dbpedia.org/resource/John_W._Griggs, http://dbpedia.org/resource/Joseph_J._Sisco, http://dbpedia.org/resource/Joseph_Ruggles_Wilson, http://dbpedia.org/resource/José_Madriz, http://dbpedia.org/resource/Judith_Rodin, http://dbpedia.org/resource/Leonard_Goldberg, http://dbpedia.org/resource/List_of_Presidents_of_the_United_States_by_education, http://dbpedia.org/resource/List_of_Vice_Presidents_of_Zambia, http://dbpedia.org/resource/Lord_George_Hamilton, http://dbpedia.org/resource/Mamdouh_Shoukri, http://dbpedia.org/resource/Marcolino_Gomes_Candau, http://dbpedia.org/resource/Michael_Gorman_(librarian), http://dbpedia.org/resource/Michael_Sadler_(educationist), http://dbpedia.org/resource/Miško_Kranjec, http://dbpedia.org/resource/Mohammad_Mohammadullah, http://dbpedia.org/resource/Murray_G._Ross, http://dbpedia.org/resource/Mushtaq_Ahmed_Gurmani, http://dbpedia.org/resource/New_Nationalism, http://dbpedia.org/resource/Noah_Porter, http://dbpedia.org/resource/Olivier_Long, http://dbpedia.org/resource/Peter_Salovey, http://dbpedia.org/resource/Philip_D._Curtin, http://dbpedia.org/resource/Philip_Kelland, http://dbpedia.org/resource/Rabah_Bitat, http://dbpedia.org/resource/Rafi_Daham_al-Tikriti, http://dbpedia.org/resource/Richard_H._Brodhead, http://dbpedia.org/resource/Rick_Levin, http://dbpedia.org/resource/Robert_Hemenway, http://dbpedia.org/resource/Rose_Kennedy_Cocktail, http://dbpedia.org/resource/Sam_Nujoma_Street, http://dbpedia.org/resource/Samuel_Andrew, http://dbpedia.org/resource/Spencer_Horatio_Walpole, http://dbpedia.org/resource/Stane_Dolanc, http://dbpedia.org/resource/Suha_Arafat, http://dbpedia.org/resource/Susan_Mann, http://dbpedia.org/resource/Taft_family, http://dbpedia.org/resource/Temple_of_Music, http://dbpedia.org/resource/Theodore_Dwight_Woolsey, http://dbpedia.org/resource/Theodore_Friend, http://dbpedia.org/resource/Timothy_Dwight_V, http://dbpedia.org/resource/Tony_Penikett, http://dbpedia.org/resource/William_C._Preston, http://dbpedia.org/resource/William_Edward_Ayrton, http://dbpedia.org/resource/William_Howard_Taft_National_Historic_Site, http://dbpedia.org/resource/Woodrow_Wilson_House_(Washington,_D.C.), http://dbpedia.org/resource/Woodrow_Wilson_Presidential_Library, http://dbpedia.org/resource/Zephaniah_Swift_Moore, http://dbpedia.org/resource/Arthur_R._Taylor, http://dbpedia.org/resource/Bud_Grant_(broadcaster), http://dbpedia.org/resource/Casa_Rosada, http://dbpedia.org/resource/Casa_de_Nariño, http://dbpedia.org/resource/Charles_E._Young, http://dbpedia.org/resource/Constantin_Argetoianu, http://dbpedia.org/resource/Daniel_Weiss_(art_historian), http://dbpedia.org/resource/David_Adamany, http://dbpedia.org/resource/Diosdado_Cabello, http://dbpedia.org/resource/Franklin_David_Murphy, http://dbpedia.org/resource/Gene_D._Block, http://dbpedia.org/resource/Hedayat_Amin_Arsala, http://dbpedia.org/resource/Ivonne_Ortega_Pacheco, http://dbpedia.org/resource/James_David_Barber, http://dbpedia.org/resource/Janez_Stanovnik, http://dbpedia.org/resource/John_Kitzhaber, http://dbpedia.org/resource/Jonathan_Dickinson_(New_Jersey), http://dbpedia.org/resource/Jonathan_Edwards_(Washington_&_Jefferson_College), http://dbpedia.org/resource/Jonathan_Edwards_(the_younger), http://dbpedia.org/resource/Josef_Skupa, http://dbpedia.org/resource/José_Joaquín_de_Herrera, http://dbpedia.org/resource/José_Mariano_Salas, http://dbpedia.org/resource/Lenny_Skutnik, http://dbpedia.org/resource/Mark_Huddleston, http://dbpedia.org/resource/Maurice_Davin, http://dbpedia.org/resource/Michael_Dann, http://dbpedia.org/resource/Miguel_d'Escoto_Brockmann, http://dbpedia.org/resource/Nancy_Tellem, http://dbpedia.org/resource/Nina_Tassler, http://dbpedia.org/resource/President_of_the_Balearic_Islands, http://dbpedia.org/resource/President_of_the_Generalitat_Valenciana, http://dbpedia.org/resource/Ralph_W._Conant, http://dbpedia.org/resource/Rashtrapati_Nilayam, http://dbpedia.org/resource/Sean_McManus_(television_executive), http://dbpedia.org/resource/Theodore_Paul_Wright, http://dbpedia.org/resource/Vladimir_Maksimov, http://dbpedia.org/resource/W._Robert_Parks, http://dbpedia.org/resource/William_Emerson, http://dbpedia.org/resource/Yan_Fu, http://dbpedia.org/resource/Zeenat_Karzai, http://dbpedia.org/resource/Alexander_Lawton, http://dbpedia.org/resource/Amir_Kabir, http://dbpedia.org/resource/Bagatur, http://dbpedia.org/resource/Barbara_Rosenkranz, http://dbpedia.org/resource/Charles_P._Neill, http://dbpedia.org/resource/Democrats_for_Nixon, http://dbpedia.org/resource/Eileen_Wilson-Oyelaran, http://dbpedia.org/resource/Electoral_history_of_Lyndon_B._Johnson, http://dbpedia.org/resource/Eliphalet_Nott, http://dbpedia.org/resource/Evarts_Boutell_Greene, http://dbpedia.org/resource/Fred_Pooley, http://dbpedia.org/resource/Gail_McDonald, http://dbpedia.org/resource/George_Washington_Baines, http://dbpedia.org/resource/Guillermo_Fernández_Vara, http://dbpedia.org/resource/Horace_Jones_(architect), http://dbpedia.org/resource/Hulan_Jack, http://dbpedia.org/resource/Humphrey–Hawkins_Full_Employment_Act, http://dbpedia.org/resource/James_F._Jones, http://dbpedia.org/resource/Javier_Fernández_Fernández, http://dbpedia.org/resource/John_Frederick_Bateman, http://dbpedia.org/resource/John_Hall_Gladstone, http://dbpedia.org/resource/John_L._Buono, http://dbpedia.org/resource/John_Milton_Gregory, http://dbpedia.org/resource/John_Scott_Everton, http://dbpedia.org/resource/Josep_Tarradellas_i_Joan, http://dbpedia.org/resource/Juan_Vicente_Herrera, http://dbpedia.org/resource/Li_Yuan-tsu, http://dbpedia.org/resource/List_of_Vice_Presidents_of_the_Republic_of_China, http://dbpedia.org/resource/Lloyd_Braun, http://dbpedia.org/resource/Louis_J._Gallagher, http://dbpedia.org/resource/Lynn_Steen, http://dbpedia.org/resource/Miguel_Antonio_Caro, http://dbpedia.org/resource/Miguel_Ángel_Revilla, http://dbpedia.org/resource/Momčilo_Ninčić, http://dbpedia.org/resource/Operation_Texas, http://dbpedia.org/resource/Richard_Delafield, http://dbpedia.org/resource/Sam_Houston_Johnson, http://dbpedia.org/resource/Tahawus,_New_York, http://dbpedia.org/resource/Thomas_Roderick_Dew, http://dbpedia.org/resource/Timothy_Light, http://dbpedia.org/resource/Tom_Johnson_(journalist), http://dbpedia.org/resource/Truman_Committee, http://dbpedia.org/resource/USS_John_Adams_(1799), http://dbpedia.org/resource/Weimer_Hicks, http://dbpedia.org/resource/William_Archibald_Dunning, http://dbpedia.org/resource/Willis_C._Hawley, http://dbpedia.org/resource/Wu_Youxun, http://dbpedia.org/resource/Zealous_Bates_Tower, http://dbpedia.org/resource/Abram_L._Sachar, http://dbpedia.org/resource/Adolfo_Diz, http://dbpedia.org/resource/Arthur_Hay,_9th_Marquess_of_Tweeddale, http://dbpedia.org/resource/Christian_Gottfried_Daniel_Nees_von_Esenbeck, http://dbpedia.org/resource/Daniel_Zaïdani, http://dbpedia.org/resource/David_Dingwall, http://dbpedia.org/resource/David_Miller_(entomologist), http://dbpedia.org/resource/Frank_Johnson_Goodnow, http://dbpedia.org/resource/Frederick_Wollaston_Hutton, http://dbpedia.org/resource/Gervais_Rentoul, http://dbpedia.org/resource/Holden_Thorp, http://dbpedia.org/resource/John_F._Kennedy_Memorial_Bridge, http://dbpedia.org/resource/John_Ramsbottom_(mycologist), http://dbpedia.org/resource/Joseph_Gibson_Hoyt, http://dbpedia.org/resource/Larry_R._Thompson, http://dbpedia.org/resource/Operation_Coffee_Cup, http://dbpedia.org/resource/Richard_Dell, http://dbpedia.org/resource/Richard_Salter_Storrs, http://dbpedia.org/resource/Richard_W._Lariviere, http://dbpedia.org/resource/Sheng_Xuanhuai, http://dbpedia.org/resource/Stephanus_Schoeman, http://dbpedia.org/resource/Valentín_Gómez_Farías, http://dbpedia.org/resource/William_Jacob_Holland, http://dbpedia.org/resource/Worcester_Reed_Warner, http://dbpedia.org/resource/Zachary_Taylor_National_Cemetery, http://dbpedia.org/resource/Abraham_K._Allison, http://dbpedia.org/resource/Alberto_Fabra, http://dbpedia.org/resource/Aníbal_Zañartu, http://dbpedia.org/resource/Arbusto_Energy, http://dbpedia.org/resource/Camp_Zachary_Taylor, http://dbpedia.org/resource/Charley_Eugene_Johns, http://dbpedia.org/resource/Don_Johnston, http://dbpedia.org/resource/Frank_Brogan, http://dbpedia.org/resource/Frank_McGlynn,_Sr., http://dbpedia.org/resource/Ignacio_Urrutia_Manzano, http://dbpedia.org/resource/Jeff_Atwater, http://dbpedia.org/resource/Lyon_Gardiner_Tyler, http://dbpedia.org/resource/Park_Trammell, http://dbpedia.org/resource/President_of_the_Florida_Senate, http://dbpedia.org/resource/Robert_R._Reid, http://dbpedia.org/resource/Samuel_Irving_Rosenman, http://dbpedia.org/resource/Sherwood_Forest_Plantation, http://dbpedia.org/resource/Stephen_LeDrew, http://dbpedia.org/resource/Toni_Jennings, http://dbpedia.org/resource/Abraham_Lincoln_Memorial_Bridge, http://dbpedia.org/resource/Brian_K._Johnson, http://dbpedia.org/resource/Carlos_Castillo_Peraza, http://dbpedia.org/resource/Carlos_Ferrero, http://dbpedia.org/resource/Carol_Folt, http://dbpedia.org/resource/Charlene_R._Nunley, http://dbpedia.org/resource/Cornelius_Plantinga, http://dbpedia.org/resource/David_Waisman, http://dbpedia.org/resource/DeRionne_P._Pollard, http://dbpedia.org/resource/Dr._Sun_Yat-Sen_Classical_Chinese_Garden, http://dbpedia.org/resource/Ernesto_Ruffo_Appel, http://dbpedia.org/resource/Eugene_Allen_Gilmore, http://dbpedia.org/resource/Gérard_Larcher, http://dbpedia.org/resource/James_Moeser, http://dbpedia.org/resource/John_J._Gardner, http://dbpedia.org/resource/Joseph_Cowen, http://dbpedia.org/resource/List_of_Presidents_of_China, http://dbpedia.org/resource/Louis_Round_Wilson, http://dbpedia.org/resource/Léon_Say, http://dbpedia.org/resource/Mariano_Palacios_Alcocer, http://dbpedia.org/resource/Moorfield_Storey, http://dbpedia.org/resource/Mornings_on_Horseback, http://dbpedia.org/resource/Nyerere_Cup, http://dbpedia.org/resource/Pascual_Ortiz_Rubio, http://dbpedia.org/resource/Preparedness_Movement, http://dbpedia.org/resource/President_of_the_Institutional_Revolutionary_Party, http://dbpedia.org/resource/Raúl_Diez_Canseco, http://dbpedia.org/resource/Robert_Jefferson_Breckinridge, http://dbpedia.org/resource/Roosevelt's_World_War_I_volunteers, http://dbpedia.org/resource/Samuel_Stanhope_Smith, http://dbpedia.org/resource/Stan_Musial_Veterans_Memorial_Bridge, http://dbpedia.org/resource/Steven_Ballard, http://dbpedia.org/resource/Sun_Yat-sen_House_(Nanjing), http://dbpedia.org/resource/Sun_Yat_Sen_Nanyang_Memorial_Hall, http://dbpedia.org/resource/Tomás_Yarrington, http://dbpedia.org/resource/Ultimate_Thor, http://dbpedia.org/resource/United_States_Constitution_and_worldwide_influence]... [INFO] P: 6.680919294494922E-5, R: 1.0, F: 1.336094595497361E-4 -[INFO] Cumulative metrics, sample: 954, P: 0.503, R: 0.559, macro F1: 0.511, macro F1*: 0.530 -[INFO] Current time: 1186 ms, average: 2947.48 ms/ques, entity detection: 497.07 ms/ques (linking tool: 403.20 ms/ques), relation detection: 964.76 ms/ques (similarity: 715.07 ms/ques), EDG generation: 46.36 ms/ques +[INFO] Cumulative metrics, sample: 954, P: 0.503, R: 0.559, macro F1: 0.530 +[INFO] Current question consumed time: 1193 ms, average: 2628.46 ms/ques, entity detection: 434.06 ms/ques (linking tool: 356.28 ms/ques), relation detection: 882.61 ms/ques (similarity: 707.44 ms/ques), EDG generation: 46.88 ms/ques [INFO] Entity Linking P: 0.740041928721174, R: 0.6918238993710691, micro F1: 0.707372466806429, macro F1: 0.7151210438594068 [INFO] Relation Linking P: 0.5963487071977638, R: 0.5419287211740041, micro F1: 0.5576869322152347, macro F1: 0.5678378297068416 [INFO] Type Linking P: 0.5545073375262054, R: 0.5545073375262054, micro F1: 0.5545073375262054, macro F1: 0.5545073375262054 @@ -13388,8 +13388,8 @@ Question 954: Name some ww1 veterans? [INFO] Golden Answer: [http://dbpedia.org/resource/Carl_Gustaf_Emil_Mannerheim, http://dbpedia.org/resource/Eddie_Rickenbacker, http://dbpedia.org/resource/Gabriele_D'Annunzio, http://dbpedia.org/resource/Henning_von_Tresckow, http://dbpedia.org/resource/Hubert_Work, http://dbpedia.org/resource/John_McAuley_Palmer_(general), http://dbpedia.org/resource/John_W._Bricker, http://dbpedia.org/resource/Józef_Piłsudski, http://dbpedia.org/resource/Paul_Fussell, http://dbpedia.org/resource/Somerset_Light_Infantry, http://dbpedia.org/resource/3rd_Light_Horse_Brigade, http://dbpedia.org/resource/Aristeidis_Moraitinis_(aviator), http://dbpedia.org/resource/Arthur_R._Wilson, http://dbpedia.org/resource/Basil_Brooke,_1st_Viscount_Brookeborough, http://dbpedia.org/resource/Cyriaque_Gillain, http://dbpedia.org/resource/Dietrich_von_Choltitz, http://dbpedia.org/resource/Dudley_de_Chair, http://dbpedia.org/resource/Edward_Mann_Lewis, http://dbpedia.org/resource/Erich_Fellgiebel, http://dbpedia.org/resource/Ernest_Troubridge, http://dbpedia.org/resource/Frederick_Barton_Maurice, http://dbpedia.org/resource/Frederick_Sykes, http://dbpedia.org/resource/Fritz_Thiele, http://dbpedia.org/resource/Gerd_von_Rundstedt, http://dbpedia.org/resource/Harry_Crerar, http://dbpedia.org/resource/Henry_Balding_Lewis, http://dbpedia.org/resource/Henry_Kent_Hewitt, http://dbpedia.org/resource/Hunter_Liggett, http://dbpedia.org/resource/James_Van_Fleet, http://dbpedia.org/resource/Jean_de_Lattre_de_Tassigny, http://dbpedia.org/resource/John_Dill, http://dbpedia.org/resource/John_L._Hines, http://dbpedia.org/resource/Julian_Byng,_1st_Viscount_Byng_of_Vimy, http://dbpedia.org/resource/Louis_Franchet_d'Espèrey, http://dbpedia.org/resource/Mariano_Goybet, http://dbpedia.org/resource/Martin_O'Meara, http://dbpedia.org/resource/Miles_Dempsey, http://dbpedia.org/resource/Neill_Malcolm, http://dbpedia.org/resource/Peyton_C._March, http://dbpedia.org/resource/Pietro_Badoglio, http://dbpedia.org/resource/Reginald_Bacon, http://dbpedia.org/resource/Richard_O'Connor, http://dbpedia.org/resource/Robert_L._Ghormley, http://dbpedia.org/resource/Roderic_Hill, http://dbpedia.org/resource/Roman_von_Ungern-Sternberg, http://dbpedia.org/resource/Rosslyn_Wemyss,_1st_Baron_Wester_Wemyss, http://dbpedia.org/resource/Sir_Charles_Madden,_1st_Baronet, http://dbpedia.org/resource/Stanley_Colville, http://dbpedia.org/resource/Svetozar_Boroević, http://dbpedia.org/resource/Tasker_H._Bliss, http://dbpedia.org/resource/Werner_Dankwort, http://dbpedia.org/resource/Westminster_Dragoons, http://dbpedia.org/resource/Wilfrid_Freeman, http://dbpedia.org/resource/Wilhelm_Ritter_von_Thoma, http://dbpedia.org/resource/William_Birdwood,_1st_Baron_Birdwood, http://dbpedia.org/resource/William_J._Donovan, http://dbpedia.org/resource/Cecil_Arthur_Lewis, http://dbpedia.org/resource/Dallas_Brooks, http://dbpedia.org/resource/David_Murray_Anderson, http://dbpedia.org/resource/Eugeniusz_Kwiatkowski, http://dbpedia.org/resource/Friedrich_Ruge, http://dbpedia.org/resource/Ivan_Grigorovich, http://dbpedia.org/resource/John_Loder,_2nd_Baron_Wakehurst, http://dbpedia.org/resource/Józef_Unrug, http://dbpedia.org/resource/Leverett_Saltonstall, http://dbpedia.org/resource/Marijan_Varešanin, http://dbpedia.org/resource/Nikolai_Pavlovich_Sablin, http://dbpedia.org/resource/Nikolai_Ruzsky, http://dbpedia.org/resource/Oskar_Potiorek, http://dbpedia.org/resource/Otto_Strasser, http://dbpedia.org/resource/Philip_Game, http://dbpedia.org/resource/RAF_Digby, http://dbpedia.org/resource/Robert_Coontz, http://dbpedia.org/resource/Winston_Dugan,_1st_Baron_Dugan_of_Victoria, http://dbpedia.org/resource/Franz_Böhme, http://dbpedia.org/resource/Harold_Rainsford_Stark, http://dbpedia.org/resource/Herbert_Cyril_Thacker, http://dbpedia.org/resource/Infantry_Branch_(United_States), http://dbpedia.org/resource/John_Francis_Young, http://dbpedia.org/resource/Nevil_Macready, http://dbpedia.org/resource/Royal_Monmouthshire_Royal_Engineers, http://dbpedia.org/resource/Semyon_Budyonny, http://dbpedia.org/resource/Semyon_Timoshenko, http://dbpedia.org/resource/Sir_Henry_Wilson,_1st_Baronet, http://dbpedia.org/resource/Tom_Barry_(soldier), http://dbpedia.org/resource/United_States_Army_Rangers, http://dbpedia.org/resource/Wilhelm_Rediess, http://dbpedia.org/resource/29th_Training_Systems_Squadron, http://dbpedia.org/resource/Albert_Cushing_Read, http://dbpedia.org/resource/Arthur_Rostron, http://dbpedia.org/resource/Brehon_B._Somervell, http://dbpedia.org/resource/Charles_E._Rosendahl, http://dbpedia.org/resource/Francis_Ventris, http://dbpedia.org/resource/Franz_Künstler, http://dbpedia.org/resource/James_C._Marshall, http://dbpedia.org/resource/John_Simpson_Kirkpatrick, http://dbpedia.org/resource/Justin_Tuveri, http://dbpedia.org/resource/Leonidas_Paraskevopoulos, http://dbpedia.org/resource/Leslie_Averill, http://dbpedia.org/resource/Oliver_P._Echols, http://dbpedia.org/resource/Robert_P._Patterson, http://dbpedia.org/resource/Royal_Welch_Fusiliers, http://dbpedia.org/resource/Wilhelm_D._Styer, http://dbpedia.org/resource/Alexander_Andreyevich_Svechin, http://dbpedia.org/resource/Charles_Pelot_Summerall, http://dbpedia.org/resource/Charles_Symonds, http://dbpedia.org/resource/George_Egerton_(Royal_Navy_officer), http://dbpedia.org/resource/George_William_Goddard, http://dbpedia.org/resource/Hugh_Trenchard,_1st_Viscount_Trenchard, http://dbpedia.org/resource/Jukums_Vācietis, http://dbpedia.org/resource/Nikolay_Dukhonin, http://dbpedia.org/resource/Sergeant_Stubby, http://dbpedia.org/resource/19th_Light_Brigade_(United_Kingdom), http://dbpedia.org/resource/60th_Infantry_Regiment_(United_States), http://dbpedia.org/resource/III_Corps_(United_States), http://dbpedia.org/resource/Max_Hoffmann, http://dbpedia.org/resource/Grigory_Kulik, http://dbpedia.org/resource/Hans_von_Ahlfen, http://dbpedia.org/resource/Henry_Grace_(Royal_Navy_officer), http://dbpedia.org/resource/Hermann_Niehoff, http://dbpedia.org/resource/Ivan_Isakov, http://dbpedia.org/resource/John_Whelchel, http://dbpedia.org/resource/Master-at-arms, http://dbpedia.org/resource/Otto_Hofmann, http://dbpedia.org/resource/Peter_Strasser, http://dbpedia.org/resource/Wilfred_Curtis, http://dbpedia.org/resource/111th_Reconnaissance_Squadron, http://dbpedia.org/resource/Arthur_Hauffe, http://dbpedia.org/resource/Constantin_Prezan, http://dbpedia.org/resource/Constantin_Sănătescu, http://dbpedia.org/resource/Flaminio_Piccoli, http://dbpedia.org/resource/Ilie_Șteflea, http://dbpedia.org/resource/Ion_Gigurtu, http://dbpedia.org/resource/Jagdstaffel_49, http://dbpedia.org/resource/Yates_Stirling,_Jr., http://dbpedia.org/resource/Nikifor_Grigoriev, http://dbpedia.org/resource/Schutztruppe, http://dbpedia.org/resource/VIII_Corps_(German_Empire), http://dbpedia.org/resource/Wilhelm_Schepmann, http://dbpedia.org/resource/29th_Infantry_Brigade_(United_Kingdom), http://dbpedia.org/resource/30th_Infantry_Regiment_(United_States), http://dbpedia.org/resource/32nd_Army_Air_&_Missile_Defense_Command, http://dbpedia.org/resource/6th_Infantry_Brigade_(United_Kingdom), http://dbpedia.org/resource/Frank_P._Lahm, http://dbpedia.org/resource/John_Capper, http://dbpedia.org/resource/41st_Electronic_Combat_Squadron, http://dbpedia.org/resource/George_Braund, http://dbpedia.org/resource/Josias,_Hereditary_Prince_of_Waldeck_and_Pyrmont, http://dbpedia.org/resource/Le_Régiment_de_Maisonneuve, http://dbpedia.org/resource/Melvin_Maas, http://dbpedia.org/resource/Royal_North_Devon_Yeomanry, http://dbpedia.org/resource/6th_Infantry_Division_(United_States), http://dbpedia.org/resource/Robert_A._McClure, http://dbpedia.org/resource/10th_(Irish)_Division, http://dbpedia.org/resource/Charles_R._Codman, http://dbpedia.org/resource/Curtis_Boozman, http://dbpedia.org/resource/Frank_Voelker,_Sr., http://dbpedia.org/resource/Hoffman_L._Fuller, http://dbpedia.org/resource/Larry_Sale, http://dbpedia.org/resource/Robert_Herring_(RAF_officer), http://dbpedia.org/resource/Robert_Quigg, http://dbpedia.org/resource/Robert_Scott_(VC), http://dbpedia.org/resource/Thomas_Crisp, http://dbpedia.org/resource/Alexander_Wilkinson, http://dbpedia.org/resource/Charles_McMorris, http://dbpedia.org/resource/Dudley_Wright_Knox, http://dbpedia.org/resource/Ernest_Psichari, http://dbpedia.org/resource/Ernst_Schirlitz, http://dbpedia.org/resource/Evelyn_Barker, http://dbpedia.org/resource/Francis_Octavius_Grenfell, http://dbpedia.org/resource/Frederick_Steiwer, http://dbpedia.org/resource/Henry_A._Wiley, http://dbpedia.org/resource/James_K._Parsons, http://dbpedia.org/resource/John_Hood_(naval_officer), http://dbpedia.org/resource/John_Murray_(general), http://dbpedia.org/resource/Karl-Wilhelm_von_Schlieben, http://dbpedia.org/resource/New_Zealand_Tunnelling_Company, http://dbpedia.org/resource/Robert_Lee_Rowe, http://dbpedia.org/resource/Sidney_E._Manning, http://dbpedia.org/resource/Thomas_W._Herren, http://dbpedia.org/resource/Walter_Hennecke, http://dbpedia.org/resource/89th_Infantry_Division_(United_States), http://dbpedia.org/resource/Albert_H._Blanding, http://dbpedia.org/resource/Alexander_R._Bolling, http://dbpedia.org/resource/Carlstrom_Field, http://dbpedia.org/resource/Emory_S._Land, http://dbpedia.org/resource/George_Windle_Read,_Jr., http://dbpedia.org/resource/James_Garesche_Ord, http://dbpedia.org/resource/325th_Infantry_Regiment_(United_States), http://dbpedia.org/resource/326th_Infantry_Regiment_(United_States), http://dbpedia.org/resource/3rd_Battalion,_319th_Field_Artillery_Regiment, http://dbpedia.org/resource/407th_Forward_Support_Battalion, http://dbpedia.org/resource/53rd_Infantry_Regiment_(United_States), http://dbpedia.org/resource/Arnold_E._True, http://dbpedia.org/resource/Arthur_S._Champeny, http://dbpedia.org/resource/Frank_Friday_Fletcher, http://dbpedia.org/resource/Frank_J._Lowry, http://dbpedia.org/resource/George_B._Duncan, http://dbpedia.org/resource/Hugh_J._Knerr, http://dbpedia.org/resource/John_Winterton, http://dbpedia.org/resource/Julius_Ochs_Adler, http://dbpedia.org/resource/Kurt_von_Tippelskirch, http://dbpedia.org/resource/Milo_Lemert, http://dbpedia.org/resource/Samuel_Woodfill, http://dbpedia.org/resource/Thomas_Jefferson_Cowie, http://dbpedia.org/resource/William_P._Burnham, http://dbpedia.org/resource/Willie_Sandlin, http://dbpedia.org/resource/Charles_Burnett_(RAF_officer), http://dbpedia.org/resource/Charles_P._Hall, http://dbpedia.org/resource/Deming_Bronson, http://dbpedia.org/resource/Harry_Hill_Bandholtz, http://dbpedia.org/resource/Moses_Hardy, http://dbpedia.org/resource/Alva_J._Brasted, http://dbpedia.org/resource/Donald_W._McGowan, http://dbpedia.org/resource/John_Knowles_Herr, http://dbpedia.org/resource/Kenneth_F._Cramer, http://dbpedia.org/resource/Mark_Lambert_Bristol, http://dbpedia.org/resource/Michael_Valente, http://dbpedia.org/resource/Wilfred_Beaver, http://dbpedia.org/resource/Francis_Thomas_Evans,_Sr., http://dbpedia.org/resource/Harley_H._Christy, http://dbpedia.org/resource/Herbert_T._Perrin, http://dbpedia.org/resource/John_F._Hunter, http://dbpedia.org/resource/Joseph_Gabriel_Harner, http://dbpedia.org/resource/Marcus_W._Robertson, http://dbpedia.org/resource/Orlando_Henderson_Petty, http://dbpedia.org/resource/Russell_Coffey, http://dbpedia.org/resource/Stanley_Eric_Reinhart, http://dbpedia.org/resource/Wilbur_M._White, http://dbpedia.org/resource/Alfred_P.C._Petsch, http://dbpedia.org/resource/Arnold_Williams_(American_politician), http://dbpedia.org/resource/Barron_Field, http://dbpedia.org/resource/David_B._Barkley, http://dbpedia.org/resource/Everett_Hughes_(general), http://dbpedia.org/resource/George_Peddy, http://dbpedia.org/resource/George_W._Hardy,_Jr., http://dbpedia.org/resource/Gordon_Browning, http://dbpedia.org/resource/H._Lane_Mitchell, http://dbpedia.org/resource/Harold_Earthman, http://dbpedia.org/resource/Herold_J._Weiler, http://dbpedia.org/resource/Joseph_B._Adkison, http://dbpedia.org/resource/Naval_Support_Activity_Mid-South, http://dbpedia.org/resource/20th_(Light)_Division, http://dbpedia.org/resource/29th_Division_(United_Kingdom), http://dbpedia.org/resource/62nd_(2nd_West_Riding)_Division, http://dbpedia.org/resource/Berkeley_Vincent, http://dbpedia.org/resource/Charles_Kavanagh, http://dbpedia.org/resource/Guards_Division_(United_Kingdom), http://dbpedia.org/resource/John_Leach_(Royal_Navy_officer), http://dbpedia.org/resource/John_Ponsonby_(British_Army_officer), http://dbpedia.org/resource/Judge_Advocate_General's_Corps,_United_States_Army, http://dbpedia.org/resource/Leonard_Porter_Ayres, http://dbpedia.org/resource/Louis_McGuffie, http://dbpedia.org/resource/Somerset_Gough-Calthorpe, http://dbpedia.org/resource/Wolf-Günther_Trierenberg, http://dbpedia.org/resource/121st_Field_Artillery_Regiment_(United_States), http://dbpedia.org/resource/Amyas_Borton, http://dbpedia.org/resource/Arthur_Longmore, http://dbpedia.org/resource/Charles_Edmonds, http://dbpedia.org/resource/George_Jeffreys,_1st_Baron_Jeffreys_(second_creation), http://dbpedia.org/resource/Martin_Moffat, http://dbpedia.org/resource/Alfred_Keller, http://dbpedia.org/resource/Bernard_Warburton-Lee, http://dbpedia.org/resource/Harry_Miner, http://dbpedia.org/resource/Alexei_Evert, http://dbpedia.org/resource/Duke_of_Lancaster's_Own_Yeomanry, http://dbpedia.org/resource/John_W._Reeves,_Jr., http://dbpedia.org/resource/Natal_Field_Artillery, http://dbpedia.org/resource/Wilson_Brown_(admiral), http://dbpedia.org/resource/Frank_W._Milburn, http://dbpedia.org/resource/Rudolf_Holste, http://dbpedia.org/resource/Theodor_Busse, http://dbpedia.org/resource/Beauvoir_De_Lisle, http://dbpedia.org/resource/Clifford_Sanderson, http://dbpedia.org/resource/Francis_Fogarty, http://dbpedia.org/resource/Hugh_Saunders, http://dbpedia.org/resource/Loretta_Perfectus_Walsh, http://dbpedia.org/resource/No._267_Squadron_RAF, http://dbpedia.org/resource/Alec_Cunningham-Reid, http://dbpedia.org/resource/Charles_Hubert_Boulby_Blount, http://dbpedia.org/resource/Fletcher_Stewart_Thomas, http://dbpedia.org/resource/4th_Battalion_42nd_Field_Artillery_Regiment_(United_States), http://dbpedia.org/resource/Edward_Donald_Bellew, http://dbpedia.org/resource/Elliott_Cutler, http://dbpedia.org/resource/George_H._Cameron, http://dbpedia.org/resource/Harold_Ernest_Goettler, http://dbpedia.org/resource/Percy_W._Nelles, http://dbpedia.org/resource/William_Shemin, http://dbpedia.org/resource/Christian_F._Schilt, http://dbpedia.org/resource/Herbert_Dargue, http://dbpedia.org/resource/Nikolai_Kolomeitsev, http://dbpedia.org/resource/Ross_Tollerton, http://dbpedia.org/resource/121st_Infantry_Division_(German_Empire), http://dbpedia.org/resource/14th_Light_Horse_Regiment_(Australia), http://dbpedia.org/resource/2nd/14th_Light_Horse_Regiment, http://dbpedia.org/resource/2nd_Army_(Russian_Empire), http://dbpedia.org/resource/2nd_Light_Horse_Brigade, http://dbpedia.org/resource/8th_Light_Horse_Regiment_(Australia), http://dbpedia.org/resource/Eric_Girdwood, http://dbpedia.org/resource/George_Alexander_Weir, http://dbpedia.org/resource/II_ANZAC_(XXII_Corps)_Mounted_Regiment, http://dbpedia.org/resource/John_Royston, http://dbpedia.org/resource/Viktor_Linnarz, http://dbpedia.org/resource/10th_Indian_Division, http://dbpedia.org/resource/10th_Reserve_Division_(German_Empire), http://dbpedia.org/resource/13th_Reserve_Division_(German_Empire), http://dbpedia.org/resource/162nd_Infantry_Regiment_(United_States), http://dbpedia.org/resource/1st_Australian_Tunnelling_Company, http://dbpedia.org/resource/1st_Australian_Wireless_Signal_Squadron, http://dbpedia.org/resource/1st_Cavalry_Division_(German_Empire), http://dbpedia.org/resource/1st_Light_Car_Patrol_(Australia), http://dbpedia.org/resource/1st_Mounted_Division, http://dbpedia.org/resource/29th_Battalion_(Australia), http://dbpedia.org/resource/2nd_Cavalry_Division_(German_Empire), http://dbpedia.org/resource/34th_Battalion_(Australia), http://dbpedia.org/resource/35th_Battalion_(Australia), http://dbpedia.org/resource/38th_Battalion_(Australia), http://dbpedia.org/resource/39th_Battalion_(Australia), http://dbpedia.org/resource/3rd_Cavalry_Division_(German_Empire), http://dbpedia.org/resource/3rd_Lahore_Divisional_Area, http://dbpedia.org/resource/3rd_Mounted_Division, http://dbpedia.org/resource/40th_Battalion_(Australia), http://dbpedia.org/resource/42nd_Battalion_(Australia), http://dbpedia.org/resource/43rd_Battalion_(Australia), http://dbpedia.org/resource/44th_Battalion_(Australia), http://dbpedia.org/resource/50th_Battalion_(Australia), http://dbpedia.org/resource/52nd_Reserve_Division_(German_Empire), http://dbpedia.org/resource/54th_Battalion_(Australia), http://dbpedia.org/resource/57th_Battalion_(Australia), http://dbpedia.org/resource/58th_Battalion_(Australia), http://dbpedia.org/resource/59th_Battalion_(Australia), http://dbpedia.org/resource/5th_Battalion_(Australia), http://dbpedia.org/resource/60th_Battalion_(Australia), http://dbpedia.org/resource/Auckland_Mounted_Rifles_Regiment, http://dbpedia.org/resource/Australian_Army_Veterinary_Corps, http://dbpedia.org/resource/Australian_Cycling_Corps, http://dbpedia.org/resource/Edmund_Drake-Brockman, http://dbpedia.org/resource/Ewen_Sinclair-Maclagan, http://dbpedia.org/resource/Frank_McIntyre, http://dbpedia.org/resource/Gheorghe_Flondor, http://dbpedia.org/resource/Harold_Edward_Elliott, http://dbpedia.org/resource/Henry_Normand_MacLaurin, http://dbpedia.org/resource/Herbert_Cox, http://dbpedia.org/resource/II_ANZAC_Corps, http://dbpedia.org/resource/I_ANZAC_Corps, http://dbpedia.org/resource/Nelson_Victor_Carter, http://dbpedia.org/resource/Walter_Peeler, http://dbpedia.org/resource/Wellington_Mounted_Rifles_Regiment, http://dbpedia.org/resource/XXII_Corps_(United_Kingdom), http://dbpedia.org/resource/101st_Infantry_Division_(France), http://dbpedia.org/resource/101st_Infantry_Division_(German_Empire), http://dbpedia.org/resource/103rd_Infantry_Division_(German_Empire), http://dbpedia.org/resource/105th_Infantry_Division_(German_Empire), http://dbpedia.org/resource/107th_Infantry_Division_(German_Empire), http://dbpedia.org/resource/108th_Infantry_Division_(German_Empire), http://dbpedia.org/resource/109th_Infantry_Division_(German_Empire), http://dbpedia.org/resource/110th_Mahratta_Light_Infantry, http://dbpedia.org/resource/113th_Infantry_Division_(German_Empire), http://dbpedia.org/resource/115th_Infantry_Division_(German_Empire), http://dbpedia.org/resource/117th_Infantry_Division_(German_Empire), http://dbpedia.org/resource/119th_Infantry_Division_(German_Empire), http://dbpedia.org/resource/123rd_Infantry_Division_(German_Empire), http://dbpedia.org/resource/12th_Reserve_Division_(German_Empire), http://dbpedia.org/resource/14th_Indian_Division, http://dbpedia.org/resource/14th_Reserve_Division_(German_Empire), http://dbpedia.org/resource/152nd_Punjabis, http://dbpedia.org/resource/153rd_Punjabis, http://dbpedia.org/resource/154th_Infantry, http://dbpedia.org/resource/15th_Indian_Division, http://dbpedia.org/resource/15th_Reserve_Division_(German_Empire), http://dbpedia.org/resource/16th_Indian_Division, http://dbpedia.org/resource/16th_Reserve_Division_(German_Empire), http://dbpedia.org/resource/17th_Indian_Division, http://dbpedia.org/resource/17th_Reserve_Division_(German_Empire), http://dbpedia.org/resource/18th_Indian_Division, http://dbpedia.org/resource/18th_Reserve_Division_(German_Empire), http://dbpedia.org/resource/19th_Ersatz_Division_(German_Empire), http://dbpedia.org/resource/1st_Composite_Mounted_Brigade, http://dbpedia.org/resource/1st_Landwehr_Division_(German_Empire), http://dbpedia.org/resource/20th_Indian_Brigade, http://dbpedia.org/resource/21st_Punjabis, http://dbpedia.org/resource/21st_Reserve_Division_(German_Empire), http://dbpedia.org/resource/22nd_Reserve_Division_(German_Empire), http://dbpedia.org/resource/23rd_Reserve_Division_(German_Empire), http://dbpedia.org/resource/242nd_Infantry_Division_(German_Empire), http://dbpedia.org/resource/243rd_Infantry_Division_(German_Empire), http://dbpedia.org/resource/24th_Reserve_Division_(German_Empire), http://dbpedia.org/resource/25th_Reserve_Division_(German_Empire), http://dbpedia.org/resource/26th_Division_(German_Empire), http://dbpedia.org/resource/27th_Division_(German_Empire), http://dbpedia.org/resource/28th_Reserve_Division_(German_Empire), http://dbpedia.org/resource/2nd_Landwehr_Division_(German_Empire), http://dbpedia.org/resource/30th_Bavarian_Reserve_Division, http://dbpedia.org/resource/32nd_Division_(German_Empire), http://dbpedia.org/resource/33rd_Division_(German_Empire), http://dbpedia.org/resource/33rd_Reserve_Division_(German_Empire), http://dbpedia.org/resource/34th_Division_(German_Empire), http://dbpedia.org/resource/35th_Division_(German_Empire), http://dbpedia.org/resource/35th_Reserve_Division_(German_Empire), http://dbpedia.org/resource/36th_Reserve_Division_(German_Empire), http://dbpedia.org/resource/37th_Division_(German_Empire), http://dbpedia.org/resource/38th_Division_(German_Empire), http://dbpedia.org/resource/39th_Bavarian_Reserve_Division, http://dbpedia.org/resource/39th_Division_(German_Empire), http://dbpedia.org/resource/3rd_Guards_Infantry_Division_(German_Empire), http://dbpedia.org/resource/3rd_Landwehr_Division_(German_Empire), http://dbpedia.org/resource/40th_Division_(German_Empire), http://dbpedia.org/resource/41st_Division_(German_Empire), http://dbpedia.org/resource/42nd_Division_(German_Empire), http://dbpedia.org/resource/43rd_Reserve_Division_(German_Empire), http://dbpedia.org/resource/44th_Reserve_Division_(German_Empire), http://dbpedia.org/resource/46th_Reserve_Division_(German_Empire), http://dbpedia.org/resource/47th_Reserve_Division_(German_Empire), http://dbpedia.org/resource/48th_Reserve_Division_(German_Empire), http://dbpedia.org/resource/49th_Reserve_Division_(German_Empire), http://dbpedia.org/resource/4th_Cavalry_Division_(India), http://dbpedia.org/resource/4th_Landwehr_Division_(German_Empire), http://dbpedia.org/resource/4th_Mounted_Division, http://dbpedia.org/resource/51st_Reserve_Division_(German_Empire), http://dbpedia.org/resource/53rd_Reserve_Division_(German_Empire), http://dbpedia.org/resource/54th_Reserve_Division_(German_Empire), http://dbpedia.org/resource/56th_Infantry_Division_(German_Empire), http://dbpedia.org/resource/58th_Infantry_Division_(German_Empire), http://dbpedia.org/resource/5th_Bavarian_Reserve_Division, http://dbpedia.org/resource/5th_Cavalry_Division_(India), http://dbpedia.org/resource/5th_Ersatz_Division_(German_Empire), http://dbpedia.org/resource/5th_Guards_Infantry_Division_(German_Empire), http://dbpedia.org/resource/65th_(2nd_Lowland)_Division, http://dbpedia.org/resource/67th_(2nd_Home_Counties)_Division, http://dbpedia.org/resource/68th_(2nd_Welsh)_Division, http://dbpedia.org/resource/69th_(2nd_East_Anglian)_Division, http://dbpedia.org/resource/6th_Bavarian_Reserve_Division, http://dbpedia.org/resource/6th_Cavalry_Division_(German_Empire), http://dbpedia.org/resource/6th_Poona_Divisional_Area, http://dbpedia.org/resource/75th_Reserve_Division_(German_Empire), http://dbpedia.org/resource/76th_Infantry_Division_(United_States), http://dbpedia.org/resource/76th_Reserve_Division_(German_Empire), http://dbpedia.org/resource/77th_Reserve_Division_(German_Empire), http://dbpedia.org/resource/78th_Reserve_Division_(German_Empire), http://dbpedia.org/resource/7th_Cavalry_Division_(German_Empire), http://dbpedia.org/resource/7th_Landwehr_Division_(German_Empire), http://dbpedia.org/resource/7th_Meerut_Divisional_Area, http://dbpedia.org/resource/7th_Reserve_Division_(German_Empire), http://dbpedia.org/resource/80th_Reserve_Division_(German_Empire), http://dbpedia.org/resource/81st_Reserve_Division_(German_Empire), http://dbpedia.org/resource/82nd_Reserve_Division_(German_Empire), http://dbpedia.org/resource/8th_Bavarian_Reserve_Division, http://dbpedia.org/resource/8th_Cavalry_Division_(German_Empire), http://dbpedia.org/resource/8th_Ersatz_Division_(German_Empire), http://dbpedia.org/resource/8th_Landwehr_Division_(German_Empire), http://dbpedia.org/resource/9th_Bavarian_Reserve_Division, http://dbpedia.org/resource/9th_Cavalry_Division_(German_Empire), http://dbpedia.org/resource/9th_Reserve_Division_(German_Empire), http://dbpedia.org/resource/Alfred_Robert_Wilkinson, http://dbpedia.org/resource/Archibald_Montgomery-Massingberd, http://dbpedia.org/resource/Charles_Johnston_Badger, http://dbpedia.org/resource/Christopher_Woolner, http://dbpedia.org/resource/Donald_Charles_Cameron_(politician), http://dbpedia.org/resource/Francis_Lloyd_(British_Army_officer), http://dbpedia.org/resource/Guard_Ersatz_Division_(German_Empire), http://dbpedia.org/resource/Henry_Hodgson_(British_Army_officer), http://dbpedia.org/resource/John_Alexander_Christie, http://dbpedia.org/resource/John_Shea_(Indian_Army_officer), http://dbpedia.org/resource/Russell_P._Hartle, http://dbpedia.org/resource/South_Wales_Brigade, http://dbpedia.org/resource/Walter_Mills_(VC), http://dbpedia.org/resource/Warwickshire_Yeomanry, http://dbpedia.org/resource/William_Forshaw, http://dbpedia.org/resource/Wolfgang_Erdmann, http://dbpedia.org/resource/11th_Army_Corps_(Russian_Empire), http://dbpedia.org/resource/12th_Army_Corps_(France), http://dbpedia.org/resource/2nd_Battalion,_11th_Field_Artillery, http://dbpedia.org/resource/36th_Army_Corps_(France), http://dbpedia.org/resource/51st_Corps_(German_Empire), http://dbpedia.org/resource/52nd_Corps_(German_Empire), http://dbpedia.org/resource/53rd_Corps_(German_Empire), http://dbpedia.org/resource/54th_Corps_(German_Empire), http://dbpedia.org/resource/55th_Corps_(German_Empire), http://dbpedia.org/resource/60th_Corps_(German_Empire), http://dbpedia.org/resource/61st_Corps_(German_Empire), http://dbpedia.org/resource/62nd_Corps_(German_Empire), http://dbpedia.org/resource/63rd_Corps_(German_Empire), http://dbpedia.org/resource/64th_Corps_(German_Empire), http://dbpedia.org/resource/66th_Corps_(German_Empire), http://dbpedia.org/resource/67th_Corps_(German_Empire), http://dbpedia.org/resource/68th_Corps_(German_Empire), http://dbpedia.org/resource/Alan_Gomme-Duncan, http://dbpedia.org/resource/Alan_McKibbin, http://dbpedia.org/resource/Cavalry_Corps_Schmettow_(German_Empire), http://dbpedia.org/resource/Edmund_De_Wind, http://dbpedia.org/resource/Frederick_Oscar_Warren_Loomis, http://dbpedia.org/resource/Helmuth_Brinkmann, http://dbpedia.org/resource/II_Corps_(German_Empire), http://dbpedia.org/resource/II_Royal_Bavarian_Reserve_Corps, http://dbpedia.org/resource/I_Cavalry_Corps_(German_Empire), http://dbpedia.org/resource/John_Cunningham_(VC_1916), http://dbpedia.org/resource/Landwehr_Corps, http://dbpedia.org/resource/Lewis_Heath, http://dbpedia.org/resource/Louis_R._de_Steiguer, http://dbpedia.org/resource/Malcolm_Mercer, http://dbpedia.org/resource/Otto_Schultze, http://dbpedia.org/resource/Robert_Hill_Hanna, http://dbpedia.org/resource/VIII_Reserve_Corps_(German_Empire), http://dbpedia.org/resource/VII_Reserve_Corps_(German_Empire), http://dbpedia.org/resource/VI_Cavalry_Corps_(German_Empire), http://dbpedia.org/resource/VI_Reserve_Corps_(German_Empire), http://dbpedia.org/resource/William_Benjamin_Kean, http://dbpedia.org/resource/William_Michaelis, http://dbpedia.org/resource/XII_(1st_Royal_Saxon)_Corps, http://dbpedia.org/resource/XII_(Royal_Saxon)_Reserve_Corps, http://dbpedia.org/resource/XIV_Reserve_Corps_(German_Empire), http://dbpedia.org/resource/XIX_(2nd_Royal_Saxon)_Corps, http://dbpedia.org/resource/XVIII_Reserve_Corps_(German_Empire), http://dbpedia.org/resource/XVII_Reserve_Corps_(German_Empire), http://dbpedia.org/resource/XVI_Corps_(German_Empire), http://dbpedia.org/resource/XVI_Corps_(United_Kingdom), http://dbpedia.org/resource/XV_Royal_Bavarian_Reserve_Corps, http://dbpedia.org/resource/XXIII_Reserve_Corps_(German_Empire), http://dbpedia.org/resource/XXIV_Reserve_Corps_(German_Empire), http://dbpedia.org/resource/XXVII_Reserve_Corps_(German_Empire), http://dbpedia.org/resource/XXVI_Reserve_Corps_(German_Empire), http://dbpedia.org/resource/XXXIX_Reserve_Corps_(German_Empire), http://dbpedia.org/resource/XXXVIII_Reserve_Corps_(German_Empire), http://dbpedia.org/resource/X_Reserve_Corps_(German_Empire), http://dbpedia.org/resource/14th_Army_(German_Empire), http://dbpedia.org/resource/17th_Army_(German_Empire), http://dbpedia.org/resource/19th_Army_(German_Empire), http://dbpedia.org/resource/1st_South_Western_Mounted_Brigade, http://dbpedia.org/resource/2nd_South_Western_Mounted_Brigade, http://dbpedia.org/resource/Ahmet_Ali_Çelikten, http://dbpedia.org/resource/Charles_P._Cecil, http://dbpedia.org/resource/Eastern_Mounted_Brigade, http://dbpedia.org/resource/First_Army_(Italy), http://dbpedia.org/resource/Frank_J._Becker, http://dbpedia.org/resource/James_L._Kauffman, http://dbpedia.org/resource/John_Chipman_Kerr, http://dbpedia.org/resource/Lowland_Mounted_Brigade, http://dbpedia.org/resource/Ninth_Army_(Italy), http://dbpedia.org/resource/Robert_L._Bacon, http://dbpedia.org/resource/Robert_W._Hayler, http://dbpedia.org/resource/South_Eastern_Mounted_Brigade, http://dbpedia.org/resource/South_Wales_Mounted_Brigade, http://dbpedia.org/resource/Walter_G._Andrews, http://dbpedia.org/resource/Welsh_Border_Mounted_Brigade, http://dbpedia.org/resource/Yorkshire_Mounted_Brigade, http://dbpedia.org/resource/Arthur_Edward_Cumming, http://dbpedia.org/resource/Edmund_Hoffmeister, http://dbpedia.org/resource/Ernest_Seaman, http://dbpedia.org/resource/Friedrich_Herrlein, http://dbpedia.org/resource/Georg_Pfeiffer, http://dbpedia.org/resource/Gerard_Muirhead-Gould, http://dbpedia.org/resource/Hans_Jordan, http://dbpedia.org/resource/John_Miles_Steel, http://dbpedia.org/resource/Oliver_Nugent, http://dbpedia.org/resource/Paul_Völckers, http://dbpedia.org/resource/Ralph_Van_Deman, http://dbpedia.org/resource/Robert_Martinek, http://dbpedia.org/resource/Rolf_Wuthmann, http://dbpedia.org/resource/130th_Baluchis, http://dbpedia.org/resource/Cemil_Conk, http://dbpedia.org/resource/Charles_George_Gass, http://dbpedia.org/resource/David_W._Bagley, http://dbpedia.org/resource/George_Brink, http://dbpedia.org/resource/Mehmet_Arif_Bey, http://dbpedia.org/resource/Mehmet_Atıf_Ateşdağlı, http://dbpedia.org/resource/Nikolaos_Trikoupis, http://dbpedia.org/resource/Rand_Light_Infantry, http://dbpedia.org/resource/Ridley_Pakenham-Walsh, http://dbpedia.org/resource/Rüştü_Sakarya, http://dbpedia.org/resource/Albert_Parker_Niblack, http://dbpedia.org/resource/John_Moyney, http://dbpedia.org/resource/Thomas_Langford-Sainsbury, http://dbpedia.org/resource/Thomas_Woodcock_(VC), http://dbpedia.org/resource/Henry_Winslow_Woollett, http://dbpedia.org/resource/Horatio_Barber, http://dbpedia.org/resource/John_A._Macready, http://dbpedia.org/resource/Reginald_Warneford, http://dbpedia.org/resource/Ronald_Ivelaw-Chapman, http://dbpedia.org/resource/Samuel_Kinkead, http://dbpedia.org/resource/Theodor_von_Hippel, http://dbpedia.org/resource/Adrian_Holman, http://dbpedia.org/resource/Alan_Lees, http://dbpedia.org/resource/Frederick_Thornton_Peters, http://dbpedia.org/resource/John_Adrian_Chamier, http://dbpedia.org/resource/Carl_Frederick_Holden, http://dbpedia.org/resource/Eugen_Müller, http://dbpedia.org/resource/George_Edwin_Ellison, http://dbpedia.org/resource/George_Howell_(VC), http://dbpedia.org/resource/James_Jonas_Madison, http://dbpedia.org/resource/Joseph_Burnett, http://dbpedia.org/resource/Maurice_E._Shearer, http://dbpedia.org/resource/Syd_Lucas, http://dbpedia.org/resource/Alfred_Joseph_Richards, http://dbpedia.org/resource/Arthur_Walderne_St_Clair_Tisdall, http://dbpedia.org/resource/Francis_Palmer_Selleck, http://dbpedia.org/resource/William_Cosgrove, http://dbpedia.org/resource/91a_Squadriglia, http://dbpedia.org/resource/Alfred_Joseph_Knight, http://dbpedia.org/resource/Charles_Smith_Rutherford, http://dbpedia.org/resource/Erich_Petersen, http://dbpedia.org/resource/Hans_Korte_(general), http://dbpedia.org/resource/Tedford_H._Cann, http://dbpedia.org/resource/Wilder_D._Baker, http://dbpedia.org/resource/Herbert_Augustine_Carter, http://dbpedia.org/resource/Kamio_Mitsuomi, http://dbpedia.org/resource/Oliver_Bryson, http://dbpedia.org/resource/Walter_Atlee_Edwards, http://dbpedia.org/resource/Clifford_McEwen, http://dbpedia.org/resource/Edwin_Campion_Vaughan, http://dbpedia.org/resource/George_Roupell, http://dbpedia.org/resource/Guy_Henry_(equestrian), http://dbpedia.org/resource/Samuel_Reeves_Keesler, http://dbpedia.org/resource/Bertram_J._Rodgers, http://dbpedia.org/resource/John_W._Roper, http://dbpedia.org/resource/William_Fadjo_Cravens, http://dbpedia.org/resource/Geoffrey_Forrest_Hughes, http://dbpedia.org/resource/John_Whitworth-Jones, http://dbpedia.org/resource/George_Colborne_Nugent, http://dbpedia.org/resource/61st_Pioneers, http://dbpedia.org/resource/Edward_Charles_Ingouville-Williams, http://dbpedia.org/resource/Hans_Bütow, http://dbpedia.org/resource/Harold_de_Riemer_Morgan, http://dbpedia.org/resource/John_Conover_Nichols, http://dbpedia.org/resource/Karl-Friedrich_Brill, http://dbpedia.org/resource/Kristian_Løken, http://dbpedia.org/resource/Kurt-Caesar_Hoffmann, http://dbpedia.org/resource/Kurt_Böhmer, http://dbpedia.org/resource/List_of_regiments_of_the_Indian_Army_(1903), http://dbpedia.org/resource/List_of_regiments_of_the_Indian_Army_(1922), http://dbpedia.org/resource/Maxwell_Brander, http://dbpedia.org/resource/Reginald_Kerr, http://dbpedia.org/resource/Richard_Pellengahr, http://dbpedia.org/resource/Samuel_J._Montgomery, http://dbpedia.org/resource/Stanford_Caldwell_Hooper, http://dbpedia.org/resource/Theodore_G._Ellyson, http://dbpedia.org/resource/Walther_Fischer, http://dbpedia.org/resource/Werner_Lange, http://dbpedia.org/resource/102nd_Infantry_Regiment_(United_States), http://dbpedia.org/resource/52nd_Infantry_Regiment_(United_States), http://dbpedia.org/resource/Alexander_Young_(VC), http://dbpedia.org/resource/Armee-Abteilung_A, http://dbpedia.org/resource/Ceylon_Mounted_Rifles, http://dbpedia.org/resource/Charles_Dudley_Rhodes, http://dbpedia.org/resource/Clement_Flagler, http://dbpedia.org/resource/Erich_Kahsnitz, http://dbpedia.org/resource/Eugen_Garski, http://dbpedia.org/resource/Georg_Jauer, http://dbpedia.org/resource/Hermann_Schulte-Heuthaus, http://dbpedia.org/resource/High_Command_of_Coastal_Defence, http://dbpedia.org/resource/James_Edward_Wharton, http://dbpedia.org/resource/Karl_Ludwig_d'Elsa, http://dbpedia.org/resource/Robert_V._Maraist, http://dbpedia.org/resource/Thomas_Orde_Lawder_Wilkinson, http://dbpedia.org/resource/William_Abram_Mann, http://dbpedia.org/resource/William_McFadzean, http://dbpedia.org/resource/William_R._Furlong, http://dbpedia.org/resource/Gaston_Cros, http://dbpedia.org/resource/Montagu_Brocas_Burrows, http://dbpedia.org/resource/Thomas_Jacomb_Hutton, http://dbpedia.org/resource/Georg-Wilhelm_Postel, http://dbpedia.org/resource/Henry_Leader, http://dbpedia.org/resource/Samuel_D._Sturgis,_Jr., http://dbpedia.org/resource/Werner_Richter, http://dbpedia.org/resource/Duncan_Maxwell, http://dbpedia.org/resource/Merton_Beckwith-Smith, http://dbpedia.org/resource/102nd_Prince_of_Wales's_Own_Grenadiers, http://dbpedia.org/resource/124th_Duchess_of_Connaught's_Own_Baluchistan_Infantry, http://dbpedia.org/resource/Archibald_Alexander_Gordon, http://dbpedia.org/resource/Clarence_Jeffries, http://dbpedia.org/resource/David_Whitehead, http://dbpedia.org/resource/Desiderius_Hampel, http://dbpedia.org/resource/Hans_Friedemann_Götze, http://dbpedia.org/resource/Josef_Fitzthum, http://dbpedia.org/resource/Karl-Gustav_Sauberzweig, http://dbpedia.org/resource/Sydney_Muspratt, http://dbpedia.org/resource/Wilhelm_Trabandt, http://dbpedia.org/resource/Benjamin_Muse, http://dbpedia.org/resource/Claud_Castleton, http://dbpedia.org/resource/George_Gater, http://dbpedia.org/resource/Max_von_Boehn_(general), http://dbpedia.org/resource/Maximilian_Felzmann, http://dbpedia.org/resource/Norman_Carlyle_Craig, http://dbpedia.org/resource/David_Sinton_Ingalls, http://dbpedia.org/resource/Mir_Dast, http://dbpedia.org/resource/Victor_Groom, http://dbpedia.org/resource/Robert_Eyssen, http://dbpedia.org/resource/202nd_(2/1st_Kent)_Brigade, http://dbpedia.org/resource/30th_Indian_Brigade, http://dbpedia.org/resource/34th_Indian_Brigade, http://dbpedia.org/resource/54th_Indian_Brigade, http://dbpedia.org/resource/Arthur_Lynden-Bell, http://dbpedia.org/resource/James_Goodrich_(Royal_Navy_officer), http://dbpedia.org/resource/Ahmet_Zeki_Soydemir, http://dbpedia.org/resource/Arthur_Kullmer, http://dbpedia.org/resource/Arthur_Sloggett, http://dbpedia.org/resource/Ethem_Necdet_Karabudak, http://dbpedia.org/resource/Friedrich-August_Schack, http://dbpedia.org/resource/Julius_von_Bernuth, http://dbpedia.org/resource/Kurt_Brennecke, http://dbpedia.org/resource/Mehmet_Emin_Koral, http://dbpedia.org/resource/Charles_D._Barrett, http://dbpedia.org/resource/Ira_B._Thompson, http://dbpedia.org/resource/Nikolai_Essen, http://dbpedia.org/resource/106th_Battalion_(Nova_Scotia_Rifles),_CEF, http://dbpedia.org/resource/John_George_Walters_Clark, http://dbpedia.org/resource/Friedrich_Hochbaum, http://dbpedia.org/resource/Helmut_Friebe, http://dbpedia.org/resource/Ludwig_Wolff_(general), http://dbpedia.org/resource/Otto_Stapf_(officer), http://dbpedia.org/resource/Richard_Whitehead_Young, http://dbpedia.org/resource/Rudolf_Gerhardt, http://dbpedia.org/resource/Rudolf_Toussaint, http://dbpedia.org/resource/Wilhelm_Heun, http://dbpedia.org/resource/Alfred_Bruer, http://dbpedia.org/resource/Henning_Schönfeld, http://dbpedia.org/resource/John_Glossop, http://dbpedia.org/resource/Karl_Böttcher, http://dbpedia.org/resource/Vollrath_Lübbe, http://dbpedia.org/resource/Erpo_Freiherr_von_Bodenhausen, http://dbpedia.org/resource/Erwin_Jollasse, http://dbpedia.org/resource/Franz_Landgraf, http://dbpedia.org/resource/Gustav_Hundt, http://dbpedia.org/resource/Johannes_Bäßler, http://dbpedia.org/resource/Johannes_Schulz, http://dbpedia.org/resource/Rudolf_Freiherr_von_Waldenfels, http://dbpedia.org/resource/Walter_Denkert, http://dbpedia.org/resource/Werner_Friebe, http://dbpedia.org/resource/Werner_Hühner, http://dbpedia.org/resource/1st_Machine-Gun_Squadron_(New_Zealand), http://dbpedia.org/resource/Sir_Walter_Barttelot,_3rd_Baronet, http://dbpedia.org/resource/Eberhard_von_Hofacker, http://dbpedia.org/resource/Hans_Rauch, http://dbpedia.org/resource/Sir_John_Milbanke,_10th_Baronet, http://dbpedia.org/resource/William_Tasker, http://dbpedia.org/resource/Gerhard_Lindner, http://dbpedia.org/resource/William_Kelly_Harrison,_Jr., http://dbpedia.org/resource/André_Debry, http://dbpedia.org/resource/Wilhelm_Bohnstedt, http://dbpedia.org/resource/Claude_Vincent, http://dbpedia.org/resource/Harold_Carrington, http://dbpedia.org/resource/Hugh_Massy, http://dbpedia.org/resource/John_Bradley_(RAF_officer), http://dbpedia.org/resource/Peter_Conover_Hains, http://dbpedia.org/resource/Arnold_Freiherr_von_Biegeleben, http://dbpedia.org/resource/Carl_Rodenburg, http://dbpedia.org/resource/Harry_Lee_(cricketer), http://dbpedia.org/resource/Horst_Großmann, http://dbpedia.org/resource/Otto-Hermann_Brücker, http://dbpedia.org/resource/Rudolf_Konrad, http://dbpedia.org/resource/Theodor_Freiherr_von_Wrede, http://dbpedia.org/resource/Wilhelm_Wetzel, http://dbpedia.org/resource/Jean_César_Graziani, http://dbpedia.org/resource/John_Nixon_(Indian_Army_officer), http://dbpedia.org/resource/Werner_Sanne, http://dbpedia.org/resource/William_Wyndham_Green, http://dbpedia.org/resource/Guy_Salisbury-Jones, http://dbpedia.org/resource/Ivan_de_la_Bere, http://dbpedia.org/resource/Frances_Cluett, http://dbpedia.org/resource/Karl_Rothenburg, http://dbpedia.org/resource/Burford_Sampson, http://dbpedia.org/resource/20_Battery_Royal_Artillery, http://dbpedia.org/resource/Hartwig_von_Ludwiger, http://dbpedia.org/resource/Werner_von_Erdmannsdorff, http://dbpedia.org/resource/Chester_H._West, http://dbpedia.org/resource/Frank_de_Pass, http://dbpedia.org/resource/Ernest_Squires, http://dbpedia.org/resource/54th_Infantry_Regiment_(France), http://dbpedia.org/resource/Rudolf_Pannier, http://dbpedia.org/resource/66th_Armor_Regiment, http://dbpedia.org/resource/Alexander_Rochfort, http://dbpedia.org/resource/Edward_Willis_(British_Army_officer), http://dbpedia.org/resource/Frank_William_Foster, http://dbpedia.org/resource/Gresham_Nicholson, http://dbpedia.org/resource/Horace_Martelli, http://dbpedia.org/resource/Dalrymple_Arbuthnot, http://dbpedia.org/resource/7th_Air_Defense_Artillery_Regiment, http://dbpedia.org/resource/George_H._Morgan, http://dbpedia.org/resource/Gilbert_T._Rude, http://dbpedia.org/resource/Reginald_Spencer_Browne, http://dbpedia.org/resource/Raymond_Brownell, http://dbpedia.org/resource/William_Lancelot_Jordan, http://dbpedia.org/resource/Cecil_Noble, http://dbpedia.org/resource/Gabar_Singh_Negi, http://dbpedia.org/resource/James_Willcocks, http://dbpedia.org/resource/Henry_Chambers_(Indian_Army_officer), http://dbpedia.org/resource/Jagdstaffel_12, http://dbpedia.org/resource/Jagdstaffel_13, http://dbpedia.org/resource/Jagdstaffel_18, http://dbpedia.org/resource/Jagdstaffel_4, http://dbpedia.org/resource/David_Dixon_Porter_(Medal_of_Honor), http://dbpedia.org/resource/Hazelton_Nicholl, http://dbpedia.org/resource/Nikolaos_Christodoulou, http://dbpedia.org/resource/Jobst-Hilmar_von_Bose, http://dbpedia.org/resource/John_Eaton_(Royal_Navy_officer), http://dbpedia.org/resource/Francišak_Kušal, http://dbpedia.org/resource/Arthur_Daly, http://dbpedia.org/resource/Friedrich-Wilhelm_Morzik, http://dbpedia.org/resource/Philip_Glazebrook, http://dbpedia.org/resource/Cecil_Foott, http://dbpedia.org/resource/Marcel_Pourchier, http://dbpedia.org/resource/Jagdstaffel_6, http://dbpedia.org/resource/Z_Battery_Royal_Artillery, http://dbpedia.org/resource/Reginald_Graham, http://dbpedia.org/resource/Bertram_Allen, http://dbpedia.org/resource/Norbert_Holm, http://dbpedia.org/resource/George_Mortimer_Morris, http://dbpedia.org/resource/Owen_Baldwin, http://dbpedia.org/resource/William_Sinclair-Burgess, http://dbpedia.org/resource/Hovsep_Arghutian, http://dbpedia.org/resource/Keri_(fedayi), http://dbpedia.org/resource/Makhluto, http://dbpedia.org/resource/Nikolaos_Zafeiriou, http://dbpedia.org/resource/166th_Infantry_Regiment_(United_States), http://dbpedia.org/resource/Walter_Schlags-Koch, http://dbpedia.org/resource/Fritz_Klasing, http://dbpedia.org/resource/Wilhelm_Berlin, http://dbpedia.org/resource/Karl_Bock_(officer), http://dbpedia.org/resource/Aubrey_Weinman, http://dbpedia.org/resource/Wilfred_Brittain, http://dbpedia.org/resource/Hans_Bergen, http://dbpedia.org/resource/Ralph_Graf_von_Oriola, http://dbpedia.org/resource/John_Coleridge_(Indian_Army_officer), http://dbpedia.org/resource/Ernest_Clive_Atkins, http://dbpedia.org/resource/German_cavalry_in_World_War_I, http://dbpedia.org/resource/Hubertus_Lamey, http://dbpedia.org/resource/TOE,_German_Cavalry_Division,_August_1914, http://dbpedia.org/resource/Walther_Krause, http://dbpedia.org/resource/John_Russell_Young_Blakely, http://dbpedia.org/resource/2nd_Royal_Bavarian_Uhlans, http://dbpedia.org/resource/John_James_Clements, http://dbpedia.org/resource/Basil_Blackett_(aviator), http://dbpedia.org/resource/Gustav_Globočnik_Edler_von_Vojka, http://dbpedia.org/resource/Otto_Drescher, http://dbpedia.org/resource/Francis_Mellersh_(RAF_officer), http://dbpedia.org/resource/Friedrich-August_Weinknecht, http://dbpedia.org/resource/Johnson_Paudash, http://dbpedia.org/resource/Sofija_Jovanović, http://dbpedia.org/resource/12th_(Otago)_Mounted_Rifles, http://dbpedia.org/resource/5th_Mounted_Rifles_(Otago_Hussars), http://dbpedia.org/resource/8th_(South_Canterbury)_Mounted_Rifles, http://dbpedia.org/resource/Leslie_Harvey_(RAF_officer), http://dbpedia.org/resource/Alfred_Baxter-Cox, http://dbpedia.org/resource/Cuthbert_Hoare, http://dbpedia.org/resource/Percy_Statton, http://dbpedia.org/resource/Jagdstaffel_32, http://dbpedia.org/resource/Cyril_Cooke, http://dbpedia.org/resource/Edward_A._Gisburne, http://dbpedia.org/resource/Harold_E._Bradshaw, http://dbpedia.org/resource/Frederick_Edward-Collins, http://dbpedia.org/resource/Albert_Tonkin, http://dbpedia.org/resource/Arthur_Britton, http://dbpedia.org/resource/Douglas_Carbery, http://dbpedia.org/resource/Ernest_Antcliffe, http://dbpedia.org/resource/Geoffrey_Hemming, http://dbpedia.org/resource/Lumley_Jones, http://dbpedia.org/resource/Noel_Keeble, http://dbpedia.org/resource/Walter_Beales, http://dbpedia.org/resource/Jagdstaffel_1, http://dbpedia.org/resource/Lajos_Keresztes-Fischer, http://dbpedia.org/resource/Paul_Scheuerpflug, http://dbpedia.org/resource/Robert_Meißner, http://dbpedia.org/resource/Carl_Anders, http://dbpedia.org/resource/Hans_Wagner_(general), http://dbpedia.org/resource/Erich_Schopper, http://dbpedia.org/resource/Ernst_Meiners, http://dbpedia.org/resource/16th_(Schleswig-Holstein)_Hussars_%22Emperor_Francis_Joseph_of_Austria,_King_of_Hungary%22, http://dbpedia.org/resource/Franz_Gall_(general), http://dbpedia.org/resource/Cyril_Frisby, http://dbpedia.org/resource/Louis_Tronnier, http://dbpedia.org/resource/Werner_Ehrig, http://dbpedia.org/resource/Aubrey_E._Strode, http://dbpedia.org/resource/Hans_de_Salengre-Drabbe, http://dbpedia.org/resource/Hermann_von_Wedel, http://dbpedia.org/resource/Josef_Schmidt_(general), http://dbpedia.org/resource/Cecil_Sewell, http://dbpedia.org/resource/Charles_Spackman, http://dbpedia.org/resource/Fort_Vrmac, http://dbpedia.org/resource/Kurt_Heyser, http://dbpedia.org/resource/Eugen_Baur, http://dbpedia.org/resource/Karl_Brassert, http://dbpedia.org/resource/Ernst_Knebel, http://dbpedia.org/resource/Walter_Henrich, http://dbpedia.org/resource/Alfred_Kuzmany, http://dbpedia.org/resource/2nd_Cavalry_Regiment_(United_States), http://dbpedia.org/resource/Adolf_Hitler, http://dbpedia.org/resource/Ahmed_Izzet_Pasha, http://dbpedia.org/resource/Ali-Agha_Shikhlinski, http://dbpedia.org/resource/British_Indian_Army, http://dbpedia.org/resource/Canadian_Army, http://dbpedia.org/resource/Charles_G._Dawes, http://dbpedia.org/resource/Charles_de_Gaulle, http://dbpedia.org/resource/Dwight_D._Eisenhower, http://dbpedia.org/resource/E._H._Shepard, http://dbpedia.org/resource/Erhard_Milch, http://dbpedia.org/resource/Ernst_Udet, http://dbpedia.org/resource/Eugene_Reybold, http://dbpedia.org/resource/George_E._Stewart, http://dbpedia.org/resource/Gladys_Powers, http://dbpedia.org/resource/Hans_Baur, http://dbpedia.org/resource/Harold_Macmillan, http://dbpedia.org/resource/Hastings_Ismay,_1st_Baron_Ismay, http://dbpedia.org/resource/Heinrich_Himmler, http://dbpedia.org/resource/Heinrich_Kreipe, http://dbpedia.org/resource/Hudson_Fysh, http://dbpedia.org/resource/Imperial_Japanese_Navy, http://dbpedia.org/resource/Italian_Army, http://dbpedia.org/resource/Joachim_von_Ribbentrop, http://dbpedia.org/resource/Josip_Broz_Tito, http://dbpedia.org/resource/Karl_Dönitz, http://dbpedia.org/resource/Louis_Mountbatten,_1st_Earl_Mountbatten_of_Burma, http://dbpedia.org/resource/László_Almásy, http://dbpedia.org/resource/Maciej_Sulkiewicz, http://dbpedia.org/resource/Pyotr_Nikolayevich_Wrangel, http://dbpedia.org/resource/Richard_Marshall_(general), http://dbpedia.org/resource/Royal_Canadian_Navy, http://dbpedia.org/resource/United_States_Army, http://dbpedia.org/resource/Winston_Churchill, http://dbpedia.org/resource/İsmet_İnönü, http://dbpedia.org/resource/86th_Infantry_Division_(United_States), http://dbpedia.org/resource/Alexander_Kolchak, http://dbpedia.org/resource/Alf_Landon, http://dbpedia.org/resource/Border_Guards_Bangladesh, http://dbpedia.org/resource/Charles_Groves_Wright_Anderson, http://dbpedia.org/resource/Charles_Lambe, http://dbpedia.org/resource/Ernst_Põdder, http://dbpedia.org/resource/Francis_Cromie, http://dbpedia.org/resource/Harry_Hines_Woodring, http://dbpedia.org/resource/Herbert_Annesley_Packer, http://dbpedia.org/resource/Husband_E._Kimmel, http://dbpedia.org/resource/Imperial_Japanese_Army_Air_Service, http://dbpedia.org/resource/Italo_Balbo, http://dbpedia.org/resource/Johan_Laidoner, http://dbpedia.org/resource/Joseph_Gallieni, http://dbpedia.org/resource/Leonid_Gobyato, http://dbpedia.org/resource/Marie-Pierre_Kœnig, http://dbpedia.org/resource/Maximilian_Fretter-Pico, http://dbpedia.org/resource/Michael_Moutoussis, http://dbpedia.org/resource/Nikolai_Reek, http://dbpedia.org/resource/Raoul_Salan, http://dbpedia.org/resource/Rodolfo_Graziani, http://dbpedia.org/resource/Roger_Keyes,_1st_Baron_Keyes, http://dbpedia.org/resource/Saitō_Makoto, http://dbpedia.org/resource/Samad_bey_Mehmandarov, http://dbpedia.org/resource/Sydney_Raw, http://dbpedia.org/resource/Theodoros_Pangalos_(general), http://dbpedia.org/resource/Werner_Fürbringer, http://dbpedia.org/resource/William_Dickson_(RAF_officer), http://dbpedia.org/resource/78th_Infantry_Division_(United_States), http://dbpedia.org/resource/Andrew_McNaughton, http://dbpedia.org/resource/Arthur_Edward_Barstow, http://dbpedia.org/resource/Charles_Young_(United_States_Army), http://dbpedia.org/resource/Colombo_Town_Guard, http://dbpedia.org/resource/George_Owen_Squier, http://dbpedia.org/resource/George_Raymond_Dallas_Moor, http://dbpedia.org/resource/Helmuth_Weidling, http://dbpedia.org/resource/Herbert_O'Conor, http://dbpedia.org/resource/Imperial_Guard_(Japan), http://dbpedia.org/resource/Isamu_Takeshita, http://dbpedia.org/resource/John_Baker_(RAF_officer), http://dbpedia.org/resource/Karl-Jesko_von_Puttkamer, http://dbpedia.org/resource/Lazare_Ponticelli, http://dbpedia.org/resource/Ralph_Royce, http://dbpedia.org/resource/Werner_Kienitz, http://dbpedia.org/resource/William_George_Barker, http://dbpedia.org/resource/55th_Battalion_(Australia), http://dbpedia.org/resource/British_South_Africa_Police, http://dbpedia.org/resource/Ewen_Cameron_Bruce, http://dbpedia.org/resource/Frederick_George_Room, http://dbpedia.org/resource/Grigory_Semyonov, http://dbpedia.org/resource/John_Little_McClellan, http://dbpedia.org/resource/Karl_Wolff, http://dbpedia.org/resource/Mikhail_Artemyevich_Muravyov, http://dbpedia.org/resource/Mikhail_Dmitriyevich_Bonch-Bruyevich, http://dbpedia.org/resource/No._5_Squadron_RAAF, http://dbpedia.org/resource/Richard_Butler_(British_Army_officer), http://dbpedia.org/resource/Vernon_Treatt, http://dbpedia.org/resource/Viktor_Lutze, http://dbpedia.org/resource/Vladimir_Kappel, http://dbpedia.org/resource/Wilfrid_Kent_Hughes, http://dbpedia.org/resource/4th_Army_(German_Empire), http://dbpedia.org/resource/80th_Division_(United_States), http://dbpedia.org/resource/Aaron_S._Merrill, http://dbpedia.org/resource/Adolphe_Pégoud, http://dbpedia.org/resource/Albrecht,_Duke_of_Württemberg, http://dbpedia.org/resource/Alessandro_Guidoni, http://dbpedia.org/resource/Alexander_Godley, http://dbpedia.org/resource/Alfred_A._Cunningham, http://dbpedia.org/resource/Anton_Dostler, http://dbpedia.org/resource/Arthur_Hayes-Sadler, http://dbpedia.org/resource/Arthur_L._Bristol, http://dbpedia.org/resource/Bertram_Ramsay, http://dbpedia.org/resource/Bruno_von_Mudra, http://dbpedia.org/resource/Campbell_Tait, http://dbpedia.org/resource/Charles_D._Herron, http://dbpedia.org/resource/Charles_James_Briggs, http://dbpedia.org/resource/Charles_P._Snyder_(admiral), http://dbpedia.org/resource/Charles_Walter_Allfrey, http://dbpedia.org/resource/Christoffel_Venter, http://dbpedia.org/resource/Clarence_Stewart_Williams, http://dbpedia.org/resource/Clifton_Sprague]... [INFO] Predicted Answer: [Big Picture: Atomic Battlefield@en, Big Picture: The Atom Soldier@en] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 955, P: 0.502, R: 0.559, macro F1: 0.510, macro F1*: 0.529 -[INFO] Current time: 806 ms, average: 2945.24 ms/ques, entity detection: 496.66 ms/ques (linking tool: 402.83 ms/ques), relation detection: 963.96 ms/ques (similarity: 714.45 ms/ques), EDG generation: 46.32 ms/ques +[INFO] Cumulative metrics, sample: 955, P: 0.502, R: 0.559, macro F1: 0.529 +[INFO] Current question consumed time: 686 ms, average: 2626.43 ms/ques, entity detection: 433.69 ms/ques (linking tool: 355.96 ms/ques), relation detection: 881.88 ms/ques (similarity: 706.83 ms/ques), EDG generation: 46.83 ms/ques [INFO] Entity Linking P: 0.7392670157068063, R: 0.6910994764397905, micro F1: 0.706631762652705, macro F1: 0.7143722260124336 [INFO] Relation Linking P: 0.5957242582897033, R: 0.5413612565445026, micro F1: 0.5571029668411873, macro F1: 0.5672432351207612 [INFO] Type Linking P: 0.5549738219895288, R: 0.5549738219895288, micro F1: 0.5549738219895288, macro F1: 0.5549738219895288 @@ -13402,8 +13402,8 @@ Question 955: Who appointed both John Drayton and William Clark to their respect [INFO] Golden Answer: [http://dbpedia.org/resource/James_Madison] [INFO] Predicted Answer: [http://dbpedia.org/resource/James_Madison] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 956, P: 0.503, R: 0.559, macro F1: 0.511, macro F1*: 0.530 -[INFO] Current time: 2499 ms, average: 2944.77 ms/ques, entity detection: 496.63 ms/ques (linking tool: 402.69 ms/ques), relation detection: 963.79 ms/ques (similarity: 714.17 ms/ques), EDG generation: 46.33 ms/ques +[INFO] Cumulative metrics, sample: 956, P: 0.503, R: 0.559, macro F1: 0.530 +[INFO] Current question consumed time: 2022 ms, average: 2625.79 ms/ques, entity detection: 433.63 ms/ques (linking tool: 355.81 ms/ques), relation detection: 881.71 ms/ques (similarity: 706.56 ms/ques), EDG generation: 46.84 ms/ques [INFO] Entity Linking P: 0.7395397489539749, R: 0.6908995815899581, micro F1: 0.7065899581589957, macro F1: 0.7143926935050828 [INFO] Relation Linking P: 0.5961471408647141, R: 0.5418410041841004, micro F1: 0.5575662482566254, macro F1: 0.5676982960727787 [INFO] Type Linking P: 0.555439330543933, R: 0.555439330543933, micro F1: 0.555439330543933, macro F1: 0.555439330543933 @@ -13416,8 +13416,8 @@ Question 956: Which operator of New Sanno Hotel is also the governing body of t [INFO] Golden Answer: [http://dbpedia.org/resource/United_States_Navy] [INFO] Predicted Answer: [http://dbpedia.org/resource/United_States_Navy] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 957, P: 0.503, R: 0.560, macro F1: 0.511, macro F1*: 0.530 -[INFO] Current time: 1512 ms, average: 2943.28 ms/ques, entity detection: 496.46 ms/ques (linking tool: 402.51 ms/ques), relation detection: 963.13 ms/ques (similarity: 713.64 ms/ques), EDG generation: 46.38 ms/ques +[INFO] Cumulative metrics, sample: 957, P: 0.503, R: 0.560, macro F1: 0.530 +[INFO] Current question consumed time: 1328 ms, average: 2624.44 ms/ques, entity detection: 433.49 ms/ques (linking tool: 355.65 ms/ques), relation detection: 881.09 ms/ques (similarity: 706.03 ms/ques), EDG generation: 46.90 ms/ques [INFO] Entity Linking P: 0.7398119122257053, R: 0.6912225705329154, micro F1: 0.7068965517241378, macro F1: 0.7146923401786113 [INFO] Relation Linking P: 0.596569139672588, R: 0.542319749216301, micro F1: 0.5580285614768379, macro F1: 0.5681523972598637 [INFO] Type Linking P: 0.5559038662486938, R: 0.5559038662486938, micro F1: 0.5559038662486938, macro F1: 0.5559038662486938 @@ -13430,8 +13430,8 @@ Question 957: Which predecessor of Alexander Downer was deputy to Neil Brown ? [INFO] Golden Answer: [http://dbpedia.org/resource/John_Howard] [INFO] Predicted Answer: [http://dbpedia.org/resource/John_Howard] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 958, P: 0.504, R: 0.560, macro F1: 0.512, macro F1*: 0.531 -[INFO] Current time: 2457 ms, average: 2942.77 ms/ques, entity detection: 496.33 ms/ques (linking tool: 402.32 ms/ques), relation detection: 963.09 ms/ques (similarity: 713.46 ms/ques), EDG generation: 46.35 ms/ques +[INFO] Cumulative metrics, sample: 958, P: 0.504, R: 0.560, macro F1: 0.531 +[INFO] Current question consumed time: 1995 ms, average: 2623.78 ms/ques, entity detection: 433.33 ms/ques (linking tool: 355.46 ms/ques), relation detection: 881.04 ms/ques (similarity: 705.87 ms/ques), EDG generation: 46.87 ms/ques [INFO] Entity Linking P: 0.7400835073068893, R: 0.691544885177453, micro F1: 0.7072025052192066, macro F1: 0.7149913577700535 [INFO] Relation Linking P: 0.5969902574808629, R: 0.5427974947807933, micro F1: 0.5584899095337514, macro F1: 0.5686055417355695 [INFO] Type Linking P: 0.5563674321503131, R: 0.5563674321503131, micro F1: 0.5563674321503131, macro F1: 0.5563674321503131 @@ -13444,8 +13444,8 @@ Question 958: Which are the predecessor of Chevrolet Caprice? [INFO] Golden Answer: [http://dbpedia.org/resource/Chevrolet_Lumina, http://dbpedia.org/resource/Mexico, http://dbpedia.org/resource/North_America, http://dbpedia.org/resource/Pontiac_G8, http://dbpedia.org/resource/Chevrolet_Impala] [INFO] Predicted Answer: [http://dbpedia.org/resource/Chevrolet_Lumina, http://dbpedia.org/resource/Mexico, http://dbpedia.org/resource/North_America, http://dbpedia.org/resource/Pontiac_G8, http://dbpedia.org/resource/Chevrolet_Impala] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 959, P: 0.504, R: 0.561, macro F1: 0.512, macro F1*: 0.531 -[INFO] Current time: 1127 ms, average: 2940.88 ms/ques, entity detection: 496.22 ms/ques (linking tool: 402.24 ms/ques), relation detection: 962.37 ms/ques (similarity: 712.88 ms/ques), EDG generation: 46.31 ms/ques +[INFO] Cumulative metrics, sample: 959, P: 0.504, R: 0.561, macro F1: 0.531 +[INFO] Current question consumed time: 934 ms, average: 2622.02 ms/ques, entity detection: 433.19 ms/ques (linking tool: 355.35 ms/ques), relation detection: 880.38 ms/ques (similarity: 705.30 ms/ques), EDG generation: 46.83 ms/ques [INFO] Entity Linking P: 0.7403545359749739, R: 0.691866527632951, micro F1: 0.7075078206465066, macro F1: 0.7152897482626795 [INFO] Relation Linking P: 0.5974104970455335, R: 0.543274244004171, micro F1: 0.5589502954466464, macro F1: 0.5690577325403635 [INFO] Type Linking P: 0.556830031282586, R: 0.556830031282586, micro F1: 0.556830031282586, macro F1: 0.556830031282586 @@ -13458,8 +13458,8 @@ Question 959: Who wrote the album coming right before Dead Man's Chest's OST? [INFO] Golden Answer: [http://dbpedia.org/resource/Hans_Zimmer, Klaus Badelt@en, Geoffrey Zanelli@en] [INFO] Predicted Answer: [http://dbpedia.org/resource/To_Hell_You_Ride, http://dbpedia.org/resource/What_Is_It%3F, http://dbpedia.org/resource/The_Brave, http://dbpedia.org/resource/Macao_(film), http://dbpedia.org/resource/The_Ballad_of_Thunder_Road, http://dbpedia.org/resource/Camouflage_(2001_film), http://dbpedia.org/resource/One_False_Move, http://dbpedia.org/resource/A_Family_Thing, http://dbpedia.org/resource/Daddy_and_Them, http://dbpedia.org/resource/Jayne_Mansfield's_Car, http://dbpedia.org/resource/Sling_Blade, http://dbpedia.org/resource/Some_Folks_Call_It_a_Sling_Blade, http://dbpedia.org/resource/The_Gift_(2000_film), http://dbpedia.org/resource/China_Girl_(song), http://dbpedia.org/resource/Search_and_Destroy_(The_Stooges_song), http://dbpedia.org/resource/Bang_Bang_(Iggy_Pop_song), http://dbpedia.org/resource/Candy_(Iggy_Pop_song), http://dbpedia.org/resource/Tonight_(Iggy_Pop_song), http://dbpedia.org/resource/Nightclubbing_(song), http://dbpedia.org/resource/Lust_for_Life_(song), http://dbpedia.org/resource/I_Wanna_Be_Your_Dog, http://dbpedia.org/resource/Some_Weird_Sin, http://dbpedia.org/resource/Bad_Boyz, http://dbpedia.org/resource/The_Last_of_the_High_Kings, http://dbpedia.org/resource/Late_Night_Tales:_Fatboy_Slim, http://dbpedia.org/resource/Don't_Look_Back_(1996_film), http://dbpedia.org/resource/%22The_Spaghetti_Incident%3F%22, http://dbpedia.org/resource/New_Values, http://dbpedia.org/resource/Soldier_(album), http://dbpedia.org/resource/Conflict_&_Catalysis:_Productions_&_Arrangements_1966-2006, http://dbpedia.org/resource/Copper_Blue, http://dbpedia.org/resource/Nightclubbing_(Grace_Jones_album), http://dbpedia.org/resource/Up_Your_Alley_(album), http://dbpedia.org/resource/Wax_Ecstatic, http://dbpedia.org/resource/Control_(2007_film), http://dbpedia.org/resource/Through_the_Looking_Glass_(Siouxsie_and_the_Banshees_album), http://dbpedia.org/resource/Sick_(Loaded_album), http://dbpedia.org/resource/Sons_of_Anarchy:_Songs_of_Anarchy_Vol._2, http://dbpedia.org/resource/Flex_Your_Head, http://dbpedia.org/resource/Undisputed_Attitude, http://dbpedia.org/resource/Smash_It_Up:_The_Anthology_1976–1987, http://dbpedia.org/resource/He_Put_the_Bomp!_In_the_Bomp, http://dbpedia.org/resource/Damned_Damned_Damned, http://dbpedia.org/resource/Reload_(Tom_Jones_album), http://dbpedia.org/resource/Renegades_(Rage_Against_the_Machine_album), http://dbpedia.org/resource/The_Crow:_City_of_Angels_(soundtrack), http://dbpedia.org/resource/Shame,_Humility,_Revenge, http://dbpedia.org/resource/Travelogue_(The_Human_League_album), http://dbpedia.org/resource/Roadkill_Rising, http://dbpedia.org/resource/Yeah!_(Def_Leppard_album), http://dbpedia.org/resource/Picture_This_Live, http://dbpedia.org/resource/Nude_&_Rude:_The_Best_of_Iggy_Pop, http://dbpedia.org/resource/Tony_Hawk's_American_Wasteland, http://dbpedia.org/resource/Émilie_Simon_(album), http://dbpedia.org/resource/Cooper-S] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 960, P: 0.504, R: 0.560, macro F1: 0.512, macro F1*: 0.530 -[INFO] Current time: 3691 ms, average: 2941.66 ms/ques, entity detection: 496.08 ms/ques (linking tool: 402.03 ms/ques), relation detection: 962.93 ms/ques (similarity: 713.19 ms/ques), EDG generation: 46.29 ms/ques +[INFO] Cumulative metrics, sample: 960, P: 0.504, R: 0.560, macro F1: 0.530 +[INFO] Current question consumed time: 3255 ms, average: 2622.68 ms/ques, entity detection: 433.05 ms/ques (linking tool: 355.16 ms/ques), relation detection: 880.96 ms/ques (similarity: 705.71 ms/ques), EDG generation: 46.81 ms/ques [INFO] Entity Linking P: 0.7395833333333334, R: 0.6911458333333333, micro F1: 0.7067708333333332, macro F1: 0.714544654774906 [INFO] Relation Linking P: 0.5973090277777777, R: 0.5432291666666667, micro F1: 0.5588888888888894, macro F1: 0.5689869694548036 [INFO] Type Linking P: 0.55625, R: 0.55625, micro F1: 0.55625, macro F1: 0.55625 @@ -13472,8 +13472,8 @@ Question 960: What is the series of the book which is a subsequent work of Blade [INFO] Golden Answer: [http://dbpedia.org/resource/Blade_Runner] [INFO] Predicted Answer: [http://dbpedia.org/resource/Rick_Deckard, http://dbpedia.org/resource/Blade_Runner, Blade Runner #3@en] [INFO] P: 0.3333333333333333, R: 1.0, F: 0.5 -[INFO] Cumulative metrics, sample: 961, P: 0.504, R: 0.560, macro F1: 0.512, macro F1*: 0.531 -[INFO] Current time: 2221 ms, average: 2940.91 ms/ques, entity detection: 495.92 ms/ques (linking tool: 401.94 ms/ques), relation detection: 962.51 ms/ques (similarity: 712.92 ms/ques), EDG generation: 46.30 ms/ques +[INFO] Cumulative metrics, sample: 961, P: 0.504, R: 0.560, macro F1: 0.531 +[INFO] Current question consumed time: 1930 ms, average: 2621.96 ms/ques, entity detection: 432.90 ms/ques (linking tool: 355.07 ms/ques), relation detection: 880.58 ms/ques (similarity: 705.45 ms/ques), EDG generation: 46.82 ms/ques [INFO] Entity Linking P: 0.7398543184183143, R: 0.6914672216441207, micro F1: 0.7070759625390217, macro F1: 0.714842885625547 [INFO] Relation Linking P: 0.5977280610475199, R: 0.5437044745057232, micro F1: 0.5593479014915025, macro F1: 0.5694378094306692 [INFO] Type Linking P: 0.5556711758584808, R: 0.5556711758584808, micro F1: 0.5556711758584808, macro F1: 0.5556711758584808 @@ -13486,8 +13486,8 @@ Question 961: For how many cities's teams did David Bowens play? [INFO] Golden Answer: [16] [INFO] Predicted Answer: [1] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 962, P: 0.503, R: 0.560, macro F1: 0.511, macro F1*: 0.530 -[INFO] Current time: 1004 ms, average: 2938.89 ms/ques, entity detection: 495.56 ms/ques (linking tool: 401.62 ms/ques), relation detection: 961.80 ms/ques (similarity: 712.48 ms/ques), EDG generation: 46.27 ms/ques +[INFO] Cumulative metrics, sample: 962, P: 0.503, R: 0.560, macro F1: 0.530 +[INFO] Current question consumed time: 917 ms, average: 2620.18 ms/ques, entity detection: 432.58 ms/ques (linking tool: 354.78 ms/ques), relation detection: 879.94 ms/ques (similarity: 705.01 ms/ques), EDG generation: 46.79 ms/ques [INFO] Entity Linking P: 0.7401247401247402, R: 0.6917879417879418, micro F1: 0.7073804573804573, macro F1: 0.7151404929989322 [INFO] Relation Linking P: 0.5971067221067221, R: 0.5431392931392931, micro F1: 0.5587664587664594, macro F1: 0.568845878235835 [INFO] Type Linking P: 0.5550935550935551, R: 0.5550935550935551, micro F1: 0.5550935550935551, macro F1: 0.5550935550935551 @@ -13500,8 +13500,8 @@ Question 962: Is Anne Hidalgo the mayor of Paris? [INFO] Golden Answer: [true] [INFO] Predicted Answer: [true] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 963, P: 0.504, R: 0.560, macro F1: 0.512, macro F1*: 0.531 -[INFO] Current time: 7616 ms, average: 2943.75 ms/ques, entity detection: 497.09 ms/ques (linking tool: 403.04 ms/ques), relation detection: 965.29 ms/ques (similarity: 713.87 ms/ques), EDG generation: 46.23 ms/ques +[INFO] Cumulative metrics, sample: 963, P: 0.504, R: 0.560, macro F1: 0.531 +[INFO] Current question consumed time: 7027 ms, average: 2624.76 ms/ques, entity detection: 433.97 ms/ques (linking tool: 356.09 ms/ques), relation detection: 883.20 ms/ques (similarity: 706.41 ms/ques), EDG generation: 46.76 ms/ques [INFO] Entity Linking P: 0.7403946002076843, R: 0.6921079958463136, micro F1: 0.7076843198338524, macro F1: 0.7154374788523692 [INFO] Relation Linking P: 0.5975250951886466, R: 0.543613707165109, micro F1: 0.5592246452059542, macro F1: 0.5692959199173546 [INFO] Type Linking P: 0.5555555555555556, R: 0.5555555555555556, micro F1: 0.5555555555555556, macro F1: 0.5555555555555556 @@ -13514,8 +13514,8 @@ Question 963: In which countries have models hanged themselves? [INFO] Golden Answer: [http://dbpedia.org/resource/India, http://dbpedia.org/resource/Mauritius] [INFO] Predicted Answer: [] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 964, P: 0.503, R: 0.560, macro F1: 0.511, macro F1*: 0.530 -[INFO] Current time: 324 ms, average: 2941.03 ms/ques, entity detection: 496.57 ms/ques (linking tool: 402.63 ms/ques), relation detection: 964.56 ms/ques (similarity: 713.30 ms/ques), EDG generation: 46.19 ms/ques +[INFO] Cumulative metrics, sample: 964, P: 0.503, R: 0.560, macro F1: 0.530 +[INFO] Current question consumed time: 297 ms, average: 2622.35 ms/ques, entity detection: 433.52 ms/ques (linking tool: 355.72 ms/ques), relation detection: 882.53 ms/ques (similarity: 705.84 ms/ques), EDG generation: 46.72 ms/ques [INFO] Entity Linking P: 0.7396265560165975, R: 0.6913900414937759, micro F1: 0.7069502074688796, macro F1: 0.7146953237913191 [INFO] Relation Linking P: 0.5969052558782849, R: 0.5430497925311203, micro F1: 0.558644536652836, macro F1: 0.5687053639838304 [INFO] Type Linking P: 0.5549792531120332, R: 0.5549792531120332, micro F1: 0.5549792531120332, macro F1: 0.5549792531120332 @@ -13528,8 +13528,8 @@ Question 964: Is the Vadodara railway junction onwed by Indian Railway Ministry? [INFO] Golden Answer: [true] [INFO] Predicted Answer: [false] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 965, P: 0.503, R: 0.559, macro F1: 0.511, macro F1*: 0.529 -[INFO] Current time: 3828 ms, average: 2941.95 ms/ques, entity detection: 496.48 ms/ques (linking tool: 402.54 ms/ques), relation detection: 966.06 ms/ques (similarity: 714.01 ms/ques), EDG generation: 46.16 ms/ques +[INFO] Cumulative metrics, sample: 965, P: 0.503, R: 0.559, macro F1: 0.529 +[INFO] Current question consumed time: 3259 ms, average: 2623.01 ms/ques, entity detection: 433.42 ms/ques (linking tool: 355.62 ms/ques), relation detection: 883.81 ms/ques (similarity: 706.56 ms/ques), EDG generation: 46.68 ms/ques [INFO] Entity Linking P: 0.738860103626943, R: 0.6906735751295336, micro F1: 0.7062176165803108, macro F1: 0.7139547068754731 [INFO] Relation Linking P: 0.596286701208981, R: 0.5424870466321243, micro F1: 0.5580656303972372, macro F1: 0.5681160320004275 [INFO] Type Linking P: 0.555440414507772, R: 0.555440414507772, micro F1: 0.555440414507772, macro F1: 0.555440414507772 @@ -13542,8 +13542,8 @@ Question 965: What city gave birth to Mishal Raheja and also houses New India As [INFO] Golden Answer: [http://dbpedia.org/resource/India, http://dbpedia.org/resource/Mumbai] [INFO] Predicted Answer: [http://dbpedia.org/resource/Mumbai] [INFO] P: 1.0, R: 0.5, F: 0.6666666666666666 -[INFO] Cumulative metrics, sample: 966, P: 0.503, R: 0.559, macro F1: 0.511, macro F1*: 0.530 -[INFO] Current time: 1496 ms, average: 2940.46 ms/ques, entity detection: 496.34 ms/ques (linking tool: 402.34 ms/ques), relation detection: 965.28 ms/ques (similarity: 713.53 ms/ques), EDG generation: 46.16 ms/ques +[INFO] Cumulative metrics, sample: 966, P: 0.503, R: 0.559, macro F1: 0.530 +[INFO] Current question consumed time: 1219 ms, average: 2621.55 ms/ques, entity detection: 433.27 ms/ques (linking tool: 355.43 ms/ques), relation detection: 883.09 ms/ques (similarity: 706.08 ms/ques), EDG generation: 46.69 ms/ques [INFO] Entity Linking P: 0.7391304347826086, R: 0.6904761904761905, micro F1: 0.7061766735679778, macro F1: 0.7139753801593048 [INFO] Relation Linking P: 0.5967046238785368, R: 0.5424430641821946, micro F1: 0.5581780538302282, macro F1: 0.5682815107835275 [INFO] Type Linking P: 0.5548654244306418, R: 0.5548654244306418, micro F1: 0.5548654244306418, macro F1: 0.5548654244306418 @@ -13556,8 +13556,8 @@ Question 966: Which TV show whose theme is composed by someone signed up with AT [INFO] Golden Answer: [http://dbpedia.org/resource/South_Park, http://dbpedia.org/resource/State_of_Georgia_(TV_series), http://dbpedia.org/resource/The_100_(TV_series), http://dbpedia.org/resource/90210_(TV_series)] [INFO] Predicted Answer: [] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 967, P: 0.503, R: 0.558, macro F1: 0.510, macro F1*: 0.529 -[INFO] Current time: 1214 ms, average: 2938.67 ms/ques, entity detection: 495.83 ms/ques (linking tool: 401.93 ms/ques), relation detection: 964.63 ms/ques (similarity: 713.01 ms/ques), EDG generation: 46.14 ms/ques +[INFO] Cumulative metrics, sample: 967, P: 0.503, R: 0.558, macro F1: 0.529 +[INFO] Current question consumed time: 1064 ms, average: 2619.94 ms/ques, entity detection: 432.82 ms/ques (linking tool: 355.06 ms/ques), relation detection: 882.47 ms/ques (similarity: 705.55 ms/ques), EDG generation: 46.68 ms/ques [INFO] Entity Linking P: 0.7383660806618407, R: 0.6897621509824199, micro F1: 0.705446397793864, macro F1: 0.7132370395386644 [INFO] Relation Linking P: 0.5960875560151672, R: 0.5418821096173733, micro F1: 0.5576008273009312, macro F1: 0.5676938360050543 [INFO] Type Linking P: 0.5542916235780765, R: 0.5542916235780765, micro F1: 0.5542916235780765, macro F1: 0.5542916235780765 @@ -13570,8 +13570,8 @@ Question 967: Which models were featured in Playboy Playmates 1954? [INFO] Golden Answer: [http://dbpedia.org/resource/Joanne_Arnold, http://dbpedia.org/resource/Jayne_Mansfield, http://dbpedia.org/resource/Arline_Hunter] [INFO] Predicted Answer: [http://dbpedia.org/resource/Barbi_Benton] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 968, P: 0.502, R: 0.558, macro F1: 0.510, macro F1*: 0.529 -[INFO] Current time: 1453 ms, average: 2937.14 ms/ques, entity detection: 495.53 ms/ques (linking tool: 401.67 ms/ques), relation detection: 964.12 ms/ques (similarity: 712.73 ms/ques), EDG generation: 46.11 ms/ques +[INFO] Cumulative metrics, sample: 968, P: 0.502, R: 0.558, macro F1: 0.529 +[INFO] Current question consumed time: 1290 ms, average: 2618.57 ms/ques, entity detection: 432.56 ms/ques (linking tool: 354.83 ms/ques), relation detection: 882.00 ms/ques (similarity: 705.38 ms/ques), EDG generation: 46.64 ms/ques [INFO] Entity Linking P: 0.737603305785124, R: 0.6890495867768595, micro F1: 0.7047176308539943, macro F1: 0.712500224415174 [INFO] Relation Linking P: 0.5954717630853994, R: 0.5413223140495868, micro F1: 0.5570247933884303, macro F1: 0.5671073754306689 [INFO] Type Linking P: 0.5537190082644629, R: 0.5537190082644629, micro F1: 0.5537190082644629, macro F1: 0.5537190082644629 @@ -13584,8 +13584,8 @@ Question 968: What currencies have been prevalent in the country whose national [INFO] Golden Answer: [http://dbpedia.org/resource/French_West_African_franc, http://dbpedia.org/resource/CFA_franc, http://dbpedia.org/resource/Euro, http://dbpedia.org/resource/Euro_sign, http://dbpedia.org/resource/French_franc, http://dbpedia.org/resource/Malagasy_franc, http://dbpedia.org/resource/ISO_4217, http://dbpedia.org/resource/CFP_franc, http://dbpedia.org/resource/Assignat, http://dbpedia.org/resource/Central_African_CFA_franc, http://dbpedia.org/resource/French_Equatorial_African_franc, http://dbpedia.org/resource/Tunisian_franc, http://dbpedia.org/resource/Tunisian_rial, http://dbpedia.org/resource/French_Indochinese_piastre, http://dbpedia.org/resource/Moroccan_franc, http://dbpedia.org/resource/Moroccan_rial, http://dbpedia.org/resource/French_Somaliland_franc, http://dbpedia.org/resource/French_Indian_rupee, http://dbpedia.org/resource/West_African_CFA_franc] [INFO] Predicted Answer: [http://dbpedia.org/resource/French_Indian_rupee, Euro@en] [INFO] P: 1.0, R: 0.10526315789473684, F: 0.1904761904761905 -[INFO] Cumulative metrics, sample: 969, P: 0.503, R: 0.557, macro F1: 0.510, macro F1*: 0.529 -[INFO] Current time: 2322 ms, average: 2936.50 ms/ques, entity detection: 495.19 ms/ques (linking tool: 401.35 ms/ques), relation detection: 964.31 ms/ques (similarity: 712.88 ms/ques), EDG generation: 46.09 ms/ques +[INFO] Cumulative metrics, sample: 969, P: 0.503, R: 0.557, macro F1: 0.529 +[INFO] Current question consumed time: 2004 ms, average: 2617.93 ms/ques, entity detection: 432.24 ms/ques (linking tool: 354.54 ms/ques), relation detection: 882.14 ms/ques (similarity: 705.53 ms/ques), EDG generation: 46.63 ms/ques [INFO] Entity Linking P: 0.737874097007224, R: 0.6893704850361198, micro F1: 0.7050223598211213, macro F1: 0.7127981154024955 [INFO] Relation Linking P: 0.5953732370141038, R: 0.5412796697626419, micro F1: 0.5569659442724464, macro F1: 0.5670392908779254 [INFO] Type Linking P: 0.5531475748194015, R: 0.5531475748194015, micro F1: 0.5531475748194015, macro F1: 0.5531475748194015 @@ -13598,8 +13598,8 @@ Question 969: Who is the relative of Jim Farley (businessman) ? [INFO] Golden Answer: [http://dbpedia.org/resource/Chris_Farley] [INFO] Predicted Answer: [http://dbpedia.org/resource/Chris_Farley] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 970, P: 0.503, R: 0.558, macro F1: 0.510, macro F1*: 0.529 -[INFO] Current time: 641 ms, average: 2934.13 ms/ques, entity detection: 494.80 ms/ques (linking tool: 401.02 ms/ques), relation detection: 963.47 ms/ques (similarity: 712.24 ms/ques), EDG generation: 46.06 ms/ques +[INFO] Cumulative metrics, sample: 970, P: 0.503, R: 0.558, macro F1: 0.529 +[INFO] Current question consumed time: 531 ms, average: 2615.78 ms/ques, entity detection: 431.90 ms/ques (linking tool: 354.24 ms/ques), relation detection: 881.38 ms/ques (similarity: 704.90 ms/ques), EDG generation: 46.60 ms/ques [INFO] Entity Linking P: 0.7381443298969073, R: 0.6896907216494845, micro F1: 0.7053264604810995, macro F1: 0.7130953887379509 [INFO] Relation Linking P: 0.5947594501718213, R: 0.540721649484536, micro F1: 0.5563917525773201, macro F1: 0.5664547142893914 [INFO] Type Linking P: 0.5536082474226804, R: 0.5536082474226804, micro F1: 0.5536082474226804, macro F1: 0.5536082474226804 @@ -13612,8 +13612,8 @@ Question 970: Is Latin the official language of Sovereign Military Order of Malt [INFO] Golden Answer: [true] [INFO] Predicted Answer: [true] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 971, P: 0.504, R: 0.558, macro F1: 0.511, macro F1*: 0.530 -[INFO] Current time: 2628 ms, average: 2933.82 ms/ques, entity detection: 495.19 ms/ques (linking tool: 401.33 ms/ques), relation detection: 962.97 ms/ques (similarity: 711.79 ms/ques), EDG generation: 46.02 ms/ques +[INFO] Cumulative metrics, sample: 971, P: 0.504, R: 0.558, macro F1: 0.530 +[INFO] Current question consumed time: 2231 ms, average: 2615.39 ms/ques, entity detection: 432.23 ms/ques (linking tool: 354.51 ms/ques), relation detection: 880.92 ms/ques (similarity: 704.45 ms/ques), EDG generation: 46.57 ms/ques [INFO] Entity Linking P: 0.7384140061791967, R: 0.690010298661174, micro F1: 0.7056299347751458, macro F1: 0.7133920463447176 [INFO] Relation Linking P: 0.5951767936834877, R: 0.5411946446961895, micro F1: 0.556848609680742, macro F1: 0.5669035361329317 [INFO] Type Linking P: 0.5540679711637487, R: 0.5540679711637487, micro F1: 0.5540679711637487, macro F1: 0.5540679711637487 @@ -13626,8 +13626,8 @@ Question 971: Where do the politicians, Blanche Bruce and John Franklin Miller w [INFO] Golden Answer: [http://dbpedia.org/resource/United_States_Senate] [INFO] Predicted Answer: [http://dbpedia.org/resource/Teacher] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 972, P: 0.503, R: 0.558, macro F1: 0.510, macro F1*: 0.529 -[INFO] Current time: 3628 ms, average: 2934.53 ms/ques, entity detection: 496.31 ms/ques (linking tool: 402.43 ms/ques), relation detection: 962.82 ms/ques (similarity: 711.53 ms/ques), EDG generation: 46.01 ms/ques +[INFO] Cumulative metrics, sample: 972, P: 0.503, R: 0.558, macro F1: 0.529 +[INFO] Current question consumed time: 3277 ms, average: 2616.07 ms/ques, entity detection: 433.28 ms/ques (linking tool: 355.54 ms/ques), relation detection: 880.83 ms/ques (similarity: 704.25 ms/ques), EDG generation: 46.55 ms/ques [INFO] Entity Linking P: 0.7381687242798354, R: 0.6898148148148148, micro F1: 0.7054183813443071, macro F1: 0.713173097449034 [INFO] Relation Linking P: 0.5945644718792866, R: 0.5406378600823045, micro F1: 0.5562757201646096, macro F1: 0.5663203020422599 [INFO] Type Linking P: 0.5545267489711934, R: 0.5545267489711934, micro F1: 0.5545267489711934, macro F1: 0.5545267489711934 @@ -13640,8 +13640,8 @@ Question 972: Name an American football player whose debuted in Chicago Bears an [INFO] Golden Answer: [http://dbpedia.org/resource/Jim_Harbaugh] [INFO] Predicted Answer: [http://dbpedia.org/resource/Jim_Harbaugh] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 973, P: 0.504, R: 0.558, macro F1: 0.511, macro F1*: 0.530 -[INFO] Current time: 4561 ms, average: 2936.20 ms/ques, entity detection: 497.01 ms/ques (linking tool: 403.02 ms/ques), relation detection: 962.96 ms/ques (similarity: 711.16 ms/ques), EDG generation: 46.04 ms/ques +[INFO] Cumulative metrics, sample: 973, P: 0.504, R: 0.558, macro F1: 0.530 +[INFO] Current question consumed time: 4149 ms, average: 2617.64 ms/ques, entity detection: 433.96 ms/ques (linking tool: 356.13 ms/ques), relation detection: 881.02 ms/ques (similarity: 703.92 ms/ques), EDG generation: 46.53 ms/ques [INFO] Entity Linking P: 0.7384378211716341, R: 0.6896197327852004, micro F1: 0.7053785542994174, macro F1: 0.7131943548127622 [INFO] Relation Linking P: 0.5949811579307982, R: 0.5405960945529291, micro F1: 0.5563891743747864, macro F1: 0.5664863215716417 [INFO] Type Linking P: 0.5539568345323741, R: 0.5539568345323741, micro F1: 0.5539568345323741, macro F1: 0.5539568345323741 @@ -13654,8 +13654,8 @@ Question 973: Name a river with source country as Zambia and source location as [INFO] Golden Answer: [http://dbpedia.org/resource/Zambezi] [INFO] Predicted Answer: [http://dbpedia.org/resource/Luvua_River, http://dbpedia.org/resource/Zambezi, http://dbpedia.org/resource/Kafue_River, http://dbpedia.org/resource/Sakeji_River] [INFO] P: 0.25, R: 1.0, F: 0.4 -[INFO] Cumulative metrics, sample: 974, P: 0.503, R: 0.559, macro F1: 0.510, macro F1*: 0.530 -[INFO] Current time: 9890 ms, average: 2943.34 ms/ques, entity detection: 497.56 ms/ques (linking tool: 403.53 ms/ques), relation detection: 966.19 ms/ques (similarity: 714.70 ms/ques), EDG generation: 46.04 ms/ques +[INFO] Cumulative metrics, sample: 974, P: 0.503, R: 0.559, macro F1: 0.530 +[INFO] Current question consumed time: 9060 ms, average: 2624.26 ms/ques, entity detection: 434.48 ms/ques (linking tool: 356.61 ms/ques), relation detection: 884.00 ms/ques (similarity: 707.46 ms/ques), EDG generation: 46.54 ms/ques [INFO] Entity Linking P: 0.7387063655030801, R: 0.6894250513347022, micro F1: 0.7053388090349073, macro F1: 0.7132154197488402 [INFO] Relation Linking P: 0.5953969883641341, R: 0.5405544147843943, micro F1: 0.5565023956194391, macro F1: 0.5666518298538225 [INFO] Type Linking P: 0.553388090349076, R: 0.553388090349076, micro F1: 0.553388090349076, macro F1: 0.553388090349076 @@ -13668,8 +13668,8 @@ Question 974: Does Ford own the Ford Kansas Assembly plant? [INFO] Golden Answer: [true] [INFO] Predicted Answer: [false] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 975, P: 0.503, R: 0.558, macro F1: 0.510, macro F1*: 0.529 -[INFO] Current time: 2921 ms, average: 2943.32 ms/ques, entity detection: 497.38 ms/ques (linking tool: 403.25 ms/ques), relation detection: 966.15 ms/ques (similarity: 714.51 ms/ques), EDG generation: 46.00 ms/ques +[INFO] Cumulative metrics, sample: 975, P: 0.503, R: 0.558, macro F1: 0.529 +[INFO] Current question consumed time: 2499 ms, average: 2624.13 ms/ques, entity detection: 434.30 ms/ques (linking tool: 356.35 ms/ques), relation detection: 883.97 ms/ques (similarity: 707.27 ms/ques), EDG generation: 46.50 ms/ques [INFO] Entity Linking P: 0.737948717948718, R: 0.6887179487179487, micro F1: 0.7046153846153844, macro F1: 0.7124839167542258 [INFO] Relation Linking P: 0.5947863247863248, R: 0.54, micro F1: 0.5559316239316243, macro F1: 0.5660706484898698 [INFO] Type Linking P: 0.5538461538461539, R: 0.5538461538461539, micro F1: 0.5538461538461539, macro F1: 0.5538461538461539 @@ -13682,8 +13682,8 @@ Question 975: Which fields of the Paul Demiville is also the faith of Koay Teng [INFO] Golden Answer: [http://dbpedia.org/resource/Buddhism] [INFO] Predicted Answer: [http://dbpedia.org/resource/Buddhism] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 976, P: 0.503, R: 0.559, macro F1: 0.510, macro F1*: 0.530 -[INFO] Current time: 3115 ms, average: 2943.50 ms/ques, entity detection: 497.33 ms/ques (linking tool: 403.02 ms/ques), relation detection: 966.32 ms/ques (similarity: 714.43 ms/ques), EDG generation: 46.05 ms/ques +[INFO] Cumulative metrics, sample: 976, P: 0.503, R: 0.559, macro F1: 0.530 +[INFO] Current question consumed time: 2710 ms, average: 2624.22 ms/ques, entity detection: 434.21 ms/ques (linking tool: 356.13 ms/ques), relation detection: 884.15 ms/ques (similarity: 707.22 ms/ques), EDG generation: 46.55 ms/ques [INFO] Entity Linking P: 0.7382172131147541, R: 0.6885245901639344, micro F1: 0.7045765027322402, macro F1: 0.712505518438944 [INFO] Relation Linking P: 0.5952015027322404, R: 0.5399590163934426, micro F1: 0.5560450819672135, macro F1: 0.5662360742051431 [INFO] Type Linking P: 0.5543032786885246, R: 0.5543032786885246, micro F1: 0.5543032786885246, macro F1: 0.5543032786885246 @@ -13696,8 +13696,8 @@ Question 976: Name the institute of Alton Ochsner ? [INFO] Golden Answer: [http://dbpedia.org/resource/Tulane_University] [INFO] Predicted Answer: [http://dbpedia.org/resource/Tulane_University] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 977, P: 0.504, R: 0.559, macro F1: 0.511, macro F1*: 0.530 -[INFO] Current time: 642 ms, average: 2941.14 ms/ques, entity detection: 496.98 ms/ques (linking tool: 402.69 ms/ques), relation detection: 965.48 ms/ques (similarity: 713.78 ms/ques), EDG generation: 46.01 ms/ques +[INFO] Cumulative metrics, sample: 977, P: 0.504, R: 0.559, macro F1: 0.530 +[INFO] Current question consumed time: 511 ms, average: 2622.05 ms/ques, entity detection: 433.89 ms/ques (linking tool: 355.83 ms/ques), relation detection: 883.36 ms/ques (similarity: 706.56 ms/ques), EDG generation: 46.51 ms/ques [INFO] Entity Linking P: 0.7384851586489253, R: 0.6888433981576254, micro F1: 0.7048788809280107, macro F1: 0.7128010208257106 [INFO] Relation Linking P: 0.5956158307744797, R: 0.5404298874104401, micro F1: 0.5564994882292736, macro F1: 0.5666824692224797 [INFO] Type Linking P: 0.5547594677584442, R: 0.5547594677584442, micro F1: 0.5547594677584442, macro F1: 0.5547594677584442 @@ -13710,8 +13710,8 @@ Question 977: What movies have been written by authors of Monty Python ? [INFO] Golden Answer: [http://dbpedia.org/resource/Monty_Python_and_the_Holy_Grail, http://dbpedia.org/resource/Monty_Python's_Life_of_Brian, http://dbpedia.org/resource/And_Now_for_Something_Completely_Different, http://dbpedia.org/resource/Monty_Python_Live_at_the_Hollywood_Bowl, http://dbpedia.org/resource/Monty_Python's_The_Meaning_of_Life] [INFO] Predicted Answer: [http://dbpedia.org/resource/Monty_Python:_Almost_the_Truth_(Lawyers_Cut)] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 978, P: 0.503, R: 0.558, macro F1: 0.510, macro F1*: 0.530 -[INFO] Current time: 1831 ms, average: 2940.01 ms/ques, entity detection: 496.72 ms/ques (linking tool: 402.44 ms/ques), relation detection: 965.17 ms/ques (similarity: 713.81 ms/ques), EDG generation: 45.98 ms/ques +[INFO] Cumulative metrics, sample: 978, P: 0.503, R: 0.558, macro F1: 0.530 +[INFO] Current question consumed time: 1563 ms, average: 2620.97 ms/ques, entity detection: 433.64 ms/ques (linking tool: 355.60 ms/ques), relation detection: 883.02 ms/ques (similarity: 706.50 ms/ques), EDG generation: 46.49 ms/ques [INFO] Entity Linking P: 0.7387525562372188, R: 0.689161554192229, micro F1: 0.7051806407634625, macro F1: 0.7130959153653929 [INFO] Relation Linking P: 0.5950068166325835, R: 0.5398773006134969, micro F1: 0.5559304703476486, macro F1: 0.5661030392948493 [INFO] Type Linking P: 0.5541922290388548, R: 0.5541922290388548, micro F1: 0.5541922290388548, macro F1: 0.5541922290388548 @@ -13724,8 +13724,8 @@ Question 978: Who was the cover artist of The Adventure of Tom Sawyer also wrote [INFO] Golden Answer: [http://dbpedia.org/resource/Mark_Twain] [INFO] Predicted Answer: [http://dbpedia.org/resource/Mark_Twain] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 979, P: 0.504, R: 0.559, macro F1: 0.511, macro F1*: 0.530 -[INFO] Current time: 2665 ms, average: 2939.73 ms/ques, entity detection: 496.83 ms/ques (linking tool: 402.43 ms/ques), relation detection: 964.83 ms/ques (similarity: 713.46 ms/ques), EDG generation: 45.99 ms/ques +[INFO] Cumulative metrics, sample: 979, P: 0.504, R: 0.559, macro F1: 0.530 +[INFO] Current question consumed time: 2279 ms, average: 2620.62 ms/ques, entity detection: 433.70 ms/ques (linking tool: 355.58 ms/ques), relation detection: 882.73 ms/ques (similarity: 706.17 ms/ques), EDG generation: 46.49 ms/ques [INFO] Entity Linking P: 0.7385086823289071, R: 0.6889683350357507, micro F1: 0.7049710589036429, macro F1: 0.7128788640155246 [INFO] Relation Linking P: 0.5943990466462377, R: 0.5393258426966292, micro F1: 0.5553626149131771, macro F1: 0.5655247930851508 [INFO] Type Linking P: 0.5546475995914198, R: 0.5546475995914198, micro F1: 0.5546475995914198, macro F1: 0.5546475995914198 @@ -13738,8 +13738,8 @@ Question 979: Name the scientist whose academic advisor is Karl Ewald Hasse and [INFO] Golden Answer: [http://dbpedia.org/resource/Robert_Koch] [INFO] Predicted Answer: [http://dbpedia.org/resource/List_of_German_football_transfers_summer_2014, Robert Koch@en] [INFO] P: 0.5, R: 1.0, F: 0.6666666666666666 -[INFO] Cumulative metrics, sample: 980, P: 0.504, R: 0.559, macro F1: 0.511, macro F1*: 0.530 -[INFO] Current time: 2239 ms, average: 2939.01 ms/ques, entity detection: 496.83 ms/ques (linking tool: 402.22 ms/ques), relation detection: 964.45 ms/ques (similarity: 713.25 ms/ques), EDG generation: 45.99 ms/ques +[INFO] Cumulative metrics, sample: 980, P: 0.504, R: 0.559, macro F1: 0.530 +[INFO] Current question consumed time: 1998 ms, average: 2619.98 ms/ques, entity detection: 433.67 ms/ques (linking tool: 355.37 ms/ques), relation detection: 882.38 ms/ques (similarity: 705.95 ms/ques), EDG generation: 46.54 ms/ques [INFO] Entity Linking P: 0.7387755102040816, R: 0.6887755102040817, micro F1: 0.7049319727891153, macro F1: 0.7128998847573338 [INFO] Relation Linking P: 0.5943027210884353, R: 0.5392857142857143, micro F1: 0.55530612244898, macro F1: 0.5654591339198645 [INFO] Type Linking P: 0.5540816326530612, R: 0.5540816326530612, micro F1: 0.5540816326530612, macro F1: 0.5540816326530612 @@ -13752,8 +13752,8 @@ Question 980: Where do current famous NatGeo photographers live? [INFO] Golden Answer: [http://dbpedia.org/resource/Virginia, http://dbpedia.org/resource/Sugar_Hollow] [INFO] Predicted Answer: [http://dbpedia.org/resource/Tim_Rawle, http://dbpedia.org/resource/Patrick_Bingham-Hall, http://dbpedia.org/resource/Hélène_Binet, http://dbpedia.org/resource/James_Austin_(photographer), http://dbpedia.org/resource/Eric_de_Maré] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 981, P: 0.503, R: 0.559, macro F1: 0.511, macro F1*: 0.530 -[INFO] Current time: 801 ms, average: 2936.83 ms/ques, entity detection: 496.41 ms/ques (linking tool: 401.87 ms/ques), relation detection: 963.65 ms/ques (similarity: 712.63 ms/ques), EDG generation: 45.94 ms/ques +[INFO] Cumulative metrics, sample: 981, P: 0.503, R: 0.559, macro F1: 0.530 +[INFO] Current question consumed time: 714 ms, average: 2618.04 ms/ques, entity detection: 433.30 ms/ques (linking tool: 355.06 ms/ques), relation detection: 881.65 ms/ques (similarity: 705.34 ms/ques), EDG generation: 46.50 ms/ques [INFO] Entity Linking P: 0.7380224260958206, R: 0.6880733944954128, micro F1: 0.7042133876996259, macro F1: 0.7121731774334223 [INFO] Relation Linking P: 0.5936969079170914, R: 0.5387359836901121, micro F1: 0.5547400611620799, macro F1: 0.5648827229780502 [INFO] Type Linking P: 0.5535168195718655, R: 0.5535168195718655, micro F1: 0.5535168195718655, macro F1: 0.5535168195718655 @@ -13766,8 +13766,8 @@ Question 981: Georg Meissner was doctoral supervisor of which scientist ? [INFO] Golden Answer: [http://dbpedia.org/resource/Robert_Koch] [INFO] Predicted Answer: [http://dbpedia.org/resource/Robert_Koch] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 982, P: 0.504, R: 0.559, macro F1: 0.511, macro F1*: 0.530 -[INFO] Current time: 731 ms, average: 2934.59 ms/ques, entity detection: 496.11 ms/ques (linking tool: 401.56 ms/ques), relation detection: 962.79 ms/ques (similarity: 712.00 ms/ques), EDG generation: 45.91 ms/ques +[INFO] Cumulative metrics, sample: 982, P: 0.504, R: 0.559, macro F1: 0.530 +[INFO] Current question consumed time: 622 ms, average: 2616.01 ms/ques, entity detection: 433.03 ms/ques (linking tool: 354.78 ms/ques), relation detection: 880.86 ms/ques (similarity: 704.72 ms/ques), EDG generation: 46.47 ms/ques [INFO] Entity Linking P: 0.7382892057026477, R: 0.6883910386965377, micro F1: 0.7045145960624573, macro F1: 0.7124675275588719 [INFO] Relation Linking P: 0.5941106585200271, R: 0.5392057026476579, micro F1: 0.5551934826883914, macro F1: 0.5653282102937106 [INFO] Type Linking P: 0.5529531568228105, R: 0.5529531568228105, micro F1: 0.5529531568228105, macro F1: 0.5529531568228105 @@ -13780,8 +13780,8 @@ Question 982: Did Stanlee write the Tales of Suspense? [INFO] Golden Answer: [true] [INFO] Predicted Answer: [false] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 983, P: 0.503, R: 0.559, macro F1: 0.511, macro F1*: 0.530 -[INFO] Current time: 885 ms, average: 2932.50 ms/ques, entity detection: 495.90 ms/ques (linking tool: 401.34 ms/ques), relation detection: 961.93 ms/ques (similarity: 711.35 ms/ques), EDG generation: 45.87 ms/ques +[INFO] Cumulative metrics, sample: 983, P: 0.503, R: 0.559, macro F1: 0.530 +[INFO] Current question consumed time: 756 ms, average: 2614.12 ms/ques, entity detection: 432.83 ms/ques (linking tool: 354.58 ms/ques), relation detection: 880.07 ms/ques (similarity: 704.07 ms/ques), EDG generation: 46.43 ms/ques [INFO] Entity Linking P: 0.7375381485249237, R: 0.6876907426246185, micro F1: 0.7037978975924039, macro F1: 0.7117427386193411 [INFO] Relation Linking P: 0.5935062733129874, R: 0.5386571719226857, micro F1: 0.554628687690743, macro F1: 0.5647531052984983 [INFO] Type Linking P: 0.5534079348931842, R: 0.5534079348931842, micro F1: 0.5534079348931842, macro F1: 0.5534079348931842 @@ -13794,8 +13794,8 @@ Question 983: Name the common home port of the Trishna (yacht) and INS Viraat (R [INFO] Golden Answer: [http://dbpedia.org/resource/Mumbai] [INFO] Predicted Answer: [http://dbpedia.org/resource/Mumbai] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 984, P: 0.504, R: 0.559, macro F1: 0.511, macro F1*: 0.530 -[INFO] Current time: 5201 ms, average: 2934.81 ms/ques, entity detection: 496.51 ms/ques (linking tool: 401.81 ms/ques), relation detection: 962.23 ms/ques (similarity: 711.27 ms/ques), EDG generation: 45.91 ms/ques +[INFO] Cumulative metrics, sample: 984, P: 0.504, R: 0.559, macro F1: 0.530 +[INFO] Current question consumed time: 4538 ms, average: 2616.07 ms/ques, entity detection: 433.31 ms/ques (linking tool: 354.96 ms/ques), relation detection: 880.33 ms/ques (similarity: 703.98 ms/ques), EDG generation: 46.47 ms/ques [INFO] Entity Linking P: 0.7378048780487805, R: 0.6875, micro F1: 0.7037601626016257, macro F1: 0.7117647058823529 [INFO] Relation Linking P: 0.5939193766937669, R: 0.5391260162601627, micro F1: 0.5550813008130085, macro F1: 0.5651978103045849 [INFO] Type Linking P: 0.5538617886178862, R: 0.5538617886178862, micro F1: 0.5538617886178862, macro F1: 0.5538617886178862 @@ -13808,8 +13808,8 @@ Question 984: does the US navy have the structure of command of us department of [INFO] Golden Answer: [true] [INFO] Predicted Answer: [true] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 985, P: 0.504, R: 0.560, macro F1: 0.512, macro F1*: 0.531 -[INFO] Current time: 829 ms, average: 2932.67 ms/ques, entity detection: 496.00 ms/ques (linking tool: 401.40 ms/ques), relation detection: 961.25 ms/ques (similarity: 710.55 ms/ques), EDG generation: 45.97 ms/ques +[INFO] Cumulative metrics, sample: 985, P: 0.504, R: 0.560, macro F1: 0.531 +[INFO] Current question consumed time: 762 ms, average: 2614.19 ms/ques, entity detection: 432.87 ms/ques (linking tool: 354.59 ms/ques), relation detection: 879.44 ms/ques (similarity: 703.27 ms/ques), EDG generation: 46.58 ms/ques [INFO] Entity Linking P: 0.7370558375634517, R: 0.6868020304568528, micro F1: 0.7030456852791874, macro F1: 0.7110421021200359 [INFO] Relation Linking P: 0.59331641285956, R: 0.5385786802030457, micro F1: 0.5545177664974623, macro F1: 0.5646240054210269 [INFO] Type Linking P: 0.5543147208121827, R: 0.5543147208121827, micro F1: 0.5543147208121827, macro F1: 0.5543147208121827 @@ -13822,8 +13822,8 @@ Question 985: Name books with publisher as Francisco de Robles and writer as Mig [INFO] Golden Answer: [http://dbpedia.org/resource/Don_Quixote] [INFO] Predicted Answer: [http://dbpedia.org/resource/Don_Quixote_(2000_film), http://dbpedia.org/resource/Don_Quixote_(1957_film), http://dbpedia.org/resource/Don_Quixote_(unfinished_film)] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 986, P: 0.504, R: 0.559, macro F1: 0.511, macro F1*: 0.530 -[INFO] Current time: 2019 ms, average: 2931.74 ms/ques, entity detection: 495.91 ms/ques (linking tool: 401.25 ms/ques), relation detection: 960.82 ms/ques (similarity: 710.35 ms/ques), EDG generation: 45.95 ms/ques +[INFO] Cumulative metrics, sample: 986, P: 0.504, R: 0.559, macro F1: 0.530 +[INFO] Current question consumed time: 1763 ms, average: 2613.33 ms/ques, entity detection: 432.77 ms/ques (linking tool: 354.46 ms/ques), relation detection: 879.03 ms/ques (similarity: 703.08 ms/ques), EDG generation: 46.57 ms/ques [INFO] Entity Linking P: 0.7373225152129818, R: 0.6866125760649088, micro F1: 0.7030087897227852, macro F1: 0.7110645908820351 [INFO] Relation Linking P: 0.5927146720757268, R: 0.5380324543610547, micro F1: 0.5539553752535501, macro F1: 0.5640513644418981 [INFO] Type Linking P: 0.5537525354969574, R: 0.5537525354969574, micro F1: 0.5537525354969574, macro F1: 0.5537525354969574 @@ -13836,8 +13836,8 @@ Question 986: Who owns the schools which have a penguin as its mascot? [INFO] Golden Answer: [http://dbpedia.org/resource/Maine] [INFO] Predicted Answer: [http://dbpedia.org/resource/Maine] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 987, P: 0.504, R: 0.559, macro F1: 0.512, macro F1*: 0.530 -[INFO] Current time: 2072 ms, average: 2930.87 ms/ques, entity detection: 495.53 ms/ques (linking tool: 400.90 ms/ques), relation detection: 960.76 ms/ques (similarity: 710.29 ms/ques), EDG generation: 45.93 ms/ques +[INFO] Cumulative metrics, sample: 987, P: 0.504, R: 0.559, macro F1: 0.530 +[INFO] Current question consumed time: 1812 ms, average: 2612.52 ms/ques, entity detection: 432.44 ms/ques (linking tool: 354.15 ms/ques), relation detection: 878.99 ms/ques (similarity: 703.02 ms/ques), EDG generation: 46.54 ms/ques [INFO] Entity Linking P: 0.7375886524822695, R: 0.6869300911854104, micro F1: 0.7033096926713944, macro F1: 0.7113586150540238 [INFO] Relation Linking P: 0.593127321850726, R: 0.538500506585613, micro F1: 0.5544072948328271, macro F1: 0.5644954202438157 [INFO] Type Linking P: 0.5531914893617021, R: 0.5531914893617021, micro F1: 0.5531914893617021, macro F1: 0.5531914893617021 @@ -13850,8 +13850,8 @@ Question 987: River which flows into the eugene mahoney state park pass through [INFO] Golden Answer: [http://dbpedia.org/resource/Nebraska] [INFO] Predicted Answer: [http://dbpedia.org/resource/Nebraska] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 988, P: 0.505, R: 0.560, macro F1: 0.512, macro F1*: 0.531 -[INFO] Current time: 1239 ms, average: 2929.16 ms/ques, entity detection: 495.13 ms/ques (linking tool: 400.55 ms/ques), relation detection: 960.18 ms/ques (similarity: 710.01 ms/ques), EDG generation: 45.90 ms/ques +[INFO] Cumulative metrics, sample: 988, P: 0.505, R: 0.560, macro F1: 0.531 +[INFO] Current question consumed time: 1106 ms, average: 2610.99 ms/ques, entity detection: 432.08 ms/ques (linking tool: 353.84 ms/ques), relation detection: 878.46 ms/ques (similarity: 702.75 ms/ques), EDG generation: 46.51 ms/ques [INFO] Entity Linking P: 0.7378542510121457, R: 0.687246963562753, micro F1: 0.703609986504723, macro F1: 0.7116520403938166 [INFO] Relation Linking P: 0.5935391363022942, R: 0.5384615384615384, micro F1: 0.5545209176788127, macro F1: 0.5646604345658067 [INFO] Type Linking P: 0.5536437246963563, R: 0.5536437246963563, micro F1: 0.5536437246963563, macro F1: 0.5536437246963563 @@ -13864,8 +13864,8 @@ Question 988: What nation is Hornito located in, where Jamie Bateman Cayn died t [INFO] Golden Answer: [http://dbpedia.org/resource/Panama] [INFO] Predicted Answer: [http://dbpedia.org/resource/Panama] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 989, P: 0.505, R: 0.560, macro F1: 0.513, macro F1*: 0.531 -[INFO] Current time: 2662 ms, average: 2928.89 ms/ques, entity detection: 494.83 ms/ques (linking tool: 400.30 ms/ques), relation detection: 960.12 ms/ques (similarity: 710.30 ms/ques), EDG generation: 45.88 ms/ques +[INFO] Cumulative metrics, sample: 989, P: 0.505, R: 0.560, macro F1: 0.531 +[INFO] Current question consumed time: 2364 ms, average: 2610.74 ms/ques, entity detection: 431.82 ms/ques (linking tool: 353.61 ms/ques), relation detection: 878.40 ms/ques (similarity: 703.05 ms/ques), EDG generation: 46.49 ms/ques [INFO] Entity Linking P: 0.7381193124368048, R: 0.6870576339737108, micro F1: 0.7035726322885065, macro F1: 0.7116737464360537 [INFO] Relation Linking P: 0.5939501179642737, R: 0.538422649140546, micro F1: 0.5546343107516013, macro F1: 0.5648249503373329 [INFO] Type Linking P: 0.5540950455005056, R: 0.5540950455005056, micro F1: 0.5540950455005056, macro F1: 0.5540950455005056 @@ -13878,8 +13878,8 @@ Question 989: Which TV show had a judge named Harry Connick, Jr. and was present [INFO] Golden Answer: [http://dbpedia.org/resource/American_Idol] [INFO] Predicted Answer: [Harry Connick, Jr.@en] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 990, P: 0.505, R: 0.560, macro F1: 0.512, macro F1*: 0.531 -[INFO] Current time: 3679 ms, average: 2929.65 ms/ques, entity detection: 494.75 ms/ques (linking tool: 400.19 ms/ques), relation detection: 960.39 ms/ques (similarity: 710.91 ms/ques), EDG generation: 45.88 ms/ques +[INFO] Cumulative metrics, sample: 990, P: 0.505, R: 0.560, macro F1: 0.531 +[INFO] Current question consumed time: 3219 ms, average: 2611.36 ms/ques, entity detection: 431.73 ms/ques (linking tool: 353.51 ms/ques), relation detection: 878.63 ms/ques (similarity: 703.67 ms/ques), EDG generation: 46.49 ms/ques [INFO] Entity Linking P: 0.7383838383838384, R: 0.6868686868686869, micro F1: 0.7035353535353531, macro F1: 0.7116952659121334 [INFO] Relation Linking P: 0.5933501683501683, R: 0.5378787878787878, micro F1: 0.5540740740740744, macro F1: 0.5642544200844669 [INFO] Type Linking P: 0.5535353535353535, R: 0.5535353535353535, micro F1: 0.5535353535353535, macro F1: 0.5535353535353535 @@ -13892,8 +13892,8 @@ Question 990: Name the resting place of Fazlur Rahman Khan ? [INFO] Golden Answer: [http://dbpedia.org/resource/Chicago, http://dbpedia.org/resource/Illinois, http://dbpedia.org/resource/Graceland_Cemetery] [INFO] Predicted Answer: [http://dbpedia.org/resource/Chicago, http://dbpedia.org/resource/Illinois, http://dbpedia.org/resource/Graceland_Cemetery, Chicago, Illinois@en] [INFO] P: 0.75, R: 1.0, F: 0.8571428571428571 -[INFO] Cumulative metrics, sample: 991, P: 0.505, R: 0.560, macro F1: 0.512, macro F1*: 0.531 -[INFO] Current time: 842 ms, average: 2927.54 ms/ques, entity detection: 494.44 ms/ques (linking tool: 399.89 ms/ques), relation detection: 959.68 ms/ques (similarity: 710.34 ms/ques), EDG generation: 45.84 ms/ques +[INFO] Cumulative metrics, sample: 991, P: 0.505, R: 0.560, macro F1: 0.531 +[INFO] Current question consumed time: 753 ms, average: 2609.48 ms/ques, entity detection: 431.44 ms/ques (linking tool: 353.23 ms/ques), relation detection: 877.97 ms/ques (similarity: 703.10 ms/ques), EDG generation: 46.46 ms/ques [INFO] Entity Linking P: 0.7386478304742684, R: 0.6871846619576185, micro F1: 0.7038345105953577, macro F1: 0.7119875053828404 [INFO] Relation Linking P: 0.5937605112680794, R: 0.5383451059535822, micro F1: 0.5545240497813659, macro F1: 0.5646965450699316 [INFO] Type Linking P: 0.5539858728557013, R: 0.5539858728557013, micro F1: 0.5539858728557013, macro F1: 0.5539858728557013 @@ -13906,8 +13906,8 @@ Question 991: Which arena of WPC Dynamo Moscow is death location of Tigran Petro [INFO] Golden Answer: [http://dbpedia.org/resource/Moscow] [INFO] Predicted Answer: [http://dbpedia.org/resource/Moscow] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 992, P: 0.506, R: 0.561, macro F1: 0.513, macro F1*: 0.532 -[INFO] Current time: 1938 ms, average: 2926.54 ms/ques, entity detection: 494.41 ms/ques (linking tool: 399.86 ms/ques), relation detection: 959.08 ms/ques (similarity: 709.83 ms/ques), EDG generation: 45.99 ms/ques +[INFO] Cumulative metrics, sample: 992, P: 0.506, R: 0.561, macro F1: 0.532 +[INFO] Current question consumed time: 1751 ms, average: 2608.61 ms/ques, entity detection: 431.43 ms/ques (linking tool: 353.23 ms/ques), relation detection: 877.42 ms/ques (similarity: 702.60 ms/ques), EDG generation: 46.62 ms/ques [INFO] Entity Linking P: 0.7389112903225806, R: 0.6875, micro F1: 0.7041330645161286, macro F1: 0.7122791519434629 [INFO] Relation Linking P: 0.5941700268817204, R: 0.5388104838709677, micro F1: 0.5549731182795702, macro F1: 0.5651377700629281 [INFO] Type Linking P: 0.5544354838709677, R: 0.5544354838709677, micro F1: 0.5544354838709677, macro F1: 0.5544354838709677 @@ -13920,8 +13920,8 @@ Question 992: what cities are close by LoDaisKa ? [INFO] Golden Answer: [http://dbpedia.org/resource/Colorado, http://dbpedia.org/resource/Morrison,_Colorado] [INFO] Predicted Answer: [http://dbpedia.org/resource/Colorado, http://dbpedia.org/resource/Morrison,_Colorado] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 993, P: 0.506, R: 0.561, macro F1: 0.513, macro F1*: 0.532 -[INFO] Current time: 503 ms, average: 2924.10 ms/ques, entity detection: 493.94 ms/ques (linking tool: 399.49 ms/ques), relation detection: 958.32 ms/ques (similarity: 709.34 ms/ques), EDG generation: 45.95 ms/ques +[INFO] Cumulative metrics, sample: 993, P: 0.506, R: 0.561, macro F1: 0.532 +[INFO] Current question consumed time: 464 ms, average: 2606.46 ms/ques, entity detection: 431.03 ms/ques (linking tool: 352.89 ms/ques), relation detection: 876.71 ms/ques (similarity: 702.11 ms/ques), EDG generation: 46.59 ms/ques [INFO] Entity Linking P: 0.7391742195367573, R: 0.6878147029204431, micro F1: 0.7044310171198385, macro F1: 0.7125702074009953 [INFO] Relation Linking P: 0.5945787176905001, R: 0.5392749244712991, micro F1: 0.5554212823095002, macro F1: 0.5655780978283108 [INFO] Type Linking P: 0.5548841893252769, R: 0.5548841893252769, micro F1: 0.5548841893252769, macro F1: 0.5548841893252769 @@ -13934,8 +13934,8 @@ Question 993: Which wars were fought when the commanders involved served under P [INFO] Golden Answer: [http://dbpedia.org/resource/Vietnam_War, http://dbpedia.org/resource/Blue_House_Raid, http://dbpedia.org/resource/Korean_DMZ_Conflict_(1966–69), http://dbpedia.org/resource/209th_Detachment,_2325th_Group__Unit_684_Mutiny__1] [INFO] Predicted Answer: [] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 994, P: 0.506, R: 0.561, macro F1: 0.513, macro F1*: 0.532 -[INFO] Current time: 1274 ms, average: 2922.44 ms/ques, entity detection: 493.72 ms/ques (linking tool: 399.29 ms/ques), relation detection: 957.78 ms/ques (similarity: 708.91 ms/ques), EDG generation: 45.94 ms/ques +[INFO] Cumulative metrics, sample: 994, P: 0.506, R: 0.561, macro F1: 0.532 +[INFO] Current question consumed time: 1145 ms, average: 2604.98 ms/ques, entity detection: 430.83 ms/ques (linking tool: 352.72 ms/ques), relation detection: 876.24 ms/ques (similarity: 701.69 ms/ques), EDG generation: 46.57 ms/ques [INFO] Entity Linking P: 0.738430583501006, R: 0.6871227364185111, micro F1: 0.7037223340040237, macro F1: 0.7118533359649784 [INFO] Relation Linking P: 0.5939805499664654, R: 0.5387323943661971, micro F1: 0.5548625083836355, macro F1: 0.5650091057781818 [INFO] Type Linking P: 0.5543259557344065, R: 0.5543259557344065, micro F1: 0.5543259557344065, macro F1: 0.5543259557344065 @@ -13948,8 +13948,8 @@ Question 994: Who is the meaning of Isa ? [INFO] Golden Answer: [http://dbpedia.org/resource/Jesus] [INFO] Predicted Answer: [Ivar Bjørnson, except where noted@en] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 995, P: 0.505, R: 0.560, macro F1: 0.512, macro F1*: 0.531 -[INFO] Current time: 989 ms, average: 2920.50 ms/ques, entity detection: 493.32 ms/ques (linking tool: 398.93 ms/ques), relation detection: 957.23 ms/ques (similarity: 708.44 ms/ques), EDG generation: 45.90 ms/ques +[INFO] Cumulative metrics, sample: 995, P: 0.505, R: 0.560, macro F1: 0.531 +[INFO] Current question consumed time: 891 ms, average: 2603.26 ms/ques, entity detection: 430.49 ms/ques (linking tool: 352.40 ms/ques), relation detection: 875.74 ms/ques (similarity: 701.23 ms/ques), EDG generation: 46.53 ms/ques [INFO] Entity Linking P: 0.7376884422110552, R: 0.6864321608040201, micro F1: 0.703015075376884, macro F1: 0.7111379054765713 [INFO] Relation Linking P: 0.5933835845896147, R: 0.5381909547738694, micro F1: 0.5543048576214409, macro F1: 0.564441257430666 [INFO] Type Linking P: 0.5547738693467337, R: 0.5547738693467337, micro F1: 0.5547738693467337, macro F1: 0.5547738693467337 @@ -13962,8 +13962,8 @@ Question 995: Who is venerated in Judaism and Islam? [INFO] Golden Answer: [http://dbpedia.org/resource/Seth, http://dbpedia.org/resource/Zechariah_ben_Jehoiada, http://dbpedia.org/resource/Amos_(prophet), http://dbpedia.org/resource/Ezekiel, http://dbpedia.org/resource/Hosea, http://dbpedia.org/resource/Isaac, http://dbpedia.org/resource/Joshua, http://dbpedia.org/resource/Moses, http://dbpedia.org/resource/Noah, http://dbpedia.org/resource/Aaron, http://dbpedia.org/resource/Elijah, http://dbpedia.org/resource/Elisha, http://dbpedia.org/resource/Gabriel, http://dbpedia.org/resource/Isaiah, http://dbpedia.org/resource/Ishmael, http://dbpedia.org/resource/Michael_(archangel), http://dbpedia.org/resource/Raphael_(archangel), http://dbpedia.org/resource/Samuel, http://dbpedia.org/resource/Jonah, http://dbpedia.org/resource/Daniel_(biblical_figure), http://dbpedia.org/resource/Habakkuk, http://dbpedia.org/resource/Micah_(prophet), http://dbpedia.org/resource/Rachel, http://dbpedia.org/resource/Abraham__Abraham__1, http://dbpedia.org/resource/David__King_David_the_Prophet__1, http://dbpedia.org/resource/Jacob__/Israel__1] [INFO] Predicted Answer: [http://dbpedia.org/resource/Seth, http://dbpedia.org/resource/Zechariah_ben_Jehoiada, http://dbpedia.org/resource/Amos_(prophet), http://dbpedia.org/resource/Ezekiel, http://dbpedia.org/resource/Hosea, http://dbpedia.org/resource/Isaac, http://dbpedia.org/resource/Joshua, http://dbpedia.org/resource/Moses, http://dbpedia.org/resource/Noah, http://dbpedia.org/resource/Aaron, http://dbpedia.org/resource/Elijah, http://dbpedia.org/resource/Elisha, http://dbpedia.org/resource/Gabriel, http://dbpedia.org/resource/Isaiah, http://dbpedia.org/resource/Ishmael, http://dbpedia.org/resource/Michael_(archangel), http://dbpedia.org/resource/Raphael_(archangel), http://dbpedia.org/resource/Samuel, http://dbpedia.org/resource/Jonah, http://dbpedia.org/resource/Daniel_(biblical_figure), http://dbpedia.org/resource/Habakkuk, http://dbpedia.org/resource/Micah_(prophet), http://dbpedia.org/resource/Rachel, http://dbpedia.org/resource/Abraham__Abraham__1, http://dbpedia.org/resource/David__King_David_the_Prophet__1, http://dbpedia.org/resource/Jacob__/Israel__1, http://dbpedia.org/resource/Abraham, http://dbpedia.org/resource/Jacob, http://dbpedia.org/resource/David] [INFO] P: 0.896551724137931, R: 1.0, F: 0.9454545454545454 -[INFO] Cumulative metrics, sample: 996, P: 0.506, R: 0.560, macro F1: 0.513, macro F1*: 0.532 -[INFO] Current time: 7301 ms, average: 2924.90 ms/ques, entity detection: 494.29 ms/ques (linking tool: 399.82 ms/ques), relation detection: 958.74 ms/ques (similarity: 708.86 ms/ques), EDG generation: 45.88 ms/ques +[INFO] Cumulative metrics, sample: 996, P: 0.506, R: 0.560, macro F1: 0.532 +[INFO] Current question consumed time: 6490 ms, average: 2607.16 ms/ques, entity detection: 431.37 ms/ques (linking tool: 353.21 ms/ques), relation detection: 877.12 ms/ques (similarity: 701.58 ms/ques), EDG generation: 46.51 ms/ques [INFO] Entity Linking P: 0.7379518072289156, R: 0.6867469879518072, micro F1: 0.7033132530120477, macro F1: 0.7114292264194197 [INFO] Relation Linking P: 0.5937918340026773, R: 0.5386546184738956, micro F1: 0.5547523427041502, macro F1: 0.5648809497316937 [INFO] Type Linking P: 0.5552208835341366, R: 0.5552208835341366, micro F1: 0.5552208835341366, macro F1: 0.5552208835341366 @@ -13976,8 +13976,8 @@ Question 996: Count the number teams which have former ice hockey players which [INFO] Golden Answer: [898] [INFO] Predicted Answer: [8] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 997, P: 0.505, R: 0.560, macro F1: 0.512, macro F1*: 0.531 -[INFO] Current time: 4509 ms, average: 2926.48 ms/ques, entity detection: 494.55 ms/ques (linking tool: 400.08 ms/ques), relation detection: 959.14 ms/ques (similarity: 709.16 ms/ques), EDG generation: 45.87 ms/ques +[INFO] Cumulative metrics, sample: 997, P: 0.505, R: 0.560, macro F1: 0.531 +[INFO] Current question consumed time: 4155 ms, average: 2608.72 ms/ques, entity detection: 431.64 ms/ques (linking tool: 353.48 ms/ques), relation detection: 877.52 ms/ques (similarity: 701.87 ms/ques), EDG generation: 46.50 ms/ques [INFO] Entity Linking P: 0.7377131394182548, R: 0.6870611835506519, micro F1: 0.7032764961551315, macro F1: 0.7114868011284823 [INFO] Relation Linking P: 0.5931962554329655, R: 0.5381143430290872, micro F1: 0.5541959210966235, macro F1: 0.5643143690398864 [INFO] Type Linking P: 0.555667001003009, R: 0.555667001003009, micro F1: 0.555667001003009, macro F1: 0.555667001003009 @@ -13990,8 +13990,8 @@ Question 997: Name the common editor of Easy Street (film) and Work (film)? [INFO] Golden Answer: [http://dbpedia.org/resource/Charlie_Chaplin] [INFO] Predicted Answer: [http://dbpedia.org/resource/Charlie_Chaplin, Charles Chaplin@en] [INFO] P: 0.5, R: 1.0, F: 0.6666666666666666 -[INFO] Cumulative metrics, sample: 998, P: 0.505, R: 0.560, macro F1: 0.512, macro F1*: 0.531 -[INFO] Current time: 984 ms, average: 2924.54 ms/ques, entity detection: 494.24 ms/ques (linking tool: 399.80 ms/ques), relation detection: 958.43 ms/ques (similarity: 708.70 ms/ques), EDG generation: 45.88 ms/ques +[INFO] Cumulative metrics, sample: 998, P: 0.505, R: 0.560, macro F1: 0.531 +[INFO] Current question consumed time: 881 ms, average: 2606.99 ms/ques, entity detection: 431.36 ms/ques (linking tool: 353.22 ms/ques), relation detection: 876.88 ms/ques (similarity: 701.42 ms/ques), EDG generation: 46.52 ms/ques [INFO] Entity Linking P: 0.7379759519038076, R: 0.68687374749499, micro F1: 0.7032398129592513, macro F1: 0.7115084599578904 [INFO] Relation Linking P: 0.5936038744154977, R: 0.5385771543086172, micro F1: 0.5546426185704746, macro F1: 0.5647532989128933 [INFO] Type Linking P: 0.5561122244488977, R: 0.5561122244488977, micro F1: 0.5561122244488977, macro F1: 0.5561122244488977 @@ -14004,8 +14004,8 @@ Question 998: List all the schools of the rugby player whose went to school name [INFO] Golden Answer: [http://dbpedia.org/resource/Pietermaritzburg, http://dbpedia.org/resource/Maritzburg_College, http://dbpedia.org/resource/South_Africa, http://dbpedia.org/resource/KwaZulu-Natal, http://dbpedia.org/resource/Hilton_College_(South_Africa)] [INFO] Predicted Answer: [http://dbpedia.org/resource/Maritzburg_College, http://dbpedia.org/resource/Hilton_College_(South_Africa), http://dbpedia.org/resource/Pietermaritzburg, http://dbpedia.org/resource/South_Africa, http://dbpedia.org/resource/KwaZulu-Natal] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 999, P: 0.506, R: 0.561, macro F1: 0.513, macro F1*: 0.532 -[INFO] Current time: 3591 ms, average: 2925.21 ms/ques, entity detection: 493.95 ms/ques (linking tool: 399.52 ms/ques), relation detection: 959.20 ms/ques (similarity: 709.68 ms/ques), EDG generation: 45.87 ms/ques +[INFO] Cumulative metrics, sample: 999, P: 0.506, R: 0.561, macro F1: 0.532 +[INFO] Current question consumed time: 3371 ms, average: 2607.75 ms/ques, entity detection: 431.12 ms/ques (linking tool: 352.98 ms/ques), relation detection: 877.59 ms/ques (similarity: 702.45 ms/ques), EDG generation: 46.53 ms/ques [INFO] Entity Linking P: 0.7382382382382382, R: 0.6871871871871872, micro F1: 0.7035368702035364, macro F1: 0.7117985260541442 [INFO] Relation Linking P: 0.5940106773440106, R: 0.539039039039039, micro F1: 0.5550884217550888, macro F1: 0.5651913416766542 [INFO] Type Linking P: 0.5555555555555556, R: 0.5555555555555556, micro F1: 0.5555555555555556, macro F1: 0.5555555555555556 @@ -14018,11 +14018,11 @@ Question 999: Name the office of Richard Coke ? [INFO] Golden Answer: [http://dbpedia.org/resource/United_States_Senate, Governor of Texas@en, from Texas@en] [INFO] Predicted Answer: [Governor of Texas, http://dbpedia.org/resource/United_States_Senate, Governor of Texas@en, from Texas@en] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 1000, P: 0.506, R: 0.561, macro F1: 0.513, macro F1*: 0.532 -[INFO] Current time: 909 ms, average: 2923.19 ms/ques, entity detection: 493.70 ms/ques (linking tool: 399.29 ms/ques), relation detection: 958.52 ms/ques (similarity: 709.13 ms/ques), EDG generation: 45.83 ms/ques +[INFO] Cumulative metrics, sample: 1000, P: 0.506, R: 0.561, macro F1: 0.532 +[INFO] Current question consumed time: 781 ms, average: 2605.92 ms/ques, entity detection: 430.90 ms/ques (linking tool: 352.79 ms/ques), relation detection: 876.97 ms/ques (similarity: 701.92 ms/ques), EDG generation: 46.50 ms/ques [INFO] Entity Linking P: 0.7385, R: 0.6875, micro F1: 0.7038333333333328, macro F1: 0.7120880084151472 [INFO] Relation Linking P: 0.5944166666666666, R: 0.5395, micro F1: 0.5555333333333337, macro F1: 0.5656285000367458 [INFO] Type Linking P: 0.556, R: 0.556, micro F1: 0.556, macro F1: 0.556 =============================================== -[INFO] QuestionSolver total time: 2923189 ms +[INFO] QuestionSolver total time: 2605924 ms [INFO] query generation finished diff --git a/query_logs/query_log_test_qald-9_2021_07_04 17_50.txt b/query_logs/query_log_test_qald-9_2021_07_09 00_51.txt similarity index 95% rename from query_logs/query_log_test_qald-9_2021_07_04 17_50.txt rename to query_logs/query_log_test_qald-9_2021_07_09 00_51.txt index 0fc85f6..b6c2a20 100644 --- a/query_logs/query_log_test_qald-9_2021_07_04 17_50.txt +++ b/query_logs/query_log_test_qald-9_2021_07_09 00_51.txt @@ -7,9 +7,9 @@ Question 0: What is the time zone of Salt Lake City? [INFO] Golden Answer: [http://dbpedia.org/resource/Mountain_Time_Zone] [INFO] Predicted Answer: [http://dbpedia.org/resource/Mountain_Time_Zone] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 1, P: 1.000, R: 1.000, macro F1: 1.000, macro F1*: 1.000 -[INFO] QALD Cumulative metrics, sample: 1, P: 1.000, R: 1.000, macro F1: 1.000, QALD macro F1: 1.000 -[INFO] Current time: 5870 ms, average: 5892.00 ms/ques, entity detection: 520.00 ms/ques (linking tool: 456.00 ms/ques), relation detection: 2069.00 ms/ques (similarity: 1366.00 ms/ques), EDG generation: 1611.00 ms/ques +[INFO] Cumulative metrics, sample: 1, P: 1.000, R: 1.000, macro F1: 1.000 +[INFO] QALD Cumulative metrics, sample: 1, P: 1.000, R: 1.000, average F1: 1.000, QALD macro F1: 1.000 +[INFO] Current question consumed time: 7577 ms, average: 7598.00 ms/ques, entity detection: 462.00 ms/ques (linking tool: 403.00 ms/ques), relation detection: 2246.00 ms/ques (similarity: 1634.00 ms/ques), EDG generation: 3280.00 ms/ques [INFO] Entity Linking P: 0.0, R: 0.0, micro F1: 0.0, macro F1: 0.0 [INFO] Relation Linking P: 0.0, R: 0.0, micro F1: 0.0, macro F1: 0.0 [INFO] Type Linking P: 1.0, R: 1.0, micro F1: 1.0, macro F1: 1.0 @@ -22,9 +22,9 @@ Question 1: Who killed Caesar? [INFO] Golden Answer: [http://dbpedia.org/resource/Quintus_Antistius_Labeo, http://dbpedia.org/resource/Decimus_Junius_Brutus_Albinus, http://dbpedia.org/resource/Gaius_Cassius_Longinus, http://dbpedia.org/resource/Lucius_Cornelius_Cinna_(suffect_consul), http://dbpedia.org/resource/Marcus_Junius_Brutus_the_Younger, http://dbpedia.org/resource/Servilius_Casca, http://dbpedia.org/resource/Tillius_Cimber, http://dbpedia.org/resource/Lucius_Minucius_Basilus, http://dbpedia.org/resource/Gaius_Cassius_Parmensis] [INFO] Predicted Answer: [] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 2, P: 0.500, R: 0.500, macro F1: 0.500, macro F1*: 0.500 -[INFO] QALD Cumulative metrics, sample: 2, P: 1.000, R: 0.500, macro F1: 0.500, QALD macro F1: 0.667 -[INFO] Current time: 1012 ms, average: 3452.00 ms/ques, entity detection: 363.50 ms/ques (linking tool: 272.50 ms/ques), relation detection: 1167.00 ms/ques (similarity: 758.50 ms/ques), EDG generation: 814.50 ms/ques +[INFO] Cumulative metrics, sample: 2, P: 0.500, R: 0.500, macro F1: 0.500 +[INFO] QALD Cumulative metrics, sample: 2, P: 1.000, R: 0.500, average F1: 0.500, QALD macro F1: 0.667 +[INFO] Current question consumed time: 922 ms, average: 4260.50 ms/ques, entity detection: 318.50 ms/ques (linking tool: 242.00 ms/ques), relation detection: 1249.50 ms/ques (similarity: 893.00 ms/ques), EDG generation: 1647.50 ms/ques [INFO] Entity Linking P: 0.5, R: 0.5, micro F1: 0.5, macro F1: 0.5 [INFO] Relation Linking P: 0.5, R: 0.5, micro F1: 0.5, macro F1: 0.5 [INFO] Type Linking P: 1.0, R: 1.0, micro F1: 1.0, macro F1: 1.0 @@ -37,9 +37,9 @@ Question 2: What is the highest mountain in Germany? [INFO] Golden Answer: [http://dbpedia.org/resource/Zugspitze] [INFO] Predicted Answer: [http://dbpedia.org/resource/Fuchskaute, http://dbpedia.org/resource/Taubenkopf_(Haardt)] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 3, P: 0.333, R: 0.333, macro F1: 0.333, macro F1*: 0.333 -[INFO] QALD Cumulative metrics, sample: 3, P: 0.667, R: 0.333, macro F1: 0.333, QALD macro F1: 0.444 -[INFO] Current time: 19073 ms, average: 8659.00 ms/ques, entity detection: 1972.67 ms/ques (linking tool: 1864.67 ms/ques), relation detection: 2564.67 ms/ques (similarity: 1182.33 ms/ques), EDG generation: 551.33 ms/ques +[INFO] Cumulative metrics, sample: 3, P: 0.333, R: 0.333, macro F1: 0.333 +[INFO] QALD Cumulative metrics, sample: 3, P: 0.667, R: 0.333, average F1: 0.333, QALD macro F1: 0.444 +[INFO] Current question consumed time: 18549 ms, average: 9023.33 ms/ques, entity detection: 1861.33 ms/ques (linking tool: 1760.33 ms/ques), relation detection: 2625.67 ms/ques (similarity: 1271.33 ms/ques), EDG generation: 1103.67 ms/ques [INFO] Entity Linking P: 0.3333333333333333, R: 0.3333333333333333, micro F1: 0.3333333333333333, macro F1: 0.3333333333333333 [INFO] Relation Linking P: 0.3333333333333333, R: 0.3333333333333333, micro F1: 0.3333333333333333, macro F1: 0.3333333333333333 [INFO] Type Linking P: 1.0, R: 1.0, micro F1: 1.0, macro F1: 1.0 @@ -52,9 +52,9 @@ Question 3: Which American presidents were in office during the Vietnam War? [INFO] Golden Answer: [] [INFO] Predicted Answer: [http://dbpedia.org/resource/Forrest_Gump_(character), http://dbpedia.org/resource/David_Wade_(Louisiana)] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 4, P: 0.250, R: 0.250, macro F1: 0.250, macro F1*: 0.250 -[INFO] QALD Cumulative metrics, sample: 4, P: 0.500, R: 0.250, macro F1: 0.250, QALD macro F1: 0.333 -[INFO] Current time: 2778 ms, average: 7188.75 ms/ques, entity detection: 1685.00 ms/ques (linking tool: 1583.00 ms/ques), relation detection: 2112.00 ms/ques (similarity: 1044.75 ms/ques), EDG generation: 421.00 ms/ques +[INFO] Cumulative metrics, sample: 4, P: 0.250, R: 0.250, macro F1: 0.250 +[INFO] QALD Cumulative metrics, sample: 4, P: 0.500, R: 0.250, average F1: 0.250, QALD macro F1: 0.333 +[INFO] Current question consumed time: 2856 ms, average: 7481.50 ms/ques, entity detection: 1592.25 ms/ques (linking tool: 1496.50 ms/ques), relation detection: 2156.75 ms/ques (similarity: 1113.75 ms/ques), EDG generation: 836.00 ms/ques [INFO] Entity Linking P: 0.25, R: 0.25, micro F1: 0.25, macro F1: 0.25 [INFO] Relation Linking P: 0.25, R: 0.25, micro F1: 0.25, macro F1: 0.25 [INFO] Type Linking P: 1.0, R: 1.0, micro F1: 1.0, macro F1: 1.0 @@ -67,9 +67,9 @@ Question 4: Butch Otter is the governor of which U.S. state? [INFO] Golden Answer: [http://dbpedia.org/resource/Idaho] [INFO] Predicted Answer: [http://dbpedia.org/resource/Brad_Little_(politician), http://dbpedia.org/resource/Jim_Risch, http://dbpedia.org/resource/Lori_Otter, http://dbpedia.org/resource/Donna_Jones_(Idaho_politician), http://dbpedia.org/resource/Lawerence_Denney, http://dbpedia.org/resource/Lawrence_Wasden, http://dbpedia.org/resource/Brandon_Woolf, http://dbpedia.org/resource/Phil_Batt, http://dbpedia.org/resource/Dirk_Kempthorne, http://dbpedia.org/resource/Cecil_D._Andrus, http://dbpedia.org/resource/Idaho] [INFO] P: 0.09090909090909091, R: 1.0, F: 0.16666666666666669 -[INFO] Cumulative metrics, sample: 5, P: 0.218, R: 0.400, macro F1: 0.233, macro F1*: 0.282 -[INFO] QALD Cumulative metrics, sample: 5, P: 0.418, R: 0.400, macro F1: 0.233, QALD macro F1: 0.409 -[INFO] Current time: 2926 ms, average: 6336.20 ms/ques, entity detection: 1509.60 ms/ques (linking tool: 1397.00 ms/ques), relation detection: 1793.20 ms/ques (similarity: 901.40 ms/ques), EDG generation: 343.60 ms/ques +[INFO] Cumulative metrics, sample: 5, P: 0.218, R: 0.400, macro F1: 0.282 +[INFO] QALD Cumulative metrics, sample: 5, P: 0.418, R: 0.400, average F1: 0.233, QALD macro F1: 0.409 +[INFO] Current question consumed time: 2727 ms, average: 6530.60 ms/ques, entity detection: 1417.00 ms/ques (linking tool: 1308.80 ms/ques), relation detection: 1824.00 ms/ques (similarity: 954.60 ms/ques), EDG generation: 674.00 ms/ques [INFO] Entity Linking P: 0.2, R: 0.2, micro F1: 0.2, macro F1: 0.20000000000000004 [INFO] Relation Linking P: 0.2, R: 0.2, micro F1: 0.2, macro F1: 0.20000000000000004 [INFO] Type Linking P: 1.0, R: 1.0, micro F1: 1.0, macro F1: 1.0 @@ -82,9 +82,9 @@ Question 5: How many gold medals did Michael Phelps win at the 2008 Olympics? [INFO] Golden Answer: [8] [INFO] Predicted Answer: [23] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 6, P: 0.182, R: 0.333, macro F1: 0.194, macro F1*: 0.235 -[INFO] QALD Cumulative metrics, sample: 6, P: 0.348, R: 0.333, macro F1: 0.194, QALD macro F1: 0.341 -[INFO] Current time: 2479 ms, average: 5693.33 ms/ques, entity detection: 1361.67 ms/ques (linking tool: 1243.67 ms/ques), relation detection: 1582.50 ms/ques (similarity: 845.00 ms/ques), EDG generation: 297.67 ms/ques +[INFO] Cumulative metrics, sample: 6, P: 0.182, R: 0.333, macro F1: 0.235 +[INFO] QALD Cumulative metrics, sample: 6, P: 0.348, R: 0.333, average F1: 0.194, QALD macro F1: 0.341 +[INFO] Current question consumed time: 2134 ms, average: 5797.83 ms/ques, entity detection: 1268.83 ms/ques (linking tool: 1157.00 ms/ques), relation detection: 1601.33 ms/ques (similarity: 887.83 ms/ques), EDG generation: 567.33 ms/ques [INFO] Entity Linking P: 0.16666666666666666, R: 0.16666666666666666, micro F1: 0.16666666666666666, macro F1: 0.16666666666666666 [INFO] Relation Linking P: 0.16666666666666666, R: 0.16666666666666666, micro F1: 0.16666666666666666, macro F1: 0.16666666666666666 [INFO] Type Linking P: 1.0, R: 1.0, micro F1: 1.0, macro F1: 1.0 @@ -97,9 +97,9 @@ Question 6: Which artists were born on the same date as Rachel Stevens? [INFO] Golden Answer: [http://dbpedia.org/resource/Gizem_Saka, http://dbpedia.org/resource/Cameron_Cartio, http://dbpedia.org/resource/Vesna_Pisarović] [INFO] Predicted Answer: [http://dbpedia.org/resource/Funky_Dory, http://dbpedia.org/resource/So_Good_(Rachel_Stevens_song), http://dbpedia.org/resource/Sweet_Dreams_My_LA_Ex, http://dbpedia.org/resource/Funky_Dory_(song), http://dbpedia.org/resource/More,_More,_More, http://dbpedia.org/resource/I_Said_Never_Again_(But_Here_We_Are), http://dbpedia.org/resource/Negotiate_with_Love, http://dbpedia.org/resource/Some_Girls_(Rachel_Stevens_song), http://dbpedia.org/resource/Come_and_Get_It_(Rachel_Stevens_album), http://dbpedia.org/resource/Rachel_Stevens_discography] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 7, P: 0.156, R: 0.286, macro F1: 0.167, macro F1*: 0.202 -[INFO] QALD Cumulative metrics, sample: 7, P: 0.299, R: 0.286, macro F1: 0.167, QALD macro F1: 0.292 -[INFO] Current time: 1516 ms, average: 5096.57 ms/ques, entity detection: 1192.71 ms/ques (linking tool: 1081.14 ms/ques), relation detection: 1427.71 ms/ques (similarity: 801.71 ms/ques), EDG generation: 259.14 ms/ques +[INFO] Cumulative metrics, sample: 7, P: 0.156, R: 0.286, macro F1: 0.202 +[INFO] QALD Cumulative metrics, sample: 7, P: 0.299, R: 0.286, average F1: 0.167, QALD macro F1: 0.292 +[INFO] Current question consumed time: 1379 ms, average: 5166.57 ms/ques, entity detection: 1109.71 ms/ques (linking tool: 1003.29 ms/ques), relation detection: 1441.00 ms/ques (similarity: 837.57 ms/ques), EDG generation: 490.86 ms/ques [INFO] Entity Linking P: 0.14285714285714285, R: 0.14285714285714285, micro F1: 0.14285714285714285, macro F1: 0.14285714285714285 [INFO] Relation Linking P: 0.14285714285714285, R: 0.14285714285714285, micro F1: 0.14285714285714285, macro F1: 0.14285714285714285 [INFO] Type Linking P: 1.0, R: 1.0, micro F1: 1.0, macro F1: 1.0 @@ -112,9 +112,9 @@ Question 7: What is the profession of Frank Herbert? [INFO] Golden Answer: [Novelist] [INFO] Predicted Answer: [Novelist] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 8, P: 0.261, R: 0.375, macro F1: 0.271, macro F1*: 0.308 -[INFO] QALD Cumulative metrics, sample: 8, P: 0.386, R: 0.375, macro F1: 0.271, QALD macro F1: 0.381 -[INFO] Current time: 1092 ms, average: 4596.00 ms/ques, entity detection: 1070.12 ms/ques (linking tool: 962.12 ms/ques), relation detection: 1297.75 ms/ques (similarity: 727.62 ms/ques), EDG generation: 228.38 ms/ques +[INFO] Cumulative metrics, sample: 8, P: 0.261, R: 0.375, macro F1: 0.308 +[INFO] QALD Cumulative metrics, sample: 8, P: 0.386, R: 0.375, average F1: 0.271, QALD macro F1: 0.381 +[INFO] Current question consumed time: 979 ms, average: 4643.12 ms/ques, entity detection: 993.38 ms/ques (linking tool: 891.62 ms/ques), relation detection: 1305.38 ms/ques (similarity: 759.12 ms/ques), EDG generation: 431.12 ms/ques [INFO] Entity Linking P: 0.125, R: 0.125, micro F1: 0.125, macro F1: 0.125 [INFO] Relation Linking P: 0.125, R: 0.125, micro F1: 0.125, macro F1: 0.125 [INFO] Type Linking P: 1.0, R: 1.0, micro F1: 1.0, macro F1: 1.0 @@ -127,9 +127,9 @@ Question 8: Are Taiko some kind of Japanese musical instrument? [INFO] Golden Answer: [true] [INFO] Predicted Answer: [true] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 9, P: 0.343, R: 0.444, macro F1: 0.352, macro F1*: 0.387 -[INFO] QALD Cumulative metrics, sample: 9, P: 0.455, R: 0.444, macro F1: 0.352, QALD macro F1: 0.449 -[INFO] Current time: 968 ms, average: 4192.89 ms/ques, entity detection: 981.78 ms/ques (linking tool: 873.00 ms/ques), relation detection: 1160.78 ms/ques (similarity: 651.33 ms/ques), EDG generation: 204.44 ms/ques +[INFO] Cumulative metrics, sample: 9, P: 0.343, R: 0.444, macro F1: 0.387 +[INFO] QALD Cumulative metrics, sample: 9, P: 0.455, R: 0.444, average F1: 0.352, QALD macro F1: 0.449 +[INFO] Current question consumed time: 846 ms, average: 4221.22 ms/ques, entity detection: 907.78 ms/ques (linking tool: 807.11 ms/ques), relation detection: 1167.33 ms/ques (similarity: 679.22 ms/ques), EDG generation: 384.67 ms/ques [INFO] Entity Linking P: 0.1111111111111111, R: 0.1111111111111111, micro F1: 0.1111111111111111, macro F1: 0.1111111111111111 [INFO] Relation Linking P: 0.2222222222222222, R: 0.2222222222222222, micro F1: 0.2222222222222222, macro F1: 0.2222222222222222 [INFO] Type Linking P: 1.0, R: 1.0, micro F1: 1.0, macro F1: 1.0 @@ -142,9 +142,9 @@ Question 9: How many seats does the home stadium of FC Porto have? [INFO] Golden Answer: [52000] [INFO] Predicted Answer: [159] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 10, P: 0.309, R: 0.400, macro F1: 0.317, macro F1*: 0.349 -[INFO] QALD Cumulative metrics, sample: 10, P: 0.409, R: 0.400, macro F1: 0.317, QALD macro F1: 0.404 -[INFO] Current time: 6012 ms, average: 4374.80 ms/ques, entity detection: 922.50 ms/ques (linking tool: 818.50 ms/ques), relation detection: 1161.10 ms/ques (similarity: 665.70 ms/ques), EDG generation: 440.60 ms/ques +[INFO] Cumulative metrics, sample: 10, P: 0.309, R: 0.400, macro F1: 0.349 +[INFO] QALD Cumulative metrics, sample: 10, P: 0.409, R: 0.400, average F1: 0.317, QALD macro F1: 0.404 +[INFO] Current question consumed time: 4225 ms, average: 4221.60 ms/ques, entity detection: 850.50 ms/ques (linking tool: 754.00 ms/ques), relation detection: 1161.80 ms/ques (similarity: 691.50 ms/ques), EDG generation: 456.60 ms/ques [INFO] Entity Linking P: 0.1, R: 0.1, micro F1: 0.1, macro F1: 0.10000000000000002 [INFO] Relation Linking P: 0.2, R: 0.2, micro F1: 0.2, macro F1: 0.20000000000000004 [INFO] Type Linking P: 1.0, R: 1.0, micro F1: 1.0, macro F1: 1.0 @@ -157,9 +157,9 @@ Question 10: Which frequent flyer program has the most airlines? [INFO] Golden Answer: [http://dbpedia.org/resource/Miles_&_More] [INFO] Predicted Answer: [] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 11, P: 0.281, R: 0.364, macro F1: 0.288, macro F1*: 0.317 -[INFO] QALD Cumulative metrics, sample: 11, P: 0.463, R: 0.364, macro F1: 0.288, QALD macro F1: 0.407 -[INFO] Current time: 541 ms, average: 4026.27 ms/ques, entity detection: 851.91 ms/ques (linking tool: 751.64 ms/ques), relation detection: 1065.64 ms/ques (similarity: 611.55 ms/ques), EDG generation: 402.00 ms/ques +[INFO] Cumulative metrics, sample: 11, P: 0.281, R: 0.364, macro F1: 0.317 +[INFO] QALD Cumulative metrics, sample: 11, P: 0.463, R: 0.364, average F1: 0.288, QALD macro F1: 0.407 +[INFO] Current question consumed time: 488 ms, average: 3882.18 ms/ques, entity detection: 784.73 ms/ques (linking tool: 691.73 ms/ques), relation detection: 1065.64 ms/ques (similarity: 635.00 ms/ques), EDG generation: 416.45 ms/ques [INFO] Entity Linking P: 0.18181818181818182, R: 0.18181818181818182, micro F1: 0.18181818181818182, macro F1: 0.18181818181818182 [INFO] Relation Linking P: 0.2727272727272727, R: 0.2727272727272727, micro F1: 0.2727272727272727, macro F1: 0.2727272727272727 [INFO] Type Linking P: 1.0, R: 1.0, micro F1: 1.0, macro F1: 1.0 @@ -172,9 +172,9 @@ Question 11: Which European countries have a constitutional monarchy? [INFO] Golden Answer: [http://dbpedia.org/resource/Belgium, http://dbpedia.org/resource/Denmark, http://dbpedia.org/resource/Liechtenstein, http://dbpedia.org/resource/Sweden] [INFO] Predicted Answer: [http://dbpedia.org/resource/Kelantan] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 12, P: 0.258, R: 0.333, macro F1: 0.264, macro F1*: 0.291 -[INFO] QALD Cumulative metrics, sample: 12, P: 0.424, R: 0.333, macro F1: 0.264, QALD macro F1: 0.373 -[INFO] Current time: 1233 ms, average: 3793.50 ms/ques, entity detection: 796.00 ms/ques (linking tool: 698.25 ms/ques), relation detection: 1005.25 ms/ques (similarity: 587.08 ms/ques), EDG generation: 369.50 ms/ques +[INFO] Cumulative metrics, sample: 12, P: 0.258, R: 0.333, macro F1: 0.291 +[INFO] QALD Cumulative metrics, sample: 12, P: 0.424, R: 0.333, average F1: 0.264, QALD macro F1: 0.373 +[INFO] Current question consumed time: 1162 ms, average: 3655.50 ms/ques, entity detection: 733.83 ms/ques (linking tool: 643.00 ms/ques), relation detection: 1004.42 ms/ques (similarity: 608.83 ms/ques), EDG generation: 382.83 ms/ques [INFO] Entity Linking P: 0.16666666666666666, R: 0.16666666666666666, micro F1: 0.16666666666666666, macro F1: 0.16666666666666666 [INFO] Relation Linking P: 0.25, R: 0.25, micro F1: 0.25, macro F1: 0.25 [INFO] Type Linking P: 1.0, R: 1.0, micro F1: 1.0, macro F1: 1.0 @@ -187,9 +187,9 @@ Question 12: Which countries have places with more than two caves? [INFO] Golden Answer: [http://dbpedia.org/resource/Austria, http://dbpedia.org/resource/Philippines, http://dbpedia.org/resource/Republic_of_Ireland, http://dbpedia.org/resource/Turkey, http://dbpedia.org/resource/Cumbria, http://dbpedia.org/resource/India, http://dbpedia.org/resource/England, http://dbpedia.org/resource/France, http://dbpedia.org/resource/Serbia, http://dbpedia.org/resource/United_States, http://dbpedia.org/resource/Abkhazia, http://dbpedia.org/resource/Germany, http://dbpedia.org/resource/Lancashire, http://dbpedia.org/resource/North_Yorkshire, http://dbpedia.org/resource/China, http://dbpedia.org/resource/Georgia_(country), http://dbpedia.org/resource/Gibraltar, http://dbpedia.org/resource/Slovenia, http://dbpedia.org/resource/Brazil, http://dbpedia.org/resource/Mexico, http://dbpedia.org/resource/Derbyshire, http://dbpedia.org/resource/Venezuela, http://dbpedia.org/resource/Australia, http://dbpedia.org/resource/Italy, http://dbpedia.org/resource/Vietnam, http://dbpedia.org/resource/Wales, http://dbpedia.org/resource/Somerset, http://dbpedia.org/resource/South_Africa, http://dbpedia.org/resource/UK, http://dbpedia.org/resource/Greece, http://dbpedia.org/resource/Jamaica] [INFO] Predicted Answer: [http://dbpedia.org/resource/Abadan, http://dbpedia.org/resource/American_literature, http://dbpedia.org/resource/American_people, http://dbpedia.org/resource/Ancient_Egyptian, http://dbpedia.org/resource/Arab_states_of_the_Persian_Gulf, http://dbpedia.org/resource/Arkhangelsk_Oblast, http://dbpedia.org/resource/Arzawa, http://dbpedia.org/resource/Australia_cricket_team, http://dbpedia.org/resource/Australia_women's_national_basketball_team, http://dbpedia.org/resource/Australian_cricket_team, http://dbpedia.org/resource/BDSM, http://dbpedia.org/resource/Bahá'í_Faith_in_Iran, http://dbpedia.org/resource/Baku_Khanate, http://dbpedia.org/resource/Balinese_Hinduism, http://dbpedia.org/resource/Baltimore-Washington_Metropolitan_Area, http://dbpedia.org/resource/Barbados_cricket_team, http://dbpedia.org/resource/Batavian_Republic, http://dbpedia.org/resource/Beylik_of_Teke, http://dbpedia.org/resource/Bishopric_of_Lebus, http://dbpedia.org/resource/Breton_people, http://dbpedia.org/resource/British_Boxing_Board_of_Control, http://dbpedia.org/resource/British_Citizen, http://dbpedia.org/resource/British_Malaya, http://dbpedia.org/resource/British_Pakistani, http://dbpedia.org/resource/British_nationality, http://dbpedia.org/resource/Britishness, http://dbpedia.org/resource/Bronze_Age_India, http://dbpedia.org/resource/Brugge, http://dbpedia.org/resource/Canton_Ticino, http://dbpedia.org/resource/Caroline_Islands, http://dbpedia.org/resource/Carpathian_Ruthenia, http://dbpedia.org/resource/Celtic_Neopaganism, http://dbpedia.org/resource/Cherson_(theme), http://dbpedia.org/resource/Cinema_of_China, http://dbpedia.org/resource/Cinema_of_India, http://dbpedia.org/resource/Coalition_Provisional_Authority, http://dbpedia.org/resource/Colony_of_Natal, http://dbpedia.org/resource/County_of_Artois, http://dbpedia.org/resource/County_of_Barcelona, http://dbpedia.org/resource/County_of_Burgundy, http://dbpedia.org/resource/Crown_of_Bohemia, http://dbpedia.org/resource/Czech_Socialist_Republic, http://dbpedia.org/resource/Dakota_Territory, http://dbpedia.org/resource/Dardani, http://dbpedia.org/resource/Demographics_of_India, http://dbpedia.org/resource/Denmark-Norway, http://dbpedia.org/resource/Dissolution_of_Czechoslovakia, http://dbpedia.org/resource/Duchy_of_Aquitaine, http://dbpedia.org/resource/Duchy_of_Burgundy, http://dbpedia.org/resource/Duchy_of_Gascony, http://dbpedia.org/resource/Duchy_of_Nassau, http://dbpedia.org/resource/Duchy_of_Savoy, http://dbpedia.org/resource/Dutch_cuisine, http://dbpedia.org/resource/EC_postcode_area, http://dbpedia.org/resource/Early_modern_France, http://dbpedia.org/resource/East_German, http://dbpedia.org/resource/East_Roman_Empire, http://dbpedia.org/resource/East_Texas, http://dbpedia.org/resource/Egyptian_people, http://dbpedia.org/resource/Emirate_of_Afghanistan, http://dbpedia.org/resource/English_cricket_team, http://dbpedia.org/resource/European_Russia, http://dbpedia.org/resource/Extraterritoriality, http://dbpedia.org/resource/FIBA, http://dbpedia.org/resource/Federal_Territory_(Malaysia), http://dbpedia.org/resource/First_Babylonian_Dynasty, http://dbpedia.org/resource/First_Intermediate_Period_of_Egypt, http://dbpedia.org/resource/Free_City_of_Lübeck, http://dbpedia.org/resource/Free_State_Province, http://dbpedia.org/resource/French-Canadian, http://dbpedia.org/resource/French_Fifth_Republic, http://dbpedia.org/resource/French_immigration_to_Mexico, http://dbpedia.org/resource/French_nationality_law, http://dbpedia.org/resource/Fukuoka, http://dbpedia.org/resource/Fédération_Internationale_de_Volleyball, http://dbpedia.org/resource/Geats, http://dbpedia.org/resource/General_Roman_Calendar, http://dbpedia.org/resource/Gloucestershire, http://dbpedia.org/resource/Gojoseon, http://dbpedia.org/resource/Governorate_of_Livonia, http://dbpedia.org/resource/Heligoland_Bight, http://dbpedia.org/resource/Hesse-Homburg, http://dbpedia.org/resource/History_of_Germany, http://dbpedia.org/resource/International_Wheelchair_Basketball_Federation, http://dbpedia.org/resource/Italy_national_basketball_team, http://dbpedia.org/resource/Jains, http://dbpedia.org/resource/Jund_Filastin, http://dbpedia.org/resource/Kalimantan, http://dbpedia.org/resource/Karamanids, http://dbpedia.org/resource/Kerma_Culture, http://dbpedia.org/resource/Khanate_of_Kalat, http://dbpedia.org/resource/Khazars, http://dbpedia.org/resource/Kingdom_of_Pamplona, http://dbpedia.org/resource/Kingdom_of_the_Burgundians, http://dbpedia.org/resource/Kirghiz_Soviet_Socialist_Republic, http://dbpedia.org/resource/Korean_Peninsula, http://dbpedia.org/resource/Korean_cuisine, http://dbpedia.org/resource/Korean_people, http://dbpedia.org/resource/La_Belle_Alliance, http://dbpedia.org/resource/Liang_dynasty, http://dbpedia.org/resource/List_of_Anglican_Church_calendars, http://dbpedia.org/resource/List_of_serial_killers_by_country, http://dbpedia.org/resource/Lithuania_men's_national_basketball_team, http://dbpedia.org/resource/Lithuanian_Jews, http://dbpedia.org/resource/Liu_Song_dynasty, http://dbpedia.org/resource/Madrid,_Spain, http://dbpedia.org/resource/Mamluk_dynasty_of_Iraq, http://dbpedia.org/resource/Manx_people, http://dbpedia.org/resource/Marlborough_House, http://dbpedia.org/resource/Marsica, http://dbpedia.org/resource/Member_states_of_the_Commonwealth_of_Nations, http://dbpedia.org/resource/Mexican_cuisine, http://dbpedia.org/resource/Moche_culture, http://dbpedia.org/resource/Moghulistan, http://dbpedia.org/resource/Moscow_Kremlin, http://dbpedia.org/resource/Mount_Emei, http://dbpedia.org/resource/Multiple_citizenship, http://dbpedia.org/resource/Nanyue, http://dbpedia.org/resource/Negev, http://dbpedia.org/resource/Netherlands_(terminology), http://dbpedia.org/resource/Netherlands_East_Indies, http://dbpedia.org/resource/New_Britain, http://dbpedia.org/resource/North_West_Province_(South_Africa), http://dbpedia.org/resource/Northamptonshire, http://dbpedia.org/resource/Northeast_China, http://dbpedia.org/resource/Northern_Basque_Country, http://dbpedia.org/resource/Northern_Greece, http://dbpedia.org/resource/Northern_Wei, http://dbpedia.org/resource/Northwestern_United_States, http://dbpedia.org/resource/Nueva_Vizcaya,_New_Spain, http://dbpedia.org/resource/Nuevo_Santander, http://dbpedia.org/resource/Orange_Free_State, http://dbpedia.org/resource/Pakistani_cricket_team, http://dbpedia.org/resource/Palestinian_Christians, http://dbpedia.org/resource/Peerage_of_the_United_Kingdom, http://dbpedia.org/resource/Prince-Bishopric_of_Augsburg, http://dbpedia.org/resource/Principality_of_Achaea, http://dbpedia.org/resource/Province_of_Massachusetts_Bay, http://dbpedia.org/resource/Province_of_Pennsylvania, http://dbpedia.org/resource/Province_of_Quebec_(1763–91), http://dbpedia.org/resource/Pyrénées_Atlantiques, http://dbpedia.org/resource/Republic_of_Pirates, http://dbpedia.org/resource/Roman_Judea, http://dbpedia.org/resource/Russian-American, http://dbpedia.org/resource/Russian_Turkestan, http://dbpedia.org/resource/Ruthenians, http://dbpedia.org/resource/Santa_Maria_del_Mar,_Barcelona, http://dbpedia.org/resource/Scordisci, http://dbpedia.org/resource/Second_Czechoslovak_Republic, http://dbpedia.org/resource/Semirechye_Oblast, http://dbpedia.org/resource/Sened_(town), http://dbpedia.org/resource/Shi'a_Islam, http://dbpedia.org/resource/South_African, http://dbpedia.org/resource/South_African_cricket_team, http://dbpedia.org/resource/South_African_cuisine, http://dbpedia.org/resource/Southern_Qi, http://dbpedia.org/resource/Southern_Song, http://dbpedia.org/resource/Spaniard, http://dbpedia.org/resource/Spanish_State, http://dbpedia.org/resource/Sri_Lankan_cricket_team, http://dbpedia.org/resource/State_of_New_York, http://dbpedia.org/resource/Sudetenland, http://dbpedia.org/resource/Sultanate_of_Sambas, http://dbpedia.org/resource/Taiwanese_cuisine, http://dbpedia.org/resource/Thanjavur_Maratha_kingdom, http://dbpedia.org/resource/Tribe, http://dbpedia.org/resource/Trinidadian, http://dbpedia.org/resource/Upper_Hungary, http://dbpedia.org/resource/Upper_Lusatia, http://dbpedia.org/resource/Upper_Mustang, http://dbpedia.org/resource/Upstate_New_York, http://dbpedia.org/resource/Vallespir, http://dbpedia.org/resource/Varangian, http://dbpedia.org/resource/Victory_in_Europe_Day, http://dbpedia.org/resource/Vilna, http://dbpedia.org/resource/Wageningen, http://dbpedia.org/resource/Warsaw,_Poland, http://dbpedia.org/resource/Washington_state, http://dbpedia.org/resource/Weimar_Germany, http://dbpedia.org/resource/West_Bank_Areas_in_the_Oslo_II_Accord, http://dbpedia.org/resource/West_Kalimantan, http://dbpedia.org/resource/Yakutia, http://dbpedia.org/resource/Yamhad, http://dbpedia.org/resource/Yotvingians, http://dbpedia.org/resource/Adam's_Grave, http://dbpedia.org/resource/Aden_Protectorate, http://dbpedia.org/resource/America, http://dbpedia.org/resource/Anglo-Indian_cuisine, http://dbpedia.org/resource/Anglosphere, http://dbpedia.org/resource/Annam_(French_protectorate), http://dbpedia.org/resource/Annobón, http://dbpedia.org/resource/Archbishopric_of_Riga, http://dbpedia.org/resource/Aztec_Empire, http://dbpedia.org/resource/Baghdad_Vilayet, http://dbpedia.org/resource/Baltic_Germans, http://dbpedia.org/resource/Baltic_provinces, http://dbpedia.org/resource/Barghawata, http://dbpedia.org/resource/Basra_Vilayet, http://dbpedia.org/resource/Beer_in_Canada, http://dbpedia.org/resource/Brandenburger_Gold_Coast, http://dbpedia.org/resource/British_Hong_Kong, http://dbpedia.org/resource/British_Solomon_Islands, http://dbpedia.org/resource/British_Somaliland, http://dbpedia.org/resource/British_Western_Pacific_Territories, http://dbpedia.org/resource/British_Windward_Islands, http://dbpedia.org/resource/Buryatia, http://dbpedia.org/resource/Buyeo_Kingdom, http://dbpedia.org/resource/Canadian_cricket_team, http://dbpedia.org/resource/Canadian_cuisine, http://dbpedia.org/resource/Cap_Bon, http://dbpedia.org/resource/Cape_Bon, http://dbpedia.org/resource/Catalan_Republic, http://dbpedia.org/resource/Chaldia, http://dbpedia.org/resource/Chicago_Metropolitan_Area, http://dbpedia.org/resource/Chuvashia, http://dbpedia.org/resource/Cimmeria_(Conan), http://dbpedia.org/resource/Citizenship_of_the_United_States, http://dbpedia.org/resource/Colombian_cuisine, http://dbpedia.org/resource/Colony_of_New_Zealand, http://dbpedia.org/resource/Confederate_government_of_Missouri, http://dbpedia.org/resource/Constitution_of_the_United_States_of_America, http://dbpedia.org/resource/Cook_Islanders, http://dbpedia.org/resource/Crimean_Regional_Government, http://dbpedia.org/resource/Crow_Nation, http://dbpedia.org/resource/Cuisine_of_Jamaica, http://dbpedia.org/resource/Cuisine_of_Trinidad_and_Tobago, http://dbpedia.org/resource/Curacao, http://dbpedia.org/resource/Danish_Gold_Coast, http://dbpedia.org/resource/Danish_India, http://dbpedia.org/resource/Decline_of_the_Ottoman_Empire, http://dbpedia.org/resource/Delta_state, http://dbpedia.org/resource/Democratic_Republic_of_Georgia, http://dbpedia.org/resource/Demographics_of_Morocco, http://dbpedia.org/resource/Dervish_state, http://dbpedia.org/resource/Diocese_of_Macedonia, http://dbpedia.org/resource/Division_of_Northern_Territory, http://dbpedia.org/resource/Don_Republic, http://dbpedia.org/resource/Drimnagh, http://dbpedia.org/resource/Duchy_of_Courland_and_Semigallia, http://dbpedia.org/resource/Duchy_of_Courland_and_Semigallia_(1918), http://dbpedia.org/resource/Duchy_of_Greater_Poland, http://dbpedia.org/resource/Duchy_of_Masovia, http://dbpedia.org/resource/Duchy_of_Neopatras, http://dbpedia.org/resource/Duchy_of_Pomerania, http://dbpedia.org/resource/Dutch_(ethnic_group), http://dbpedia.org/resource/Dutch_Gold_Coast, http://dbpedia.org/resource/Dutch_Language_Union, http://dbpedia.org/resource/Dutch_New_Guinea, http://dbpedia.org/resource/E_postcode_area, http://dbpedia.org/resource/East_African_cricket_team, http://dbpedia.org/resource/Eastern_Hungarian_Kingdom, http://dbpedia.org/resource/Eastern_Iranian_languages, http://dbpedia.org/resource/Emirate_of_Jabal_Shammar, http://dbpedia.org/resource/England_national_cricket_team, http://dbpedia.org/resource/English_Council_of_State, http://dbpedia.org/resource/Espiritu_Santo, http://dbpedia.org/resource/Falkland_Islands_Dependencies, http://dbpedia.org/resource/Fengjie, http://dbpedia.org/resource/Florida_in_the_American_Civil_War, http://dbpedia.org/resource/Free_Imperial_City_of_Besançon, http://dbpedia.org/resource/French_Cochinchina, http://dbpedia.org/resource/French_Morocco, http://dbpedia.org/resource/French_Protectorate_of_Laos, http://dbpedia.org/resource/French_protectorate_of_Morocco, http://dbpedia.org/resource/French_protectorate_of_Tunisia, http://dbpedia.org/resource/Gaya_confederacy, http://dbpedia.org/resource/German_American, http://dbpedia.org/resource/German_East_Africa, http://dbpedia.org/resource/German_South-West_Africa, http://dbpedia.org/resource/Gilbert_and_Ellice_Islands, http://dbpedia.org/resource/Giudicato_of_Cagliari, http://dbpedia.org/resource/Global_citizenship, http://dbpedia.org/resource/Grand_Duchy_of_Finland, http://dbpedia.org/resource/Great_Lakes_region_(North_America), http://dbpedia.org/resource/Guangzhouwan, http://dbpedia.org/resource/Hejaz_Vilayet, http://dbpedia.org/resource/History_of_British_nationality_law, http://dbpedia.org/resource/Ifni, http://dbpedia.org/resource/Imperial_General_Headquarters, http://dbpedia.org/resource/India_national_cricket_team, http://dbpedia.org/resource/Indian_film, http://dbpedia.org/resource/Indian_national_cricket_team, http://dbpedia.org/resource/Indonesian_cuisine, http://dbpedia.org/resource/International_Association_of_the_Congo, http://dbpedia.org/resource/Intersex, http://dbpedia.org/resource/Ireland_cricket_team, http://dbpedia.org/resource/Irish_citizen, http://dbpedia.org/resource/Israeli_cuisine, http://dbpedia.org/resource/Italian_Libya, http://dbpedia.org/resource/Italian_Somaliland, http://dbpedia.org/resource/Javanese_cuisine, http://dbpedia.org/resource/Jin_(Chinese_state), http://dbpedia.org/resource/Jordanian_cuisine, http://dbpedia.org/resource/Joseon_Dynasty, http://dbpedia.org/resource/Judean_Hills, http://dbpedia.org/resource/Kabardino-Balkaria, http://dbpedia.org/resource/Kenya_national_cricket_team, http://dbpedia.org/resource/Kham, http://dbpedia.org/resource/Khanate_of_Khiva, http://dbpedia.org/resource/Kiautschou_Bay_concession, http://dbpedia.org/resource/Kimberley,_South_Africa, http://dbpedia.org/resource/King_of_Morocco, http://dbpedia.org/resource/Kingdom_of_Finland_(1918), http://dbpedia.org/resource/Kingdom_of_Lithuania, http://dbpedia.org/resource/Kingdom_of_Lithuania_(1918), http://dbpedia.org/resource/Kingdom_of_Rarotonga, http://dbpedia.org/resource/Komi_Republic, http://dbpedia.org/resource/Korean_peninsula, http://dbpedia.org/resource/Kovno_Governorate, http://dbpedia.org/resource/Kuban_People's_Republic, http://dbpedia.org/resource/Kunashir_Island, http://dbpedia.org/resource/Kwantung_Leased_Territory, http://dbpedia.org/resource/Later_Yan, http://dbpedia.org/resource/Leopardstown, http://dbpedia.org/resource/List_of_Canadians, http://dbpedia.org/resource/Louisiana_in_the_American_Civil_War, http://dbpedia.org/resource/Malta_Colony, http://dbpedia.org/resource/Manx_cat, http://dbpedia.org/resource/Mari_El, http://dbpedia.org/resource/Mariana_Islands, http://dbpedia.org/resource/Mauretania_Caesariensis, http://dbpedia.org/resource/Member_states_of_the_European_Union, http://dbpedia.org/resource/Military_history_of_the_United_States_during_World_War_II, http://dbpedia.org/resource/Minyue, http://dbpedia.org/resource/Mississippi_in_the_American_Civil_War, http://dbpedia.org/resource/Mongolian_cuisine, http://dbpedia.org/resource/Mount_Lebanon_Mutasarrifate, http://dbpedia.org/resource/Mutasarrifate_of_Jerusalem, http://dbpedia.org/resource/Nanking, http://dbpedia.org/resource/Nassau_(Cook_Islands), http://dbpedia.org/resource/Natal,_South_Africa, http://dbpedia.org/resource/Nationality, http://dbpedia.org/resource/Neopagan, http://dbpedia.org/resource/Neverland, http://dbpedia.org/resource/New_Westminster,_British_Columbia, http://dbpedia.org/resource/New_York_City_Fire_Department, http://dbpedia.org/resource/New_Zealand_national_cricket_team, http://dbpedia.org/resource/New_Zealand_nationality_law, http://dbpedia.org/resource/Non-citizens_(Latvia), http://dbpedia.org/resource/Northern_Rhodesia, http://dbpedia.org/resource/Nyasaland, http://dbpedia.org/resource/Odawa_people, http://dbpedia.org/resource/Pakistan_national_cricket_team, http://dbpedia.org/resource/Palestinian_cuisine, http://dbpedia.org/resource/Panama_Canal_Zone, http://dbpedia.org/resource/Pannonia_Inferior, http://dbpedia.org/resource/Penrhyn_Island, http://dbpedia.org/resource/Peruvian, http://dbpedia.org/resource/Philippine_nationality_law, http://dbpedia.org/resource/Police_area, http://dbpedia.org/resource/Portuguese-speaking_African_countries, http://dbpedia.org/resource/Portuguese_Cape_Verde, http://dbpedia.org/resource/Portuguese_Gold_Coast, http://dbpedia.org/resource/Portuguese_São_Tomé_and_Príncipe, http://dbpedia.org/resource/Praeneste, http://dbpedia.org/resource/Prince-Bishopric_of_Warmia, http://dbpedia.org/resource/Principality_of_Transylvania_(1570–1711), http://dbpedia.org/resource/Province_of_Silesia, http://dbpedia.org/resource/Republic_of_Communist_China, http://dbpedia.org/resource/Republic_of_South_Africa, http://dbpedia.org/resource/Roman_Syria, http://dbpedia.org/resource/SE_postcode_area, http://dbpedia.org/resource/SW_postcode_area, http://dbpedia.org/resource/Sandomierz_Voivodeship, http://dbpedia.org/resource/Saybrook_Colony, http://dbpedia.org/resource/Second_Hellenic_Republic, http://dbpedia.org/resource/Shanghai_French_Concession, http://dbpedia.org/resource/Shanghai_International_Settlement, http://dbpedia.org/resource/Sichuan_Province, http://dbpedia.org/resource/Somali_Republic, http://dbpedia.org/resource/Spanish_Formosa, http://dbpedia.org/resource/Spanish_Guinea, http://dbpedia.org/resource/Spanish_Main, http://dbpedia.org/resource/Spanish_West_Indies, http://dbpedia.org/resource/Sri_Lanka_national_cricket_team, http://dbpedia.org/resource/State_of_Indiana, http://dbpedia.org/resource/State_of_Lu, http://dbpedia.org/resource/State_of_the_Teutonic_Order, http://dbpedia.org/resource/Sui_generis, http://dbpedia.org/resource/Sultanate_of_Darfur, http://dbpedia.org/resource/Sultanate_of_Sulu, http://dbpedia.org/resource/Swedish_Gold_Coast, http://dbpedia.org/resource/Swedish_overseas_colonies, http://dbpedia.org/resource/Syria_Vilayet, http://dbpedia.org/resource/Taipei_County, http://dbpedia.org/resource/Taiwan_under_Japanese_rule, http://dbpedia.org/resource/Tatar, http://dbpedia.org/resource/Television_in_the_United_States, http://dbpedia.org/resource/Territory_of_Papua, http://dbpedia.org/resource/Tetbury, http://dbpedia.org/resource/The_Protectorate, http://dbpedia.org/resource/Tibetan_Empire, http://dbpedia.org/resource/Togoland, http://dbpedia.org/resource/Tonkin_(French_protectorate), http://dbpedia.org/resource/Trucial_States, http://dbpedia.org/resource/Trust_Territory_of_Somaliland, http://dbpedia.org/resource/Tuva, http://dbpedia.org/resource/US, http://dbpedia.org/resource/Udmurtia, http://dbpedia.org/resource/Ukrainian_State, http://dbpedia.org/resource/Ulaid, http://dbpedia.org/resource/Union_of_Sovereign_States, http://dbpedia.org/resource/Uyghur_Khaganate, http://dbpedia.org/resource/Valparaiso, http://dbpedia.org/resource/Vandal_Kingdom, http://dbpedia.org/resource/Venice,_Italy, http://dbpedia.org/resource/Vinland, http://dbpedia.org/resource/Voronezh_Governorate, http://dbpedia.org/resource/Walt_Disney_World_Resort, http://dbpedia.org/resource/West_Indies_Cricket_Board, http://dbpedia.org/resource/West_Indies_national_cricket_team, http://dbpedia.org/resource/Yangzhou_(ancient_China), http://dbpedia.org/resource/Yemen_Vilayet, http://dbpedia.org/resource/Zhili_Clique, http://dbpedia.org/resource/Zimbabwe_national_cricket_team, http://dbpedia.org/resource/Aceh_Sultanate, http://dbpedia.org/resource/Afghan_cuisine, http://dbpedia.org/resource/Alawite_State, http://dbpedia.org/resource/Albanian_people, http://dbpedia.org/resource/Alodia, http://dbpedia.org/resource/Arab_Federation, http://dbpedia.org/resource/Arab_Republic_of_Egypt, http://dbpedia.org/resource/Arabian, http://dbpedia.org/resource/Article_One_of_the_United_States_Constitution, http://dbpedia.org/resource/Austro-Hungary, http://dbpedia.org/resource/Ava_Kingdom, http://dbpedia.org/resource/Baltic_Assembly, http://dbpedia.org/resource/Bamana_Empire, http://dbpedia.org/resource/Barotseland, http://dbpedia.org/resource/Bosnia-Herzegovina, http://dbpedia.org/resource/Britanny, http://dbpedia.org/resource/British_National_(Overseas), http://dbpedia.org/resource/British_television, http://dbpedia.org/resource/Bunyoro, http://dbpedia.org/resource/Carinthian_Slovenes, http://dbpedia.org/resource/Central_Belt, http://dbpedia.org/resource/Cherokee_Nation, http://dbpedia.org/resource/Chola_Dynasty, http://dbpedia.org/resource/Chūzan, http://dbpedia.org/resource/Cisalpine_Republic, http://dbpedia.org/resource/Cispadane_Republic, http://dbpedia.org/resource/Colony_of_Aden, http://dbpedia.org/resource/Common_Market_for_Eastern_and_Southern_Africa, http://dbpedia.org/resource/Common_Monetary_Area, http://dbpedia.org/resource/Commune_of_Rome, http://dbpedia.org/resource/Confederation_of_the_Rhine, http://dbpedia.org/resource/Cook_Islands_Federation, http://dbpedia.org/resource/Crimean_Autonomous_Soviet_Socialist_Republic, http://dbpedia.org/resource/Darul_Islam_(Indonesia), http://dbpedia.org/resource/Democratic_Republic_of_Yemen, http://dbpedia.org/resource/Demographics_of_Mozambique, http://dbpedia.org/resource/Dendi_Kingdom, http://dbpedia.org/resource/Des_Moines_metropolitan_area, http://dbpedia.org/resource/Dominion_of_Fiji, http://dbpedia.org/resource/Duchy_of_Lucca, http://dbpedia.org/resource/Duchy_of_Massa_and_Carrara, http://dbpedia.org/resource/Duchy_of_Mecklenburg-Strelitz, http://dbpedia.org/resource/Duchy_of_Parma, http://dbpedia.org/resource/East_Frisia, http://dbpedia.org/resource/East_Jersey, http://dbpedia.org/resource/Eastern_Orthodox_liturgical_calendar, http://dbpedia.org/resource/Economy_of_Germany, http://dbpedia.org/resource/Education_in_the_United_States, http://dbpedia.org/resource/Egyptian_nationality_law, http://dbpedia.org/resource/Electorate_of_Hesse, http://dbpedia.org/resource/Emirate_of_Diriyah, http://dbpedia.org/resource/Emirate_of_Nejd, http://dbpedia.org/resource/Emirate_of_Nejd_and_Hasa, http://dbpedia.org/resource/Emirate_of_Sicily, http://dbpedia.org/resource/Empire_of_Great_Fulo, http://dbpedia.org/resource/English_national_identity, http://dbpedia.org/resource/Englishmen, http://dbpedia.org/resource/Enlightenment_in_Spain, http://dbpedia.org/resource/European_Economic_Area, http://dbpedia.org/resource/Euskal_Herria, http://dbpedia.org/resource/Far_Eastern_Republic, http://dbpedia.org/resource/Federal_Government_of_Somalia, http://dbpedia.org/resource/Federated_Malay_States, http://dbpedia.org/resource/Federation_of_Arab_Emirates_of_the_South, http://dbpedia.org/resource/Federation_of_South_Arabia, http://dbpedia.org/resource/Finnish_Democratic_Republic, http://dbpedia.org/resource/Finnish_Socialist_Workers'_Republic, http://dbpedia.org/resource/First_Hellenic_Republic, http://dbpedia.org/resource/First_Hungarian_Republic, http://dbpedia.org/resource/Five_Eyes, http://dbpedia.org/resource/Franceville,_New_Hebrides, http://dbpedia.org/resource/Free_City_of_Kraków, http://dbpedia.org/resource/Free_State_of_Fiume, http://dbpedia.org/resource/Gagauzia, http://dbpedia.org/resource/Galician_Soviet_Socialist_Republic, http://dbpedia.org/resource/German_Samoa, http://dbpedia.org/resource/Government_of_New_Zealand, http://dbpedia.org/resource/Governorate_of_Estonia, http://dbpedia.org/resource/Gozo_(independent_state), http://dbpedia.org/resource/Grand_Duchy_of_Mecklenburg-Schwerin, http://dbpedia.org/resource/Grand_Duchy_of_Mecklenburg-Strelitz, http://dbpedia.org/resource/Grand_Principality_of_Serbia, http://dbpedia.org/resource/Great_Britain_Davis_Cup_team, http://dbpedia.org/resource/Greater_China, http://dbpedia.org/resource/Greater_Republic_of_Central_America, http://dbpedia.org/resource/Habesh_Eyalet, http://dbpedia.org/resource/Himyarite_Kingdom, http://dbpedia.org/resource/Hohenzollern-Hechingen, http://dbpedia.org/resource/Hohenzollern-Sigmaringen, http://dbpedia.org/resource/Huaorani, http://dbpedia.org/resource/Hungarian_minority_in_Romania, http://dbpedia.org/resource/Idrisid_Emirate_of_Asir, http://dbpedia.org/resource/Imamate_of_Futa_Jallon, http://dbpedia.org/resource/Imamate_of_Futa_Toro, http://dbpedia.org/resource/International_Authority_for_the_Ruhr, http://dbpedia.org/resource/Irian_Jaya, http://dbpedia.org/resource/Irish_Australian, http://dbpedia.org/resource/Islands_of_Refreshment, http://dbpedia.org/resource/Italian_Empire, http://dbpedia.org/resource/Italian_Islands_of_the_Aegean, http://dbpedia.org/resource/Italian_Regency_of_Carnaro, http://dbpedia.org/resource/Italian_Republic_(Napoleonic), http://dbpedia.org/resource/Jolof_Empire, http://dbpedia.org/resource/Kaabu, http://dbpedia.org/resource/Kazakh_Khanate, http://dbpedia.org/resource/Khmers, http://dbpedia.org/resource/Kingdom_of_Aksum, http://dbpedia.org/resource/Kingdom_of_Bora_Bora, http://dbpedia.org/resource/Kingdom_of_Brittany, http://dbpedia.org/resource/Kingdom_of_Burundi, http://dbpedia.org/resource/Kingdom_of_Butua, http://dbpedia.org/resource/Kingdom_of_Corsica_(1736), http://dbpedia.org/resource/Kingdom_of_Hejaz, http://dbpedia.org/resource/Kingdom_of_Kurdistan, http://dbpedia.org/resource/Kingdom_of_Mutapa, http://dbpedia.org/resource/Kingdom_of_Nejd_and_Hejaz, http://dbpedia.org/resource/Kingdom_of_Pajang, http://dbpedia.org/resource/Kingdom_of_Rwanda, http://dbpedia.org/resource/Kingdom_of_Tahiti, http://dbpedia.org/resource/Konbaung_Dynasty, http://dbpedia.org/resource/Land_of_Ev, http://dbpedia.org/resource/Landgraviate_of_Hesse-Darmstadt, http://dbpedia.org/resource/Lanfang_Republic, http://dbpedia.org/resource/List_of_countries_in_the_Eurovision_Song_Contest, http://dbpedia.org/resource/Longueuil,_Quebec, http://dbpedia.org/resource/Mahdist_Sudan, http://dbpedia.org/resource/Majeerteen_Sultanate, http://dbpedia.org/resource/Mali_Federation, http://dbpedia.org/resource/Marstrand_Free_Port, http://dbpedia.org/resource/Mataram_Sultanate, http://dbpedia.org/resource/Medieval_Iceland, http://dbpedia.org/resource/Merina_Kingdom, http://dbpedia.org/resource/Middle_Saxons, http://dbpedia.org/resource/Miranda_do_Douro, http://dbpedia.org/resource/Mongolian_People's_Republic, http://dbpedia.org/resource/Morona-Santiago, http://dbpedia.org/resource/Mthwakazi, http://dbpedia.org/resource/Mutawakkilite_Kingdom_of_Yemen, http://dbpedia.org/resource/Netherlands_New_Guinea, http://dbpedia.org/resource/New_Zealanders, http://dbpedia.org/resource/Nicaraguan, http://dbpedia.org/resource/Nigerians, http://dbpedia.org/resource/Nogai_Horde, http://dbpedia.org/resource/North_Cape,_Norway, http://dbpedia.org/resource/North_Sea_Empire, http://dbpedia.org/resource/North_of_England, http://dbpedia.org/resource/Ojibwa, http://dbpedia.org/resource/Orange_River_Sovereignty, http://dbpedia.org/resource/Pacific_Union, http://dbpedia.org/resource/People's_Democratic_Republic_of_Ethiopia, http://dbpedia.org/resource/Philippians, http://dbpedia.org/resource/Portland_metropolitan_area, http://dbpedia.org/resource/Portuguese_East_Africa, http://dbpedia.org/resource/Principality_of_Erfurt, http://dbpedia.org/resource/Principality_of_Lippe, http://dbpedia.org/resource/Principality_of_Serbia_(medieval), http://dbpedia.org/resource/Principality_of_Transylvania_(1571–1711), http://dbpedia.org/resource/Principality_of_Trinidad, http://dbpedia.org/resource/Principality_of_Zeta, http://dbpedia.org/resource/Province_of_Saxony, http://dbpedia.org/resource/Provisional_Government_of_Hawaii, http://dbpedia.org/resource/Provisional_Revolutionary_Government_of_the_Republic_of_South_Vietnam, http://dbpedia.org/resource/Puget_Sound_region, http://dbpedia.org/resource/Rapa_Nui, http://dbpedia.org/resource/Regency_of_Algiers, http://dbpedia.org/resource/Republic_of_Acre, http://dbpedia.org/resource/Republic_of_Central_Lithuania, http://dbpedia.org/resource/Republic_of_Cospaia, http://dbpedia.org/resource/Republic_of_Ezo, http://dbpedia.org/resource/Republic_of_Formosa, http://dbpedia.org/resource/Republic_of_Hungary_(1946–1949), http://dbpedia.org/resource/Republic_of_Indian_Stream, http://dbpedia.org/resource/Republic_of_Kuwait, http://dbpedia.org/resource/Republic_of_Maryland, http://dbpedia.org/resource/Republic_of_Mountainous_Armenia, http://dbpedia.org/resource/Republic_of_North_Peru, http://dbpedia.org/resource/Republic_of_Prekmurje, http://dbpedia.org/resource/Republic_of_Rose_Island, http://dbpedia.org/resource/Republic_of_Sonora, http://dbpedia.org/resource/Republic_of_Vietnam, http://dbpedia.org/resource/Republic_of_Yucatán, http://dbpedia.org/resource/Republic_of_the_Rio_Grande, http://dbpedia.org/resource/Revolutionary_Government_Junta_of_El_Salvador, http://dbpedia.org/resource/Rif_Republic, http://dbpedia.org/resource/Romans, http://dbpedia.org/resource/Royal_Prussia, http://dbpedia.org/resource/Russian_Republic, http://dbpedia.org/resource/Ryukyu_Kingdom, http://dbpedia.org/resource/SADR, http://dbpedia.org/resource/Salento, http://dbpedia.org/resource/Saudeleur_Dynasty, http://dbpedia.org/resource/Saxe-Altenburg, http://dbpedia.org/resource/Saxe-Coburg-Saalfeld, http://dbpedia.org/resource/Saxe-Gotha, http://dbpedia.org/resource/Saxe-Hildburghausen, http://dbpedia.org/resource/Saxe-Meiningen, http://dbpedia.org/resource/Saxe-Weimar, http://dbpedia.org/resource/Saxe-Weimar-Eisenach, http://dbpedia.org/resource/Schwarzburg-Rudolstadt, http://dbpedia.org/resource/Schwarzburg-Sondershausen, http://dbpedia.org/resource/Scottish_cricket_team, http://dbpedia.org/resource/Sennar_(sultanate), http://dbpedia.org/resource/Serbian_Despotate, http://dbpedia.org/resource/Slovak_Soviet_Republic, http://dbpedia.org/resource/Slovak_people, http://dbpedia.org/resource/Socialist_Republic_of_Vietnam, http://dbpedia.org/resource/South-West_Africa, http://dbpedia.org/resource/South_Atlantic, http://dbpedia.org/resource/Southern_Jutland, http://dbpedia.org/resource/Southwell,_Nottinghamshire, http://dbpedia.org/resource/Soviet_Republic_of_Naissaar, http://dbpedia.org/resource/State_of_Aleppo, http://dbpedia.org/resource/State_of_Damascus, http://dbpedia.org/resource/State_of_Muskogee, http://dbpedia.org/resource/Sultanate_of_Cirebon, http://dbpedia.org/resource/Sultanate_of_Hobyo, http://dbpedia.org/resource/Sultanate_of_Maguindanao, http://dbpedia.org/resource/Sultanate_of_Nejd, http://dbpedia.org/resource/Surinam_(Dutch_colony), http://dbpedia.org/resource/São_Tomé, http://dbpedia.org/resource/Tanganyika_Territory, http://dbpedia.org/resource/Taungoo_Dynasty, http://dbpedia.org/resource/Telecorporación_Salvadoreña, http://dbpedia.org/resource/Terra_nullius, http://dbpedia.org/resource/Tibet_(1912–51), http://dbpedia.org/resource/Tidewater_region_of_Virginia, http://dbpedia.org/resource/Toucouleur_Empire, http://dbpedia.org/resource/Transkei, http://dbpedia.org/resource/Transpadane_Republic, http://dbpedia.org/resource/Travancore, http://dbpedia.org/resource/Tuʻi_Tonga_Empire, http://dbpedia.org/resource/United_Arab_Republic, http://dbpedia.org/resource/United_Belgian_States, http://dbpedia.org/resource/United_Colonies_of_Vancouver_Island_and_British_Columbia, http://dbpedia.org/resource/United_Kingdom_in_the_Eurovision_Song_Contest, http://dbpedia.org/resource/United_Mexican_States, http://dbpedia.org/resource/United_Principalities, http://dbpedia.org/resource/United_Suvadive_Republic, http://dbpedia.org/resource/Upper_Nile_State, http://dbpedia.org/resource/Việt_Nam, http://dbpedia.org/resource/Volga_Bulgaria, http://dbpedia.org/resource/Warsangali_Sultanate, http://dbpedia.org/resource/Wassoulou_Empire, http://dbpedia.org/resource/Wattasid_dynasty, http://dbpedia.org/resource/West_Jersey, http://dbpedia.org/resource/West_Prussia, http://dbpedia.org/resource/Western_Balkans, http://dbpedia.org/resource/Wikt:Anglo-French, http://dbpedia.org/resource/Württemberg, http://dbpedia.org/resource/Yeke_Kingdom, http://dbpedia.org/resource/Yemen_Arab_Republic, http://dbpedia.org/resource/Zimbabwe_Rhodesia, http://dbpedia.org/resource/Argentine_Federal_Police, http://dbpedia.org/resource/Avatele, http://dbpedia.org/resource/Bangsamoro_Republik, http://dbpedia.org/resource/Bavaria-Ingolstadt, http://dbpedia.org/resource/Bishopric_of_Merseburg, http://dbpedia.org/resource/British_Arctic_Territories, http://dbpedia.org/resource/Buyeo, http://dbpedia.org/resource/Central_Mexico, http://dbpedia.org/resource/Chernigov_Governorate, http://dbpedia.org/resource/Colville_tribe, http://dbpedia.org/resource/Commonwealth_of_Dominica, http://dbpedia.org/resource/Constitution_of_Namibia, http://dbpedia.org/resource/Curaçao_and_Dependencies, http://dbpedia.org/resource/Dedemsvaart, http://dbpedia.org/resource/Dependencies_of_Norway, http://dbpedia.org/resource/Dutch_nationality_law, http://dbpedia.org/resource/Freetown,_Sierra_Leone, http://dbpedia.org/resource/Government_of_South_Korea, http://dbpedia.org/resource/Gurung, http://dbpedia.org/resource/Hersfeld_Abbey, http://dbpedia.org/resource/History_of_the_Khitans, http://dbpedia.org/resource/Inuk, http://dbpedia.org/resource/Jinsha_(archaeological_site), http://dbpedia.org/resource/Khumbu, http://dbpedia.org/resource/Korean_ethnic_nationalism, http://dbpedia.org/resource/List_of_American_artists, http://dbpedia.org/resource/List_of_Canadians_of_English_descent, http://dbpedia.org/resource/List_of_Roman_Catholic_dioceses_(structured_view), http://dbpedia.org/resource/Lordship_of_Overijssel, http://dbpedia.org/resource/Lordship_of_Utrecht, http://dbpedia.org/resource/Majuro,_Marshall_Islands, http://dbpedia.org/resource/Namibia_national_cricket_team, http://dbpedia.org/resource/New_York_City_metropolitan_area, http://dbpedia.org/resource/Northern_Limit_Line, http://dbpedia.org/resource/Operational_Zone_of_the_Adriatic_Littoral, http://dbpedia.org/resource/Operational_Zone_of_the_Alpine_Foothills, http://dbpedia.org/resource/Ovamboland, http://dbpedia.org/resource/Pozsony_County, http://dbpedia.org/resource/Prince-Bishopric_of_Montenegro, http://dbpedia.org/resource/Provisional_People's_Committee_for_North_Korea, http://dbpedia.org/resource/Rai_dynasty, http://dbpedia.org/resource/Rajahnate_of_Butuan, http://dbpedia.org/resource/Reichskommissariat_Kaukasus, http://dbpedia.org/resource/Reichskommissariat_Moskowien, http://dbpedia.org/resource/Reichskommissariat_Niederlande, http://dbpedia.org/resource/Reichskommissariat_Turkestan, http://dbpedia.org/resource/Republic_of_Bulgaria, http://dbpedia.org/resource/Right_of_abode_issue,_Hong_Kong, http://dbpedia.org/resource/SSS_islands, http://dbpedia.org/resource/Serbians, http://dbpedia.org/resource/Soviet_Civil_Administration, http://dbpedia.org/resource/Subanon_people, http://dbpedia.org/resource/Suriname_(Kingdom_of_the_Netherlands), http://dbpedia.org/resource/Syrian_cuisine, http://dbpedia.org/resource/Tamang, http://dbpedia.org/resource/Taupo_Volcanic_Zone, http://dbpedia.org/resource/Tewa_people, http://dbpedia.org/resource/Thakali_people, http://dbpedia.org/resource/The_United_States, http://dbpedia.org/resource/Uruzgan, http://dbpedia.org/resource/Uzbek_Soviet_Socialist_Republic, http://dbpedia.org/resource/Visayas, http://dbpedia.org/resource/Washington,_D.C._metropolitan_area, http://dbpedia.org/resource/West_bank, http://dbpedia.org/resource/Western_Pomerania, http://dbpedia.org/resource/Western_Satraps, http://dbpedia.org/resource/Whatcom_County, http://dbpedia.org/resource/Yadav, http://dbpedia.org/resource/Yakama_Nation, http://dbpedia.org/resource/Zamboanga_Peninsula_(geographical_region), http://dbpedia.org/resource/Baghdad,_Iraq, http://dbpedia.org/resource/Basilica_of_Saint_John_Lateran, http://dbpedia.org/resource/Beer_in_Ireland, http://dbpedia.org/resource/Bishopric_of_Halberstadt, http://dbpedia.org/resource/Bishopric_of_Metz, http://dbpedia.org/resource/Bishopric_of_Minden, http://dbpedia.org/resource/Bishopric_of_Verdun, http://dbpedia.org/resource/Catepanate_of_Italy, http://dbpedia.org/resource/Congo-Brazzaville, http://dbpedia.org/resource/County_of_Kladsko, http://dbpedia.org/resource/County_of_Savoy, http://dbpedia.org/resource/Cuisine_of_Thailand, http://dbpedia.org/resource/Duchy_of_Florence, http://dbpedia.org/resource/Eastern_Orthodox_Christianity, http://dbpedia.org/resource/Engelberg_Abbey, http://dbpedia.org/resource/Hungarian-American, http://dbpedia.org/resource/Hunza_(princely_state), http://dbpedia.org/resource/Lithuanian-American, http://dbpedia.org/resource/Nabataean_Kingdom, http://dbpedia.org/resource/Norman_England, http://dbpedia.org/resource/PE_postcode_area, http://dbpedia.org/resource/Palatinate-Neuburg, http://dbpedia.org/resource/Palistin, http://dbpedia.org/resource/Posen-West_Prussia, http://dbpedia.org/resource/Principality_of_Piombino, http://dbpedia.org/resource/United_Duchies_of_Jülich-Cleves-Berg, http://dbpedia.org/resource/Vatican_Basilica, http://dbpedia.org/resource/Zanskar, http://dbpedia.org/resource/Benzingerode, http://dbpedia.org/resource/Bishopric_of_Trent, http://dbpedia.org/resource/Canada_men's_national_junior_ice_hockey_team, http://dbpedia.org/resource/Cinema_of_South_Africa, http://dbpedia.org/resource/County_of_Wernigerode, http://dbpedia.org/resource/Danish_ethnic_group, http://dbpedia.org/resource/Darug_people, http://dbpedia.org/resource/Duchy_of_Guastalla, http://dbpedia.org/resource/Duchy_of_Modena_and_Reggio, http://dbpedia.org/resource/English_(people), http://dbpedia.org/resource/Gévaudan, http://dbpedia.org/resource/Hungarian_Republic_(1919–20), http://dbpedia.org/resource/Hyolmo_people, http://dbpedia.org/resource/Ieper, http://dbpedia.org/resource/Koch_dynasty, http://dbpedia.org/resource/Kyrgyz_Republic, http://dbpedia.org/resource/Malayalis, http://dbpedia.org/resource/Marquisate_of_Finale, http://dbpedia.org/resource/New_Zealand_Cricket, http://dbpedia.org/resource/Polish_National_Government_(January_Uprising), http://dbpedia.org/resource/Pottawatomie, http://dbpedia.org/resource/San_Luis_Obispo, http://dbpedia.org/resource/Sandwich,_Ontario, http://dbpedia.org/resource/Sauerland, http://dbpedia.org/resource/Stolberg-Wernigerode, http://dbpedia.org/resource/Te_Āti_Awa, http://dbpedia.org/resource/United_States_Ambassador_to_Colombia, http://dbpedia.org/resource/United_States_Shipping_Board, http://dbpedia.org/resource/Viipuri_Province, http://dbpedia.org/resource/Wallis_(island), http://dbpedia.org/resource/Worms-Pfeddersheim, http://dbpedia.org/resource/Yotvingia, http://dbpedia.org/resource/Armilla,_Granada, http://dbpedia.org/resource/Bishopric_of_Speyer, http://dbpedia.org/resource/Counts_of_Toggenburg, http://dbpedia.org/resource/County_of_Horne, http://dbpedia.org/resource/County_of_Veldenz, http://dbpedia.org/resource/Despotate_of_Arta, http://dbpedia.org/resource/Deva_dynasty, http://dbpedia.org/resource/Duchy_of_Bouillon, http://dbpedia.org/resource/El_Aaiun, http://dbpedia.org/resource/Free_Imperial_City_of_Aachen, http://dbpedia.org/resource/German_Empire_(1848/1849), http://dbpedia.org/resource/Grand_Duchy_of_Berg, http://dbpedia.org/resource/Greater_Hesse, http://dbpedia.org/resource/Hesse-Marburg, http://dbpedia.org/resource/History_of_Burma, http://dbpedia.org/resource/History_of_Speyer, http://dbpedia.org/resource/Ionian_Islands_under_Venetian_rule, http://dbpedia.org/resource/Kadamba_dynasty, http://dbpedia.org/resource/Kagawa_prefecture, http://dbpedia.org/resource/Kalabhra_dynasty, http://dbpedia.org/resource/Kingdom_of_France_(1791–92), http://dbpedia.org/resource/Kingdom_of_Gampola, http://dbpedia.org/resource/Kingdom_of_Upatissa_Nuwara, http://dbpedia.org/resource/Maitraka, http://dbpedia.org/resource/Mamluk_Dynasty_(Delhi), http://dbpedia.org/resource/Maukhari, http://dbpedia.org/resource/Nagas_of_Padmavati, http://dbpedia.org/resource/Nayakas_of_Keladi, http://dbpedia.org/resource/Pale_of_Calais, http://dbpedia.org/resource/Paramara_dynasty, http://dbpedia.org/resource/Principality_of_Regensburg, http://dbpedia.org/resource/Ramnad_estate, http://dbpedia.org/resource/Rascians, http://dbpedia.org/resource/Starhemberg, http://dbpedia.org/resource/State_of_Aden, http://dbpedia.org/resource/Sutiya_Kingdom, http://dbpedia.org/resource/Taifa_of_Granada, http://dbpedia.org/resource/Thanjavur_Nayak_kingdom, http://dbpedia.org/resource/The_People's_Republic_of_China, http://dbpedia.org/resource/Torres_Strait_Islands, http://dbpedia.org/resource/Vishnukundina, http://dbpedia.org/resource/Weimer_Republic, http://dbpedia.org/resource/Amazon_Region_of_Colombia, http://dbpedia.org/resource/Bhopal_State, http://dbpedia.org/resource/Chandela, http://dbpedia.org/resource/Chhattisgarh_Division, http://dbpedia.org/resource/Duchy_of_Saxe-Wittenberg, http://dbpedia.org/resource/Jin_(Korean_state), http://dbpedia.org/resource/New_South_Wales,_Australia, http://dbpedia.org/resource/Palestinian_Authority_passport, http://dbpedia.org/resource/Powys_Wenwynwyn, http://dbpedia.org/resource/Principality_of_Brunswick-Wolfenbüttel, http://dbpedia.org/resource/Province_of_Québec, http://dbpedia.org/resource/Salm_(state), http://dbpedia.org/resource/Sapwuahfik, http://dbpedia.org/resource/Scheduled_Tribes, http://dbpedia.org/resource/Taebong, http://dbpedia.org/resource/Weingarten_Abbey, http://dbpedia.org/resource/Acre_Sanjak, http://dbpedia.org/resource/Argentinian_people, http://dbpedia.org/resource/Auckland_Province, http://dbpedia.org/resource/Cinema_of_Serbia, http://dbpedia.org/resource/Cuisine_of_Malaysia, http://dbpedia.org/resource/Czech_republic, http://dbpedia.org/resource/Danube_Banovina, http://dbpedia.org/resource/Eastern_Xia, http://dbpedia.org/resource/Eiderstedt, http://dbpedia.org/resource/Federation_of_Nigeria, http://dbpedia.org/resource/Free_and_Independent_State_of_Cundinamarca, http://dbpedia.org/resource/Governorate_of_New_Toledo, http://dbpedia.org/resource/Guyana_(Commonwealth_realm), http://dbpedia.org/resource/Interim_Government_of_Somalia, http://dbpedia.org/resource/Kimberley_region_of_Western_Australia, http://dbpedia.org/resource/Køge_Bay, http://dbpedia.org/resource/Malawi_(Commonwealth_realm), http://dbpedia.org/resource/Mauritius_(Commonwealth_realm), http://dbpedia.org/resource/People's_Revolutionary_Government_(Grenada), http://dbpedia.org/resource/Principality_of_Mingrelia, http://dbpedia.org/resource/Queen_of_New_Zealand, http://dbpedia.org/resource/Realm_of_Stefan_Dragutin, http://dbpedia.org/resource/Rosebud_Sioux_Tribe, http://dbpedia.org/resource/Sans_Arc, http://dbpedia.org/resource/Santa_Rosa_Island,_California, http://dbpedia.org/resource/Sayn-Wittgenstein-Berleburg, http://dbpedia.org/resource/Sayn-Wittgenstein-Hohenstein, http://dbpedia.org/resource/Sierra_Leone_(Commonwealth_realm), http://dbpedia.org/resource/Tainan_County, http://dbpedia.org/resource/The_Gambia_(Commonwealth_realm), http://dbpedia.org/resource/Third_Republic_of_Venezuela, http://dbpedia.org/resource/Trinidad_and_Tobago_(Commonwealth_realm), http://dbpedia.org/resource/Valle_di_Susa, http://dbpedia.org/resource/Venezuela_Province, http://dbpedia.org/resource/Xinjing, http://dbpedia.org/resource/British_Formula_Renault_Championship, http://dbpedia.org/resource/El_Seibo_Province, http://dbpedia.org/resource/European_Rental_Association, http://dbpedia.org/resource/Glassport,_Pennsylvania, http://dbpedia.org/resource/Matmâta, http://dbpedia.org/resource/Northern_Syria, http://dbpedia.org/resource/Rugiland, http://dbpedia.org/resource/San_José_de_Ocoa_Province, http://dbpedia.org/resource/Television_in_the_Soviet_Union, http://dbpedia.org/resource/United_States_men's_national_volleyball_team, http://dbpedia.org/resource/Anglo-Scottish_Border, http://dbpedia.org/resource/Armenians_in_Abkhazia, http://dbpedia.org/resource/Cassel,_France, http://dbpedia.org/resource/Flint/Tri-Cities, http://dbpedia.org/resource/Hui'an, http://dbpedia.org/resource/Indian_British, http://dbpedia.org/resource/Islamic_denominations, http://dbpedia.org/resource/Pakistani_British, http://dbpedia.org/resource/Qiong_Wen, http://dbpedia.org/resource/Syrian_Government, http://dbpedia.org/resource/Wikt:British, http://dbpedia.org/resource/Älvsborg_County, http://dbpedia.org/resource/Bolivian_American, http://dbpedia.org/resource/Cangnan, http://dbpedia.org/resource/Former_Zhao, http://dbpedia.org/resource/Hauraki_Plains, http://dbpedia.org/resource/Ngāti_Kahungunu, http://dbpedia.org/resource/Old_Peru,_Iowa, http://dbpedia.org/resource/Swiss-American, http://dbpedia.org/resource/Argentine_American, http://dbpedia.org/resource/Bendery, http://dbpedia.org/resource/CHINA, http://dbpedia.org/resource/Carriacou_and_Petite_Martinique, http://dbpedia.org/resource/County_of_Tecklenburg, http://dbpedia.org/resource/Dena'ina_people, http://dbpedia.org/resource/España, http://dbpedia.org/resource/Glen_Affric, http://dbpedia.org/resource/Inverpolly, http://dbpedia.org/resource/Jolo_Island, http://dbpedia.org/resource/Lake_Chaohu, http://dbpedia.org/resource/Lake_Iliamna, http://dbpedia.org/resource/Stegeborg_Castle, http://dbpedia.org/resource/Fifth_Republic_of_South_Korea, http://dbpedia.org/resource/United_States_cricket_team, http://dbpedia.org/resource/Venezuelan, http://dbpedia.org/resource/Venezuelan_American, http://dbpedia.org/resource/A_Shau_Valley, http://dbpedia.org/resource/Correctional_Service_of_Canada, http://dbpedia.org/resource/Gothenburg_and_Bohus_County, http://dbpedia.org/resource/Kristianstad_County, http://dbpedia.org/resource/List_of_reportedly_haunted_locations_in_the_United_States, http://dbpedia.org/resource/Malmöhus_County, http://dbpedia.org/resource/Aberdeenshire,_Scotland, http://dbpedia.org/resource/Dutch_cricket_team, http://dbpedia.org/resource/Prince_Rupert's_Land, http://dbpedia.org/resource/Reutje, http://dbpedia.org/resource/Cinema_of_Austria, http://dbpedia.org/resource/David_C.H._Austin, http://dbpedia.org/resource/Dominican-American, http://dbpedia.org/resource/Hrodna_Voblast, http://dbpedia.org/resource/New_Zealand_people, http://dbpedia.org/resource/Newark_Basin, http://dbpedia.org/resource/Sierra_Leonean, http://dbpedia.org/resource/Kiowa_Indian_Tribe_of_Oklahoma, http://dbpedia.org/resource/Krylbo, http://dbpedia.org/resource/Trångsund, http://dbpedia.org/resource/American_Jew, http://dbpedia.org/resource/Arenberg, http://dbpedia.org/resource/Prince-Bishopric_of_Paderborn, http://dbpedia.org/resource/United_States_Ambassador_to_Denmark, http://dbpedia.org/resource/2013_Australian_motorcycle_Grand_Prix, http://dbpedia.org/resource/Alter_Botanischer_Garten_der_Universität_Göttingen, http://dbpedia.org/resource/Arkhangelsk_Governorate, http://dbpedia.org/resource/Cangnan_County, http://dbpedia.org/resource/Columbian_Viceroyalty, http://dbpedia.org/resource/Desaguadero_River, http://dbpedia.org/resource/Kamiina_District,_Nagano, http://dbpedia.org/resource/Kolathunadu, http://dbpedia.org/resource/Northern_Krai, http://dbpedia.org/resource/Schöntal_Abbey, http://dbpedia.org/resource/U.S.A, http://dbpedia.org/resource/Victoria_County,_Ontario, http://dbpedia.org/resource/African_National_Congress_of_South_Africa, http://dbpedia.org/resource/Danava_dynasty, http://dbpedia.org/resource/England_national_rugby_sevens_team, http://dbpedia.org/resource/Japan_Sevens, http://dbpedia.org/resource/Kachari_Kingdom, http://dbpedia.org/resource/Naraka_dynasty, http://dbpedia.org/resource/Olonets_Governorate, http://dbpedia.org/resource/Rionegro,_Antioquia, http://dbpedia.org/resource/Taringa,_Queensland, http://dbpedia.org/resource/Tasmanian, http://dbpedia.org/resource/Uruguayan, http://dbpedia.org/resource/Alleppey, http://dbpedia.org/resource/Ciudad_de_Guatemala, http://dbpedia.org/resource/Duchy_of_Bytom, http://dbpedia.org/resource/Firuzkuh, http://dbpedia.org/resource/Free_Republic_of_Schwarzenberg, http://dbpedia.org/resource/Guyana_cricket_team, http://dbpedia.org/resource/KSA, http://dbpedia.org/resource/Kastamonu_Vilayet, http://dbpedia.org/resource/Orio, http://dbpedia.org/resource/Phuoc_Tuy_Province, http://dbpedia.org/resource/Saxe-Zeitz, http://dbpedia.org/resource/Torne_Valley, http://dbpedia.org/resource/Arakan_State, http://dbpedia.org/resource/German_passport, http://dbpedia.org/resource/Hsipaw_State, http://dbpedia.org/resource/Phayao_Kingdom, http://dbpedia.org/resource/Phillippines]... [INFO] P: 0.0032607552329862208, R: 1.0, F: 0.006500314531348291 -[INFO] Cumulative metrics, sample: 13, P: 0.238, R: 0.385, macro F1: 0.244, macro F1*: 0.294 -[INFO] QALD Cumulative metrics, sample: 13, P: 0.392, R: 0.385, macro F1: 0.244, QALD macro F1: 0.388 -[INFO] Current time: 580 ms, average: 3546.31 ms/ques, entity detection: 734.77 ms/ques (linking tool: 644.54 ms/ques), relation detection: 948.46 ms/ques (similarity: 555.69 ms/ques), EDG generation: 342.38 ms/ques +[INFO] Cumulative metrics, sample: 13, P: 0.238, R: 0.385, macro F1: 0.294 +[INFO] QALD Cumulative metrics, sample: 13, P: 0.392, R: 0.385, average F1: 0.244, QALD macro F1: 0.388 +[INFO] Current question consumed time: 533 ms, average: 3415.31 ms/ques, entity detection: 677.38 ms/ques (linking tool: 593.54 ms/ques), relation detection: 946.62 ms/ques (similarity: 575.46 ms/ques), EDG generation: 354.69 ms/ques [INFO] Entity Linking P: 0.23076923076923078, R: 0.23076923076923078, micro F1: 0.23076923076923078, macro F1: 0.23076923076923078 [INFO] Relation Linking P: 0.3076923076923077, R: 0.3076923076923077, micro F1: 0.3076923076923077, macro F1: 0.3076923076923077 [INFO] Type Linking P: 0.9230769230769231, R: 0.9230769230769231, micro F1: 0.9230769230769231, macro F1: 0.9230769230769231 @@ -202,9 +202,9 @@ Question 13: Which airports are located in California, USA? [INFO] Golden Answer: [http://dbpedia.org/resource/Perris_Valley_Airport, http://dbpedia.org/resource/Riverside_Municipal_Airport, http://dbpedia.org/resource/Shelter_Cove_Airport, http://dbpedia.org/resource/Los_Angeles_International_Airport, http://dbpedia.org/resource/Moffett_Federal_Airfield, http://dbpedia.org/resource/Brackett_Field, http://dbpedia.org/resource/Firebaugh_Airport, http://dbpedia.org/resource/Georgetown_Airport_(California), http://dbpedia.org/resource/Lake_Tahoe_Airport, http://dbpedia.org/resource/Livermore_Municipal_Airport, http://dbpedia.org/resource/Oroville_Municipal_Airport, http://dbpedia.org/resource/Reedley_Municipal_Airport, http://dbpedia.org/resource/San_Gabriel_Valley_Airport, http://dbpedia.org/resource/Whiteman_Airport, http://dbpedia.org/resource/Naval_Air_Weapons_Station_China_Lake, http://dbpedia.org/resource/Palo_Alto_Airport, http://dbpedia.org/resource/San_Diego_International_Airport, http://dbpedia.org/resource/Meadowlark_Airport, http://dbpedia.org/resource/Redding_Municipal_Airport, http://dbpedia.org/resource/Fall_River_Mills_Airport, http://dbpedia.org/resource/Happy_Camp_Airport, http://dbpedia.org/resource/Hayfork_Airport, http://dbpedia.org/resource/Lompoc_Airport, http://dbpedia.org/resource/Nervino_Airport, http://dbpedia.org/resource/Oakdale_Airport, http://dbpedia.org/resource/Oceano_County_Airport, http://dbpedia.org/resource/Rio_Vista_Municipal_Airport, http://dbpedia.org/resource/Rogers_Field, http://dbpedia.org/resource/Ruth_Airport, http://dbpedia.org/resource/Scott_Valley_Airport, http://dbpedia.org/resource/Siskiyou_County_Airport, http://dbpedia.org/resource/Sutter_County_Airport, http://dbpedia.org/resource/Trinity_Center_Airport, http://dbpedia.org/resource/Weaverville_Airport, http://dbpedia.org/resource/Weed_Airport, http://dbpedia.org/resource/Frazier_Lake_Airpark, http://dbpedia.org/resource/Tijuana_International_Airport, http://dbpedia.org/resource/Apple_Valley_Airport_(California), http://dbpedia.org/resource/California_Pines_Airport, http://dbpedia.org/resource/Cedarville_Airport, http://dbpedia.org/resource/Healdsburg_Municipal_Airport, http://dbpedia.org/resource/Independence_Airport, http://dbpedia.org/resource/Kern_Valley_Airport, http://dbpedia.org/resource/Lampson_Field, http://dbpedia.org/resource/Lone_Pine_Airport, http://dbpedia.org/resource/Lost_Hills_Airport, http://dbpedia.org/resource/Round_Valley_Airport, http://dbpedia.org/resource/Taft_Airport, http://dbpedia.org/resource/Tehachapi_Municipal_Airport, http://dbpedia.org/resource/Tulelake_Municipal_Airport, http://dbpedia.org/resource/Ukiah_Municipal_Airport, http://dbpedia.org/resource/Wasco_Airport, http://dbpedia.org/resource/Willits_Municipal_Airport, http://dbpedia.org/resource/Naval_Outlying_Field_San_Nicolas_Island, http://dbpedia.org/resource/Turlock_Municipal_Airport, http://dbpedia.org/resource/Dinsmore_Airport_(California), http://dbpedia.org/resource/Elk_Hills-Buttonwillow_Airport, http://dbpedia.org/resource/Hoopa_Airport, http://dbpedia.org/resource/O'Sullivan_Army_Heliport, http://dbpedia.org/resource/Woodlake_Airport, http://dbpedia.org/resource/Monterey_Regional_Airport, http://dbpedia.org/resource/Palm_Springs_International_Airport, http://dbpedia.org/resource/Sacramento_Executive_Airport, http://dbpedia.org/resource/Modesto_City–County_Airport, http://dbpedia.org/resource/Blue_Canyon–Nyack_Airport, http://dbpedia.org/resource/Montgomery_Field_Airport, http://dbpedia.org/resource/Oceanside_Municipal_Airport, http://dbpedia.org/resource/Santa_Cruz_Island_Airport, http://dbpedia.org/resource/Garberville_Airport, http://dbpedia.org/resource/Watsonville_Municipal_Airport, http://dbpedia.org/resource/Haigh_Field_Airport, http://dbpedia.org/resource/Rio_Vista_Airport, http://dbpedia.org/resource/Stanford_Health_Care__Stanford_Hospital__1, http://dbpedia.org/resource/Van_Nuys_Airport, http://dbpedia.org/resource/Brown_Field_Municipal_Airport, http://dbpedia.org/resource/Southern_California_Logistics_Airport, http://dbpedia.org/resource/Catalina_Airport, http://dbpedia.org/resource/Agua_Caliente_Airport, http://dbpedia.org/resource/Fresno_Chandler_Executive_Airport, http://dbpedia.org/resource/Sierra_Sky_Park_Airport, http://dbpedia.org/resource/Inyo_County_Airport, http://dbpedia.org/resource/Eastern_Sierra_Regional_Airport, http://dbpedia.org/resource/Fresno_Yosemite_International_Airport, http://dbpedia.org/resource/Hayward_Executive_Airport, http://dbpedia.org/resource/Oxnard_Airport, http://dbpedia.org/resource/Inyokern_Airport, http://dbpedia.org/resource/Ontario_International_Airport, http://dbpedia.org/resource/Flabob_Airport, http://dbpedia.org/resource/San_Carlos_Airport_(California), http://dbpedia.org/resource/Stockton_Metropolitan_Airport, http://dbpedia.org/resource/Hemet-Ryan_Airport, http://dbpedia.org/resource/General_William_J._Fox_Airfield, http://dbpedia.org/resource/French_Valley_Airport, http://dbpedia.org/resource/Needles_Airport, http://dbpedia.org/resource/Sonoma_Skypark, http://dbpedia.org/resource/Santa_Paula_Airport, http://dbpedia.org/resource/Twentynine_Palms_Airport, http://dbpedia.org/resource/Chiriaco_Summit_Airport, http://dbpedia.org/resource/Desert_Center_Airport, http://dbpedia.org/resource/Crystal_Airport_(California), http://dbpedia.org/resource/Bonny_Doon_Village_Airport, http://dbpedia.org/resource/Corona_Municipal_Airport, http://dbpedia.org/resource/McClellan_Airfield, http://dbpedia.org/resource/Palmdale_Regional_Airport, http://dbpedia.org/resource/Sacramento_International_Airport, http://dbpedia.org/resource/Sacramento_Mather_Airport, http://dbpedia.org/resource/Grey_Butte_Field_Airport, http://dbpedia.org/resource/San_José_International_Airport, http://dbpedia.org/resource/San_Francisco_International_Airport, http://dbpedia.org/resource/Oakland_International_Airport, http://dbpedia.org/resource/Visalia_Municipal_Airport, http://dbpedia.org/resource/Sequoia_Field_Airport, http://dbpedia.org/resource/Meadows_Field_Airport, http://dbpedia.org/resource/Bakersfield_Municipal_Airport, http://dbpedia.org/resource/Mesa_Del_Rey_Airport, http://dbpedia.org/resource/Alturas_Municipal_Airport, http://dbpedia.org/resource/Hesperia_Airport, http://dbpedia.org/resource/Nut_Tree_Airport, http://dbpedia.org/resource/Truckee_Tahoe_Airport, http://dbpedia.org/resource/Susanville_Municipal_Airport, http://dbpedia.org/resource/Yucca_Valley_Airport, http://dbpedia.org/resource/Corcoran_Airport, http://dbpedia.org/resource/Blythe_Airport, http://dbpedia.org/resource/Alpine_County_Airport, http://dbpedia.org/resource/Cameron_Airpark, http://dbpedia.org/resource/Madera_Municipal_Airport, http://dbpedia.org/resource/W._R._Byron_Airport, http://dbpedia.org/resource/William_Robert_Johnston_Municipal_Airport, http://dbpedia.org/resource/Santa_Barbara_Municipal_Airport, http://dbpedia.org/resource/Santa_Maria_Public_Airport, http://dbpedia.org/resource/Hancock_Field_(California), http://dbpedia.org/resource/Auburn_Municipal_Airport_(California), http://dbpedia.org/resource/Barstow-Daggett_Airport, http://dbpedia.org/resource/Douthitt_Strip, http://dbpedia.org/resource/Imperial_County_Airport, http://dbpedia.org/resource/San_Bernardino_International_Airport, http://dbpedia.org/resource/Rosamond_Skypark, http://dbpedia.org/resource/Yolo_County_Airport, http://dbpedia.org/resource/Salinas_Municipal_Airport, http://dbpedia.org/resource/Ocotillo_Airport, http://dbpedia.org/resource/Buchanan_Field_Airport, http://dbpedia.org/resource/John_Wayne_Airport, http://dbpedia.org/resource/Eureka_Municipal_Airport_(California), http://dbpedia.org/resource/Benton_Field, http://dbpedia.org/resource/Yuba_County_Airport, http://dbpedia.org/resource/Del_Norte_County_Airport, http://dbpedia.org/resource/Roy_Williams_Airport, http://dbpedia.org/resource/Chico_Municipal_Airport, http://dbpedia.org/resource/Ranchaero_Airport, http://dbpedia.org/resource/Placerville_Airport, http://dbpedia.org/resource/Corning_Municipal_Airport_(California), http://dbpedia.org/resource/Borges–Clarksburg_Airport, http://dbpedia.org/resource/Elk_Grove_Airport, http://dbpedia.org/resource/Cliff_Hatfield_Memorial_Airport, http://dbpedia.org/resource/New_Jerusalem_Airport, http://dbpedia.org/resource/Angwin–Parrett_Field, http://dbpedia.org/resource/California_Valley_Airport, http://dbpedia.org/resource/Butte_Valley_Airport, http://dbpedia.org/resource/Hyampom_Airport, http://dbpedia.org/resource/New_Cuyama_Airport, http://dbpedia.org/resource/Sierraville_Dearwater_Airport, http://dbpedia.org/resource/Fallbrook_Community_Airpark, http://dbpedia.org/resource/Banning_Municipal_Airport, http://dbpedia.org/resource/Adamson_Landing_Field, http://dbpedia.org/resource/Mammoth_Yosemite_Airport, http://dbpedia.org/resource/Eckert_Field_Airport, http://dbpedia.org/resource/Exeter_Airport_(California), http://dbpedia.org/resource/Baker_Airport, http://dbpedia.org/resource/Holtville_Airport, http://dbpedia.org/resource/Porterville_Municipal_Airport, http://dbpedia.org/resource/Willows-Glenn_County_Airport, http://dbpedia.org/resource/Mountain_Valley_Airport, http://dbpedia.org/resource/Pixley_Airport, http://dbpedia.org/resource/Poso_Airport, http://dbpedia.org/resource/El_Mirage_Field, http://dbpedia.org/resource/Trona_Airport, http://dbpedia.org/resource/Eagle_Field_(airport), http://dbpedia.org/resource/Ocean_Ridge_Airport, http://dbpedia.org/resource/Adin_Airport, http://dbpedia.org/resource/Arcata-Eureka_Airport, http://dbpedia.org/resource/Amador_County_Airport, http://dbpedia.org/resource/Gansner_Field, http://dbpedia.org/resource/Santa_Ynez_Airport, http://dbpedia.org/resource/Salton_Sea_Airport, http://dbpedia.org/resource/Merced_Regional_Airport, http://dbpedia.org/resource/Rankin_Field, http://dbpedia.org/resource/Dunsmuir_Municipal-Mott_Airport, http://dbpedia.org/resource/Shafter_Airport, http://dbpedia.org/resource/Mefford_Field_Airport, http://dbpedia.org/resource/Borrego_Valley_Airport, http://dbpedia.org/resource/Paradise_Skypark, http://dbpedia.org/resource/Kingdon_Airpark, http://dbpedia.org/resource/Delano_Municipal_Airport, http://dbpedia.org/resource/Calexico_International_Airport, http://dbpedia.org/resource/Little_River_Airport, http://dbpedia.org/resource/Colusa_County_Airport, http://dbpedia.org/resource/Pine_Mountain_Lake_Airport, http://dbpedia.org/resource/Brawley_Municipal_Airport, http://dbpedia.org/resource/Bryant_Field_(airport), http://dbpedia.org/resource/California_City_Municipal_Airport, http://dbpedia.org/resource/Chowchilla_Airport, http://dbpedia.org/resource/Cloverdale_Municipal_Airport, http://dbpedia.org/resource/Boonville_Airport_(California), http://dbpedia.org/resource/Chemehuevi_Valley_Airport] [INFO] Predicted Answer: [http://dbpedia.org/resource/Perris_Valley_Airport, http://dbpedia.org/resource/Riverside_Municipal_Airport, http://dbpedia.org/resource/Shelter_Cove_Airport, http://dbpedia.org/resource/Los_Angeles_International_Airport, http://dbpedia.org/resource/Moffett_Federal_Airfield, http://dbpedia.org/resource/Brackett_Field, http://dbpedia.org/resource/Firebaugh_Airport, http://dbpedia.org/resource/Georgetown_Airport_(California), http://dbpedia.org/resource/Lake_Tahoe_Airport, http://dbpedia.org/resource/Livermore_Municipal_Airport, http://dbpedia.org/resource/Oroville_Municipal_Airport, http://dbpedia.org/resource/Reedley_Municipal_Airport, http://dbpedia.org/resource/San_Gabriel_Valley_Airport, http://dbpedia.org/resource/Whiteman_Airport, http://dbpedia.org/resource/Naval_Air_Weapons_Station_China_Lake, http://dbpedia.org/resource/Palo_Alto_Airport, http://dbpedia.org/resource/San_Diego_International_Airport, http://dbpedia.org/resource/Meadowlark_Airport, http://dbpedia.org/resource/Redding_Municipal_Airport, http://dbpedia.org/resource/Fall_River_Mills_Airport, http://dbpedia.org/resource/Happy_Camp_Airport, http://dbpedia.org/resource/Hayfork_Airport, http://dbpedia.org/resource/Lompoc_Airport, http://dbpedia.org/resource/Nervino_Airport, http://dbpedia.org/resource/Oakdale_Airport, http://dbpedia.org/resource/Oceano_County_Airport, http://dbpedia.org/resource/Rio_Vista_Municipal_Airport, http://dbpedia.org/resource/Rogers_Field, http://dbpedia.org/resource/Ruth_Airport, http://dbpedia.org/resource/Scott_Valley_Airport, http://dbpedia.org/resource/Siskiyou_County_Airport, http://dbpedia.org/resource/Sutter_County_Airport, http://dbpedia.org/resource/Trinity_Center_Airport, http://dbpedia.org/resource/Weaverville_Airport, http://dbpedia.org/resource/Weed_Airport, http://dbpedia.org/resource/Frazier_Lake_Airpark, http://dbpedia.org/resource/Tijuana_International_Airport, http://dbpedia.org/resource/Apple_Valley_Airport_(California), http://dbpedia.org/resource/California_Pines_Airport, http://dbpedia.org/resource/Cedarville_Airport, http://dbpedia.org/resource/Healdsburg_Municipal_Airport, http://dbpedia.org/resource/Independence_Airport, http://dbpedia.org/resource/Kern_Valley_Airport, http://dbpedia.org/resource/Lampson_Field, http://dbpedia.org/resource/Lone_Pine_Airport, http://dbpedia.org/resource/Lost_Hills_Airport, http://dbpedia.org/resource/Round_Valley_Airport, http://dbpedia.org/resource/Taft_Airport, http://dbpedia.org/resource/Tehachapi_Municipal_Airport, http://dbpedia.org/resource/Tulelake_Municipal_Airport, http://dbpedia.org/resource/Ukiah_Municipal_Airport, http://dbpedia.org/resource/Wasco_Airport, http://dbpedia.org/resource/Willits_Municipal_Airport, http://dbpedia.org/resource/Naval_Outlying_Field_San_Nicolas_Island, http://dbpedia.org/resource/Turlock_Municipal_Airport, http://dbpedia.org/resource/Dinsmore_Airport_(California), http://dbpedia.org/resource/Elk_Hills-Buttonwillow_Airport, http://dbpedia.org/resource/Hoopa_Airport, http://dbpedia.org/resource/O'Sullivan_Army_Heliport, http://dbpedia.org/resource/Woodlake_Airport, http://dbpedia.org/resource/Monterey_Regional_Airport, http://dbpedia.org/resource/Palm_Springs_International_Airport, http://dbpedia.org/resource/Sacramento_Executive_Airport, http://dbpedia.org/resource/Modesto_City–County_Airport, http://dbpedia.org/resource/Blue_Canyon–Nyack_Airport, http://dbpedia.org/resource/Montgomery_Field_Airport, http://dbpedia.org/resource/Oceanside_Municipal_Airport, http://dbpedia.org/resource/Santa_Cruz_Island_Airport, http://dbpedia.org/resource/Garberville_Airport, http://dbpedia.org/resource/Watsonville_Municipal_Airport, http://dbpedia.org/resource/Haigh_Field_Airport, http://dbpedia.org/resource/Rio_Vista_Airport, http://dbpedia.org/resource/Stanford_Health_Care__Stanford_Hospital__1] [INFO] P: 1.0, R: 0.35784313725490197, F: 0.5270758122743682 -[INFO] Cumulative metrics, sample: 14, P: 0.292, R: 0.383, macro F1: 0.264, macro F1*: 0.332 -[INFO] QALD Cumulative metrics, sample: 14, P: 0.435, R: 0.383, macro F1: 0.264, QALD macro F1: 0.407 -[INFO] Current time: 15511 ms, average: 4400.93 ms/ques, entity detection: 920.36 ms/ques (linking tool: 825.29 ms/ques), relation detection: 1191.29 ms/ques (similarity: 765.50 ms/ques), EDG generation: 319.07 ms/ques +[INFO] Cumulative metrics, sample: 14, P: 0.292, R: 0.383, macro F1: 0.332 +[INFO] QALD Cumulative metrics, sample: 14, P: 0.435, R: 0.383, average F1: 0.264, QALD macro F1: 0.407 +[INFO] Current question consumed time: 14106 ms, average: 4178.93 ms/ques, entity detection: 840.43 ms/ques (linking tool: 750.93 ms/ques), relation detection: 1168.07 ms/ques (similarity: 776.43 ms/ques), EDG generation: 330.43 ms/ques [INFO] Entity Linking P: 0.21428571428571427, R: 0.21428571428571427, micro F1: 0.21428571428571427, macro F1: 0.21428571428571427 [INFO] Relation Linking P: 0.2857142857142857, R: 0.2857142857142857, micro F1: 0.2857142857142857, macro F1: 0.2857142857142857 [INFO] Type Linking P: 0.8571428571428571, R: 0.8571428571428571, micro F1: 0.8571428571428571, macro F1: 0.8571428571428571 @@ -217,9 +217,9 @@ Question 14: What are the nicknames of San Francisco? [INFO] Golden Answer: [The City; The City by the Bay; Fog City; San Fran;Frisco (locally disparaged); The City that Knows How (past); Baghdadby the Bay (past); The Paris of the West@en] [INFO] Predicted Answer: [The City; The City by the Bay; Fog City; San Fran; Frisco ; The City that Knows How ; Baghdad by the Bay ; The Paris of the West] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 15, P: 0.273, R: 0.357, macro F1: 0.247, macro F1*: 0.309 -[INFO] QALD Cumulative metrics, sample: 15, P: 0.406, R: 0.357, macro F1: 0.247, QALD macro F1: 0.380 -[INFO] Current time: 5164 ms, average: 4451.87 ms/ques, entity detection: 920.07 ms/ques (linking tool: 824.53 ms/ques), relation detection: 1264.27 ms/ques (similarity: 788.80 ms/ques), EDG generation: 298.67 ms/ques +[INFO] Cumulative metrics, sample: 15, P: 0.273, R: 0.357, macro F1: 0.309 +[INFO] QALD Cumulative metrics, sample: 15, P: 0.406, R: 0.357, average F1: 0.247, QALD macro F1: 0.380 +[INFO] Current question consumed time: 4922 ms, average: 4228.47 ms/ques, entity detection: 844.07 ms/ques (linking tool: 754.67 ms/ques), relation detection: 1234.67 ms/ques (similarity: 798.47 ms/ques), EDG generation: 309.47 ms/ques [INFO] Entity Linking P: 0.2, R: 0.2, micro F1: 0.2, macro F1: 0.20000000000000004 [INFO] Relation Linking P: 0.26666666666666666, R: 0.26666666666666666, micro F1: 0.26666666666666666, macro F1: 0.26666666666666666 [INFO] Type Linking P: 0.8666666666666667, R: 0.8666666666666667, micro F1: 0.8666666666666667, macro F1: 0.8666666666666667 @@ -232,9 +232,9 @@ Question 15: What is Angela Merkel’s birth name? [INFO] Golden Answer: [Angela Dorothea Kasner] [INFO] Predicted Answer: [Angela Dorothea Kasner] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 16, P: 0.318, R: 0.397, macro F1: 0.294, macro F1*: 0.354 -[INFO] QALD Cumulative metrics, sample: 16, P: 0.443, R: 0.397, macro F1: 0.294, QALD macro F1: 0.419 -[INFO] Current time: 1217 ms, average: 4249.69 ms/ques, entity detection: 877.38 ms/ques (linking tool: 784.06 ms/ques), relation detection: 1213.25 ms/ques (similarity: 754.94 ms/ques), EDG generation: 280.69 ms/ques +[INFO] Cumulative metrics, sample: 16, P: 0.318, R: 0.397, macro F1: 0.354 +[INFO] QALD Cumulative metrics, sample: 16, P: 0.443, R: 0.397, average F1: 0.294, QALD macro F1: 0.419 +[INFO] Current question consumed time: 1094 ms, average: 4032.56 ms/ques, entity detection: 804.12 ms/ques (linking tool: 717.31 ms/ques), relation detection: 1183.81 ms/ques (similarity: 764.12 ms/ques), EDG generation: 290.12 ms/ques [INFO] Entity Linking P: 0.1875, R: 0.1875, micro F1: 0.1875, macro F1: 0.1875 [INFO] Relation Linking P: 0.25, R: 0.25, micro F1: 0.25, macro F1: 0.25 [INFO] Type Linking P: 0.875, R: 0.875, micro F1: 0.875, macro F1: 0.875 @@ -247,9 +247,9 @@ Question 16: Who is the mayor of Berlin? [INFO] Golden Answer: [Michael Müller] [INFO] Predicted Answer: [http://dbpedia.org/resource/2009_World_Championships_in_Athletics, http://dbpedia.org/resource/Halensee, http://dbpedia.org/resource/Adlershof, http://dbpedia.org/resource/Elements_(League_of_Legends), http://dbpedia.org/resource/Hellersdorf, http://dbpedia.org/resource/Pankow_(locality), http://dbpedia.org/resource/Gropiusstadt, http://dbpedia.org/resource/Heiligensee, http://dbpedia.org/resource/Staaken, http://dbpedia.org/resource/Schmargendorf, http://dbpedia.org/resource/Berlin_Adler, http://dbpedia.org/resource/Fnatic, http://dbpedia.org/resource/Barenboim–Said_Academy, http://dbpedia.org/resource/1923_German_football_championship, http://dbpedia.org/resource/1924_German_football_championship, http://dbpedia.org/resource/Theater_heute, http://dbpedia.org/resource/TTC_Berlin_Eastside, http://dbpedia.org/resource/1936_Tschammerpokal_Final, http://dbpedia.org/resource/Humboldt_University_of_Berlin, http://dbpedia.org/resource/Technical_University_of_Berlin, http://dbpedia.org/resource/Wilmersdorf, http://dbpedia.org/resource/Mitte_(locality), http://dbpedia.org/resource/Tegel, http://dbpedia.org/resource/Pan_(magazine), http://dbpedia.org/resource/Tiergarten,_Berlin, http://dbpedia.org/resource/Hochschule_für_Musik_%22Hanns_Eisler%22, http://dbpedia.org/resource/Kaulsdorf_(Berlin), http://dbpedia.org/resource/British_and_Irish_Modern_Music_Institute, http://dbpedia.org/resource/Berliner_Schlittschuhclub, http://dbpedia.org/resource/Unicorns_of_Love, http://dbpedia.org/resource/SRH_Hochschule_Berlin, http://dbpedia.org/resource/1993_DFB-Pokal_Final, http://dbpedia.org/resource/Berlin_University_of_the_Arts, http://dbpedia.org/resource/2015_UEFA_Champions_League_Final, http://dbpedia.org/resource/Alt-Treptow, http://dbpedia.org/resource/Berlin_Brandenburg_Airport, http://dbpedia.org/resource/Berlin_School_of_Economics_and_Law, http://dbpedia.org/resource/Britz, http://dbpedia.org/resource/Charité, http://dbpedia.org/resource/Charlottenburg-Wilmersdorf, http://dbpedia.org/resource/Dahlem_(Berlin), http://dbpedia.org/resource/Eisbären_Berlin, http://dbpedia.org/resource/Evangelisches_Gymnasium_zum_Grauen_Kloster, http://dbpedia.org/resource/Free_University_of_Berlin, http://dbpedia.org/resource/Friedrichshain, http://dbpedia.org/resource/Gatow, http://dbpedia.org/resource/Kladow, http://dbpedia.org/resource/Kreuzberg, http://dbpedia.org/resource/Kroll_Opera_House, http://dbpedia.org/resource/Köpenick, http://dbpedia.org/resource/Lichtenberg, http://dbpedia.org/resource/Malchow_(Berlin), http://dbpedia.org/resource/Marzahn, http://dbpedia.org/resource/Mitte, http://dbpedia.org/resource/Moabit, http://dbpedia.org/resource/Neukölln, http://dbpedia.org/resource/Pankow, http://dbpedia.org/resource/Spandau, http://dbpedia.org/resource/Steglitz, http://dbpedia.org/resource/Steglitz-Zehlendorf, http://dbpedia.org/resource/Tempelhof-Schöneberg, http://dbpedia.org/resource/Wedding_(Berlin), http://dbpedia.org/resource/Zehlendorf_(Berlin), http://dbpedia.org/resource/Schöneberg, http://dbpedia.org/resource/Siemensstadt, http://dbpedia.org/resource/Neue_Rundschau, http://dbpedia.org/resource/Gesundbrunnen_(Berlin), http://dbpedia.org/resource/Marzahn-Hellersdorf, http://dbpedia.org/resource/1956_German_football_championship, http://dbpedia.org/resource/Siegessäule_(magazine), http://dbpedia.org/resource/1934_German_football_championship, http://dbpedia.org/resource/1936_German_football_championship, http://dbpedia.org/resource/1937_German_football_championship, http://dbpedia.org/resource/1939_German_football_championship, http://dbpedia.org/resource/1940_German_football_championship, http://dbpedia.org/resource/International_Psychoanalytic_University_Berlin, http://dbpedia.org/resource/2009_DFB-Pokal_Final, http://dbpedia.org/resource/1944_German_football_championship, http://dbpedia.org/resource/NPR_Berlin, http://dbpedia.org/resource/Games_Academy, http://dbpedia.org/resource/Kaveh_(magazine), http://dbpedia.org/resource/Gambit_Gaming, http://dbpedia.org/resource/Deutsche_Film-_und_Fernsehakademie_Berlin, http://dbpedia.org/resource/Karlshorst, http://dbpedia.org/resource/Hertie_School_of_Governance, http://dbpedia.org/resource/Wannsee, http://dbpedia.org/resource/Bard_College_Berlin, http://dbpedia.org/resource/Neukölln_(locality), http://dbpedia.org/resource/Ulenspiegel, http://dbpedia.org/resource/G2_Esports, http://dbpedia.org/resource/Falkenhagener_Feld, http://dbpedia.org/resource/Haselhorst, http://dbpedia.org/resource/Wilhelmstadt, http://dbpedia.org/resource/2011_DFB-Pokal_Final, http://dbpedia.org/resource/1942_German_football_championship, http://dbpedia.org/resource/Hansaviertel, http://dbpedia.org/resource/2015_UEFA_Women's_Champions_League_Final, http://dbpedia.org/resource/Beuth_University_of_Applied_Sciences_Berlin, http://dbpedia.org/resource/Charlottenburg, http://dbpedia.org/resource/Grunewald, http://dbpedia.org/resource/Tempelhof, http://dbpedia.org/resource/Friedrichshain-Kreuzberg, http://dbpedia.org/resource/Eulenspiegel_(magazine), http://dbpedia.org/resource/Lichtenberg_(locality), http://dbpedia.org/resource/Mariendorf, http://dbpedia.org/resource/Berliner_Illustrirte_Zeitung, http://dbpedia.org/resource/1941_German_football_championship, http://dbpedia.org/resource/1962_German_football_championship, http://dbpedia.org/resource/1908_German_football_championship, http://dbpedia.org/resource/1927_German_football_championship, http://dbpedia.org/resource/1950_German_football_championship, http://dbpedia.org/resource/2015_DFB-Pokal_Final, http://dbpedia.org/resource/Berlin_British_School, http://dbpedia.org/resource/Berlin_Schönefeld_Airport, http://dbpedia.org/resource/Berlin_Tegel_Airport, http://dbpedia.org/resource/Der_Adler, http://dbpedia.org/resource/Die_Freundin, http://dbpedia.org/resource/ESCP_Europe, http://dbpedia.org/resource/Weissensee_(Berlin), http://dbpedia.org/resource/European_School_of_Management_and_Technology, http://dbpedia.org/resource/Borsigwalde, http://dbpedia.org/resource/Bohnsdorf, http://dbpedia.org/resource/Berlin_Mathematical_School, http://dbpedia.org/resource/Olimpijka, http://dbpedia.org/resource/104.6_RTL, http://dbpedia.org/resource/Steinbeis-Hochschule_Berlin, http://dbpedia.org/resource/Berlin_Graduate_School_for_Transnational_Studies, http://dbpedia.org/resource/Reboot.fm, http://dbpedia.org/resource/Marienfelde, http://dbpedia.org/resource/1951_German_football_championship, http://dbpedia.org/resource/1953_German_football_championship, http://dbpedia.org/resource/Lichterfelde_(Berlin), http://dbpedia.org/resource/Westend_(Berlin), http://dbpedia.org/resource/Konradshöhe, http://dbpedia.org/resource/Spandau_(locality), http://dbpedia.org/resource/Berlin_Institute_for_Advanced_Study, http://dbpedia.org/resource/Hochschule_für_Technik_und_Wirtschaft_Berlin, http://dbpedia.org/resource/Die_Gartenlaube, http://dbpedia.org/resource/BSC_Preussen, http://dbpedia.org/resource/Berlin_Rebels, http://dbpedia.org/resource/Lodown_magazine, http://dbpedia.org/resource/Mono.Kultur, http://dbpedia.org/resource/Zuerst!, http://dbpedia.org/resource/Cicero_(magazine), http://dbpedia.org/resource/1938_German_football_championship, http://dbpedia.org/resource/2008_DFB-Pokal_Final, http://dbpedia.org/resource/2010_DFB-Pokal_Final, http://dbpedia.org/resource/2012_DFB-Pokal_Final, http://dbpedia.org/resource/2013_DFB-Pokal_Final, http://dbpedia.org/resource/2014_DFB-Pokal_Final, http://dbpedia.org/resource/2016_DFB-Pokal_Final, http://dbpedia.org/resource/Texte_zur_Kunst, http://dbpedia.org/resource/Alt-Hohenschönhausen, http://dbpedia.org/resource/Altglienicke, http://dbpedia.org/resource/Blankenfelde, http://dbpedia.org/resource/Canisius-Kolleg_Berlin, http://dbpedia.org/resource/Charlottenburg-Nord, http://dbpedia.org/resource/Fennpfuhl, http://dbpedia.org/resource/Friedrichshagen, http://dbpedia.org/resource/Heinersdorf, http://dbpedia.org/resource/Hermsdorf_(Berlin), http://dbpedia.org/resource/Lübars, http://dbpedia.org/resource/Mahlsdorf, http://dbpedia.org/resource/Märkisches_Viertel, http://dbpedia.org/resource/Müggelheim, http://dbpedia.org/resource/Neu-Hohenschönhausen, http://dbpedia.org/resource/Niederschönhausen, http://dbpedia.org/resource/Oberschöneweide, http://dbpedia.org/resource/Reinickendorf_(locality), http://dbpedia.org/resource/Rosenthal_(Berlin), http://dbpedia.org/resource/Treptow-Köpenick, http://dbpedia.org/resource/Waidmannslust, http://dbpedia.org/resource/Wittenau, http://dbpedia.org/resource/DU&ICH, http://dbpedia.org/resource/1943_German_football_championship, http://dbpedia.org/resource/Reimann_School, http://dbpedia.org/resource/Agricultural_University_of_Berlin, http://dbpedia.org/resource/Guter_Rat, http://dbpedia.org/resource/KUNST_Magazin, http://dbpedia.org/resource/1992_DFB-Pokal_Final, http://dbpedia.org/resource/2004_DFB-Pokal_Final, http://dbpedia.org/resource/2005_DFB-Pokal_Final, http://dbpedia.org/resource/2006_DFB-Pokal_Final, http://dbpedia.org/resource/Archenhold_School, http://dbpedia.org/resource/FC_Schalke_04_League_of_Legends, http://dbpedia.org/resource/Team_ROCCAT, http://dbpedia.org/resource/11_Freunde, http://dbpedia.org/resource/2007_DFB-Pokal_Final, http://dbpedia.org/resource/Hochhaus_an_der_Weberwiese, http://dbpedia.org/resource/Berlin_School_of_Library_and_Information_Science, http://dbpedia.org/resource/1959_German_football_championship_Final, http://dbpedia.org/resource/Zenith_Magazine, http://dbpedia.org/resource/John_F._Kennedy_Institute_for_North_American_Studies, http://dbpedia.org/resource/Amor_Vincit_Omnia_(Caravaggio), http://dbpedia.org/resource/In_the_Conservatory, http://dbpedia.org/resource/Interbau, http://dbpedia.org/resource/Berlin_Thunder, http://dbpedia.org/resource/Interior_with_a_Woman_weighing_Gold_Coin, http://dbpedia.org/resource/Woman_Lacing_Her_Bodice_Beside_a_Cradle, http://dbpedia.org/resource/2006_BBL_Champions_Cup, http://dbpedia.org/resource/Perseus_Freeing_Andromeda_(Rubens), http://dbpedia.org/resource/Christ_Giving_the_Keys_to_St._Peter_(Rubens), http://dbpedia.org/resource/Berlin_(sculpture), http://dbpedia.org/resource/Fraunhofer_Institute_for_Telecommunications, http://dbpedia.org/resource/Friedrichstadt-Palast, http://dbpedia.org/resource/The_Abbey_in_the_Oakwood, http://dbpedia.org/resource/Malle_Babbe, http://dbpedia.org/resource/Solly_Madonna, http://dbpedia.org/resource/Terranuova_Madonna, http://dbpedia.org/resource/Moses_Breaking_the_Tablets_of_the_Law, http://dbpedia.org/resource/Portrait_of_a_Man_(Signorelli), http://dbpedia.org/resource/1949_in_fine_arts_of_the_Soviet_Union, http://dbpedia.org/resource/The_Man_with_the_Golden_Helmet, http://dbpedia.org/resource/Otto-Suhr-Institut, http://dbpedia.org/resource/2009_Euroleague_Final_Four, http://dbpedia.org/resource/Portrait_of_Ugolino_Martelli, http://dbpedia.org/resource/Wasserfreunde_Spandau_04, http://dbpedia.org/resource/Netherlandish_Proverbs, http://dbpedia.org/resource/Portrait_of_Frederick_III_of_Saxony, http://dbpedia.org/resource/1991_Men's_Hockey_Champions_Trophy, http://dbpedia.org/resource/1995_Men's_Hockey_Champions_Trophy, http://dbpedia.org/resource/1997_Women's_Hockey_Champions_Trophy, http://dbpedia.org/resource/1986_UEFA_Cup_Final, http://dbpedia.org/resource/Self-Portrait_(Titian,_Berlin), http://dbpedia.org/resource/The_Fountain_of_Youth_(Cranach), http://dbpedia.org/resource/Bronze_statuette_of_hoplite_(Berlin_Antiquities_Collection_Misc._7470), http://dbpedia.org/resource/Woman_with_a_Pearl_Necklace, http://dbpedia.org/resource/Graft_(architects), http://dbpedia.org/resource/Internationale_Gartenausstellung_2017, http://dbpedia.org/resource/Colonna_Madonna, http://dbpedia.org/resource/Caspar_David_Friedrich_in_his_Studio, http://dbpedia.org/resource/Assumption_of_the_Virgin_(Andrea_del_Castagno), http://dbpedia.org/resource/Ibero-American_Institute, http://dbpedia.org/resource/2016_Euroleague_Final_Four, http://dbpedia.org/resource/2014_BBL_Champions_Cup, http://dbpedia.org/resource/2013_BBL_Champions_Cup, http://dbpedia.org/resource/Double_Herm_of_Socrates_and_Seneca, http://dbpedia.org/resource/UFC_Fight_Night:_Jędrzejczyk_vs._Penne, http://dbpedia.org/resource/Great_Industrial_Exposition_of_Berlin, http://dbpedia.org/resource/Berlin_Sluggers, http://dbpedia.org/resource/Monforte_Altarpiece, http://dbpedia.org/resource/Susanna_and_the_Elders_(Rembrandt), http://dbpedia.org/resource/Profile_Portrait_of_a_Young_Lady, http://dbpedia.org/resource/UFC_Fight_Night:_Munoz_vs._Mousasi, http://dbpedia.org/resource/The_Three_Musicians_(painting), http://dbpedia.org/resource/Roger_freeing_Angelica_(Böcklin), http://dbpedia.org/resource/2001_Speedway_Grand_Prix_of_Germany, http://dbpedia.org/resource/Memoria_Urbana_Berlin] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 17, P: 0.300, R: 0.374, macro F1: 0.276, macro F1*: 0.333 -[INFO] QALD Cumulative metrics, sample: 17, P: 0.417, R: 0.374, macro F1: 0.276, QALD macro F1: 0.394 -[INFO] Current time: 6508 ms, average: 4382.53 ms/ques, entity detection: 895.00 ms/ques (linking tool: 801.65 ms/ques), relation detection: 1317.82 ms/ques (similarity: 798.12 ms/ques), EDG generation: 264.82 ms/ques +[INFO] Cumulative metrics, sample: 17, P: 0.300, R: 0.374, macro F1: 0.333 +[INFO] QALD Cumulative metrics, sample: 17, P: 0.417, R: 0.374, average F1: 0.276, QALD macro F1: 0.394 +[INFO] Current question consumed time: 6079 ms, average: 4152.94 ms/ques, entity detection: 818.71 ms/ques (linking tool: 732.71 ms/ques), relation detection: 1280.53 ms/ques (similarity: 805.88 ms/ques), EDG generation: 273.06 ms/ques [INFO] Entity Linking P: 0.17647058823529413, R: 0.17647058823529413, micro F1: 0.17647058823529413, macro F1: 0.17647058823529413 [INFO] Relation Linking P: 0.23529411764705882, R: 0.23529411764705882, micro F1: 0.23529411764705882, macro F1: 0.23529411764705882 [INFO] Type Linking P: 0.8823529411764706, R: 0.8823529411764706, micro F1: 0.8823529411764706, macro F1: 0.8823529411764706 @@ -262,9 +262,9 @@ Question 17: Which countries in the European Union adopted the Euro? [INFO] Golden Answer: [http://dbpedia.org/resource/French_Fifth_Republic, http://dbpedia.org/resource/Akrotiri_and_Dhekelia, http://dbpedia.org/resource/Andorra, http://dbpedia.org/resource/Austria, http://dbpedia.org/resource/Baltic_states, http://dbpedia.org/resource/Belgium, http://dbpedia.org/resource/Cyprus, http://dbpedia.org/resource/Estonia, http://dbpedia.org/resource/European_Union, http://dbpedia.org/resource/Finland, http://dbpedia.org/resource/France, http://dbpedia.org/resource/Germany, http://dbpedia.org/resource/Greece, http://dbpedia.org/resource/Kosovo, http://dbpedia.org/resource/Latvia, http://dbpedia.org/resource/Lithuania, http://dbpedia.org/resource/Luxembourg, http://dbpedia.org/resource/Monaco, http://dbpedia.org/resource/Montenegro, http://dbpedia.org/resource/Netherlands, http://dbpedia.org/resource/Portugal, http://dbpedia.org/resource/Republic_of_Ireland, http://dbpedia.org/resource/Slovenia, http://dbpedia.org/resource/Spain, http://dbpedia.org/resource/Twente, http://dbpedia.org/resource/San_Marino, http://dbpedia.org/resource/Republic_of_Saugeais, http://dbpedia.org/resource/Republic_of_Montenegro_(1992–2006), http://dbpedia.org/resource/Adélie_Land, http://dbpedia.org/resource/French_Southern_and_Antarctic_Lands, http://dbpedia.org/resource/Macedonia_(region), http://dbpedia.org/resource/Serbia_and_Montenegro, http://dbpedia.org/resource/Vatican_City, http://dbpedia.org/resource/Basque_Country_(greater_region), http://dbpedia.org/resource/Benelux, http://dbpedia.org/resource/Collectivity_of_Saint_Martin, http://dbpedia.org/resource/Kerguelen_Islands, http://dbpedia.org/resource/Saint_Barthélemy, http://dbpedia.org/resource/Saint_Pierre_and_Miquelon, http://dbpedia.org/resource/Tromelin_Island, http://dbpedia.org/resource/United_Nations_Interim_Administration_Mission_in_Kosovo, http://dbpedia.org/resource/Autonomous_Province_of_Kosovo_and_Metohija, http://dbpedia.org/resource/French_overseas_departments_and_territories, http://dbpedia.org/resource/Kingdom_of_Wallachia, http://dbpedia.org/resource/Îles_des_Saintes, http://dbpedia.org/resource/Third_Hellenic_Republic, http://dbpedia.org/resource/Luhansk_People's_Republic] [INFO] Predicted Answer: [http://dbpedia.org/resource/EU_Battlegroup, http://dbpedia.org/resource/European_Parliament_election,_1994, http://dbpedia.org/resource/European_Parliament_election,_1999, http://dbpedia.org/resource/European_Parliament_election,_2004, http://dbpedia.org/resource/European_Parliament_election,_2009, http://dbpedia.org/resource/European_Parliament_election,_2014, http://dbpedia.org/resource/Eurofor, http://dbpedia.org/resource/Balkan_Battlegroup, http://dbpedia.org/resource/Battlegroup_107, http://dbpedia.org/resource/Battlegroup_I-2010, http://dbpedia.org/resource/British_Battlegroup, http://dbpedia.org/resource/Czech–Slovak_Battlegroup, http://dbpedia.org/resource/EUBG_2014_II, http://dbpedia.org/resource/Spanish–Italian_Amphibious_Battlegroup, http://dbpedia.org/resource/UK–Dutch_Battlegroup, http://dbpedia.org/resource/Galileo_Records, http://dbpedia.org/resource/Eszterházy_Károly_University, http://dbpedia.org/resource/European_Day_of_Remembrance_for_Victims_of_Stalinism_and_Nazism, http://dbpedia.org/resource/Tirgan, http://dbpedia.org/resource/Innsbruck_Medical_University, http://dbpedia.org/resource/Europe_Day, http://dbpedia.org/resource/Party_of_European_Socialists, http://dbpedia.org/resource/Nordic_Battlegroup, http://dbpedia.org/resource/James_Dobbins_(diplomat), http://dbpedia.org/resource/European_Trade_Union_Institute_(ETUI), http://dbpedia.org/resource/Visegrád_Battlegroup, http://dbpedia.org/resource/Weimar_Battlegroup, http://dbpedia.org/resource/Multinational_Land_Force, http://dbpedia.org/resource/European_Trade_Union_Confederation, http://dbpedia.org/resource/Rodrigo_Rivera_Salazar, http://dbpedia.org/resource/European_Maritime_Force, http://dbpedia.org/resource/Europa_TV, http://dbpedia.org/resource/Ana_Paula_Zacarias, http://dbpedia.org/resource/10_cent_euro_coin, http://dbpedia.org/resource/1_cent_euro_coin, http://dbpedia.org/resource/1_euro_coin, http://dbpedia.org/resource/20_cent_euro_coin, http://dbpedia.org/resource/2_cent_euro_coin, http://dbpedia.org/resource/50_cent_euro_coin, http://dbpedia.org/resource/5_cent_euro_coin, http://dbpedia.org/resource/European_Court_of_Justice] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 18, P: 0.283, R: 0.353, macro F1: 0.261, macro F1*: 0.314 -[INFO] QALD Cumulative metrics, sample: 18, P: 0.394, R: 0.353, macro F1: 0.261, QALD macro F1: 0.373 -[INFO] Current time: 5871 ms, average: 4465.22 ms/ques, entity detection: 892.83 ms/ques (linking tool: 797.11 ms/ques), relation detection: 1384.78 ms/ques (similarity: 824.44 ms/ques), EDG generation: 251.06 ms/ques +[INFO] Cumulative metrics, sample: 18, P: 0.283, R: 0.353, macro F1: 0.314 +[INFO] QALD Cumulative metrics, sample: 18, P: 0.394, R: 0.353, average F1: 0.261, QALD macro F1: 0.373 +[INFO] Current question consumed time: 5374 ms, average: 4220.78 ms/ques, entity detection: 819.17 ms/ques (linking tool: 730.56 ms/ques), relation detection: 1338.28 ms/ques (similarity: 833.00 ms/ques), EDG generation: 258.67 ms/ques [INFO] Entity Linking P: 0.16666666666666666, R: 0.16666666666666666, micro F1: 0.16666666666666666, macro F1: 0.16666666666666666 [INFO] Relation Linking P: 0.2222222222222222, R: 0.2222222222222222, micro F1: 0.2222222222222222, macro F1: 0.2222222222222222 [INFO] Type Linking P: 0.8888888888888888, R: 0.8888888888888888, micro F1: 0.8888888888888888, macro F1: 0.8888888888888888 @@ -277,9 +277,9 @@ Question 18: Which software has been published by Mean Hamster Software? [INFO] Golden Answer: [http://dbpedia.org/resource/Crossbow_(video_game), http://dbpedia.org/resource/Riven] [INFO] Predicted Answer: [http://dbpedia.org/resource/Crossbow_(video_game), http://dbpedia.org/resource/Riven] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 19, P: 0.321, R: 0.387, macro F1: 0.300, macro F1*: 0.351 -[INFO] QALD Cumulative metrics, sample: 19, P: 0.426, R: 0.387, macro F1: 0.300, QALD macro F1: 0.406 -[INFO] Current time: 1178 ms, average: 4292.21 ms/ques, entity detection: 855.00 ms/ques (linking tool: 760.95 ms/ques), relation detection: 1331.68 ms/ques (similarity: 799.37 ms/ques), EDG generation: 238.53 ms/ques +[INFO] Cumulative metrics, sample: 19, P: 0.321, R: 0.387, macro F1: 0.351 +[INFO] QALD Cumulative metrics, sample: 19, P: 0.426, R: 0.387, average F1: 0.300, QALD macro F1: 0.406 +[INFO] Current question consumed time: 1145 ms, average: 4058.89 ms/ques, entity detection: 785.11 ms/ques (linking tool: 697.89 ms/ques), relation detection: 1287.63 ms/ques (similarity: 807.47 ms/ques), EDG generation: 245.05 ms/ques [INFO] Entity Linking P: 0.15789473684210525, R: 0.15789473684210525, micro F1: 0.15789473684210525, macro F1: 0.15789473684210525 [INFO] Relation Linking P: 0.21052631578947367, R: 0.21052631578947367, micro F1: 0.21052631578947367, macro F1: 0.21052631578947367 [INFO] Type Linking P: 0.8947368421052632, R: 0.8947368421052632, micro F1: 0.8947368421052632, macro F1: 0.8947368421052632 @@ -292,9 +292,9 @@ Question 19: Which country was Bill Gates born in? [INFO] Golden Answer: [http://dbpedia.org/resource/United_States] [INFO] Predicted Answer: [http://dbpedia.org/resource/Seattle, Seattle, Washington, U.S.] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 20, P: 0.305, R: 0.368, macro F1: 0.285, macro F1*: 0.333 -[INFO] QALD Cumulative metrics, sample: 20, P: 0.405, R: 0.368, macro F1: 0.285, QALD macro F1: 0.385 -[INFO] Current time: 2041 ms, average: 4179.65 ms/ques, entity detection: 824.40 ms/ques (linking tool: 731.30 ms/ques), relation detection: 1301.95 ms/ques (similarity: 798.75 ms/ques), EDG generation: 227.20 ms/ques +[INFO] Cumulative metrics, sample: 20, P: 0.305, R: 0.368, macro F1: 0.333 +[INFO] QALD Cumulative metrics, sample: 20, P: 0.405, R: 0.368, average F1: 0.285, QALD macro F1: 0.385 +[INFO] Current question consumed time: 1982 ms, average: 3955.05 ms/ques, entity detection: 756.80 ms/ques (linking tool: 670.85 ms/ques), relation detection: 1260.05 ms/ques (similarity: 806.75 ms/ques), EDG generation: 233.40 ms/ques [INFO] Entity Linking P: 0.15, R: 0.15, micro F1: 0.15, macro F1: 0.15 [INFO] Relation Linking P: 0.2, R: 0.2, micro F1: 0.2, macro F1: 0.20000000000000004 [INFO] Type Linking P: 0.9, R: 0.9, micro F1: 0.9, macro F1: 0.9 @@ -307,9 +307,9 @@ Question 20: How many grand-children did Jacques Cousteau have? [INFO] Golden Answer: [4] [INFO] Predicted Answer: [2] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 21, P: 0.290, R: 0.350, macro F1: 0.271, macro F1*: 0.317 -[INFO] QALD Cumulative metrics, sample: 21, P: 0.385, R: 0.350, macro F1: 0.271, QALD macro F1: 0.367 -[INFO] Current time: 2068 ms, average: 4079.10 ms/ques, entity detection: 799.24 ms/ques (linking tool: 703.86 ms/ques), relation detection: 1266.57 ms/ques (similarity: 775.48 ms/ques), EDG generation: 217.10 ms/ques +[INFO] Cumulative metrics, sample: 21, P: 0.290, R: 0.350, macro F1: 0.317 +[INFO] QALD Cumulative metrics, sample: 21, P: 0.385, R: 0.350, average F1: 0.271, QALD macro F1: 0.367 +[INFO] Current question consumed time: 1961 ms, average: 3860.10 ms/ques, entity detection: 731.90 ms/ques (linking tool: 645.38 ms/ques), relation detection: 1229.05 ms/ques (similarity: 788.48 ms/ques), EDG generation: 223.00 ms/ques [INFO] Entity Linking P: 0.14285714285714285, R: 0.14285714285714285, micro F1: 0.14285714285714285, macro F1: 0.14285714285714285 [INFO] Relation Linking P: 0.19047619047619047, R: 0.19047619047619047, micro F1: 0.19047619047619047, macro F1: 0.19047619047619047 [INFO] Type Linking P: 0.9047619047619048, R: 0.9047619047619048, micro F1: 0.9047619047619048, macro F1: 0.9047619047619048 @@ -322,9 +322,9 @@ Question 21: Give me all professional skateboarders from Sweden. [INFO] Golden Answer: [http://dbpedia.org/resource/Ali_Boulala, http://dbpedia.org/resource/Tony_Magnusson] [INFO] Predicted Answer: [http://dbpedia.org/resource/S4_(Dota_player), http://dbpedia.org/resource/Team_Surprise, http://dbpedia.org/resource/2013–14_Alabama–Huntsville_Chargers_men's_ice_hockey_season, http://dbpedia.org/resource/Anne_Line_Gjersem, http://dbpedia.org/resource/Adam_Lindgren, http://dbpedia.org/resource/Kimberly_Karlsson, http://dbpedia.org/resource/Backyard_Babies, http://dbpedia.org/resource/Crucified_Barbara, http://dbpedia.org/resource/Melody_Club, http://dbpedia.org/resource/Mitch_Murder, http://dbpedia.org/resource/Mustasch, http://dbpedia.org/resource/Soilwork, http://dbpedia.org/resource/Crippled_Black_Phoenix, http://dbpedia.org/resource/Blue_Swede, http://dbpedia.org/resource/Bob_Hund, http://dbpedia.org/resource/Covenant_(band), http://dbpedia.org/resource/Icona_Pop, http://dbpedia.org/resource/Jani_Stefanovic, http://dbpedia.org/resource/Joakim_Thåström, http://dbpedia.org/resource/Miseration, http://dbpedia.org/resource/Solution_.45, http://dbpedia.org/resource/Sibel_Redžep, http://dbpedia.org/resource/Hardcore_Superstar, http://dbpedia.org/resource/Vains_of_Jenna, http://dbpedia.org/resource/Daniel_Norgren, http://dbpedia.org/resource/Gustav_Ejstes, http://dbpedia.org/resource/Markoolio, http://dbpedia.org/resource/Olof_Dreijer, http://dbpedia.org/resource/Scar_Symmetry, http://dbpedia.org/resource/Shirley_Clamp, http://dbpedia.org/resource/Sugarplum_Fairy, http://dbpedia.org/resource/The_Latin_Kings_(hip_hop_group), http://dbpedia.org/resource/Those_Dancing_Days, http://dbpedia.org/resource/Timoteij, http://dbpedia.org/resource/Armageddon_(Swedish_band), http://dbpedia.org/resource/Camilla_Henemark, http://dbpedia.org/resource/Vacuum_(band), http://dbpedia.org/resource/Baltimoore, http://dbpedia.org/resource/Ole_Öhman, http://dbpedia.org/resource/Ride_the_Sky, http://dbpedia.org/resource/Shotgun_Messiah, http://dbpedia.org/resource/Amanda_(singer), http://dbpedia.org/resource/Anniela, http://dbpedia.org/resource/Brolle, http://dbpedia.org/resource/Jaqee, http://dbpedia.org/resource/Newkid, http://dbpedia.org/resource/Velvet_(singer), http://dbpedia.org/resource/Yaiya, http://dbpedia.org/resource/Blond_(band), http://dbpedia.org/resource/Cemetary_(band), http://dbpedia.org/resource/Death_Breath, http://dbpedia.org/resource/Entombed_A.D., http://dbpedia.org/resource/Jörgen_Sandström, http://dbpedia.org/resource/Misery_Loves_Co., http://dbpedia.org/resource/Morbid_(band), http://dbpedia.org/resource/Paganizer, http://dbpedia.org/resource/Divinefire, http://dbpedia.org/resource/Narnia_(band), http://dbpedia.org/resource/Treat_(band), http://dbpedia.org/resource/XT_(band), http://dbpedia.org/resource/The_Field_(musician), http://dbpedia.org/resource/Dupont_(band), http://dbpedia.org/resource/Tarmvred, http://dbpedia.org/resource/Mapei_(musician), http://dbpedia.org/resource/Rönninge_Show_Chorus, http://dbpedia.org/resource/Fibes,_Oh_Fibes!, http://dbpedia.org/resource/Attack_(band), http://dbpedia.org/resource/Code_64, http://dbpedia.org/resource/Daybehavior, http://dbpedia.org/resource/Wasted_Penguinz, http://dbpedia.org/resource/The_Plan_(band), http://dbpedia.org/resource/Centinex, http://dbpedia.org/resource/Kid_Down, http://dbpedia.org/resource/Nine_(band), http://dbpedia.org/resource/Sounds_Like_Violence, http://dbpedia.org/resource/Last_Days_of_April, http://dbpedia.org/resource/Bestial_Mockery, http://dbpedia.org/resource/Bitch_Boys, http://dbpedia.org/resource/E.M.M.A., http://dbpedia.org/resource/Serpent_(band), http://dbpedia.org/resource/As_You_Drown, http://dbpedia.org/resource/Impious, http://dbpedia.org/resource/Afasi_&_Filthy, http://dbpedia.org/resource/Ansiktet_(band), http://dbpedia.org/resource/JTR_(band), http://dbpedia.org/resource/Vladi_Vargas, http://dbpedia.org/resource/Raymond_Strid, http://dbpedia.org/resource/Abalone_Dots, http://dbpedia.org/resource/Drängarna, http://dbpedia.org/resource/Elisa's, http://dbpedia.org/resource/Georg_Gulyás, http://dbpedia.org/resource/Gunilla_Backman, http://dbpedia.org/resource/Hanna_Lindblad, http://dbpedia.org/resource/Johan_Palm, http://dbpedia.org/resource/Kristin_Amparo, http://dbpedia.org/resource/LÉON_(Swedish_singer), http://dbpedia.org/resource/Norlie_&_KKV, http://dbpedia.org/resource/Opus_Atlantica, http://dbpedia.org/resource/Rolandz, http://dbpedia.org/resource/Supernatural_(group), http://dbpedia.org/resource/The_Moniker, http://dbpedia.org/resource/The_Playtones, http://dbpedia.org/resource/Trio_me'_Bumba, http://dbpedia.org/resource/Östen_med_Resten, http://dbpedia.org/resource/Simon_Cruz, http://dbpedia.org/resource/Setherial, http://dbpedia.org/resource/Frida_Sandén, http://dbpedia.org/resource/Max_Thornell, http://dbpedia.org/resource/Mimmi_Sandén, http://dbpedia.org/resource/Sofia_Talvik, http://dbpedia.org/resource/Last_Tribe_(Swedish_band), http://dbpedia.org/resource/Terra_Firma_(band), http://dbpedia.org/resource/Irma_Schultz_Keller, http://dbpedia.org/resource/Half_Man_(band), http://dbpedia.org/resource/Psychotic_Youth, http://dbpedia.org/resource/Sturm_Café, http://dbpedia.org/resource/Abjeez, http://dbpedia.org/resource/JLT_(John_Lindberg_Trio), http://dbpedia.org/resource/Nutmeg_(band), http://dbpedia.org/resource/The_Genuine_Fakes, http://dbpedia.org/resource/Angelica_Rylin, http://dbpedia.org/resource/Highlights_(band), http://dbpedia.org/resource/Calle_Real_(band), http://dbpedia.org/resource/Faceshift, http://dbpedia.org/resource/Monika_Hoffman, http://dbpedia.org/resource/New_Tide_Orquesta, http://dbpedia.org/resource/Polly_Tones, http://dbpedia.org/resource/Disconvenience, http://dbpedia.org/resource/Oscar_Grönberg, http://dbpedia.org/resource/Thor-Erics, http://dbpedia.org/resource/Thor_Görans, http://dbpedia.org/resource/Overworld_(band), http://dbpedia.org/resource/Lee_Kings, http://dbpedia.org/resource/Göingeflickorna, http://dbpedia.org/resource/Earl_and_Dallas, http://dbpedia.org/resource/Last_Amanda, http://dbpedia.org/resource/JEERK, http://dbpedia.org/resource/Case_Conrad, http://dbpedia.org/resource/A-Teens, http://dbpedia.org/resource/ABBA, http://dbpedia.org/resource/Ainbusk, http://dbpedia.org/resource/Alien_(band), http://dbpedia.org/resource/Blue_for_Two, http://dbpedia.org/resource/Dissection_(band), http://dbpedia.org/resource/Gemini_(Swedish_band), http://dbpedia.org/resource/Leather_Nun, http://dbpedia.org/resource/Ophthalamia, http://dbpedia.org/resource/The_Real_Group, http://dbpedia.org/resource/Erik_Segerstedt, http://dbpedia.org/resource/Manse_(DJ), http://dbpedia.org/resource/Sebastian_Ingrosso, http://dbpedia.org/resource/The_Ark_(Swedish_band), http://dbpedia.org/resource/Christian_Älvestam, http://dbpedia.org/resource/Peter_Jöback, http://dbpedia.org/resource/Agnes_Carlsson, http://dbpedia.org/resource/Anders_Fridén, http://dbpedia.org/resource/Andreas_Johnson, http://dbpedia.org/resource/David_Parland, http://dbpedia.org/resource/Johnossi, http://dbpedia.org/resource/The_Hives, http://dbpedia.org/resource/Åse_Kleveland, http://dbpedia.org/resource/Can_%22Stress%22_Canatan, http://dbpedia.org/resource/Lill-Babs, http://dbpedia.org/resource/Hans_Wärmling, http://dbpedia.org/resource/Whale_(band), http://dbpedia.org/resource/Adrian_Knight_(composer), http://dbpedia.org/resource/Andromeda_(Swedish_band), http://dbpedia.org/resource/Skyfire_(band), http://dbpedia.org/resource/Ultima_Thule_(Swedish_band), http://dbpedia.org/resource/Doris_(singer), http://dbpedia.org/resource/Eskobar, http://dbpedia.org/resource/Sambassadeur, http://dbpedia.org/resource/Fatima_Rainey, http://dbpedia.org/resource/Dionysus_(band), http://dbpedia.org/resource/Full_Force_(band), http://dbpedia.org/resource/I_Break_Horses, http://dbpedia.org/resource/The_Poodles, http://dbpedia.org/resource/Cryonic_Temple, http://dbpedia.org/resource/Sator_(band), http://dbpedia.org/resource/Erik_Nylander, http://dbpedia.org/resource/Lambretta_(band), http://dbpedia.org/resource/Spiders_(Swedish_band), http://dbpedia.org/resource/Eric_Gadd, http://dbpedia.org/resource/Burst_(band), http://dbpedia.org/resource/Maryslim, http://dbpedia.org/resource/King_of_Asgard, http://dbpedia.org/resource/Bombus_(band), http://dbpedia.org/resource/Kjell_Roos_Band, http://dbpedia.org/resource/Sophie_Rimheden, http://dbpedia.org/resource/Eternal_Oath, http://dbpedia.org/resource/Christian_Karlsson_(record_producer), http://dbpedia.org/resource/Billie_the_Vision_and_the_Dancers, http://dbpedia.org/resource/Eggstone, http://dbpedia.org/resource/Lucky_Twice, http://dbpedia.org/resource/Anders_Engbergs, http://dbpedia.org/resource/Chesty_Morgan_(band), http://dbpedia.org/resource/Eric_Young_(drummer), http://dbpedia.org/resource/Fredrik_(band), http://dbpedia.org/resource/Martin_Sweet, http://dbpedia.org/resource/Opa_(Swedish_band), http://dbpedia.org/resource/Dynazty, http://dbpedia.org/resource/Drabness, http://dbpedia.org/resource/Devian, http://dbpedia.org/resource/Evocation_(band), http://dbpedia.org/resource/Firespawn, http://dbpedia.org/resource/Zonata, http://dbpedia.org/resource/Crash_Nomada, http://dbpedia.org/resource/Dorlene_Love, http://dbpedia.org/resource/Räfven, http://dbpedia.org/resource/Calle_Kristiansson, http://dbpedia.org/resource/Noice, http://dbpedia.org/resource/The_Similou, http://dbpedia.org/resource/Elin_Bergman, http://dbpedia.org/resource/Fernandoz, http://dbpedia.org/resource/Stensons, http://dbpedia.org/resource/Elisa_Lindström, http://dbpedia.org/resource/Jonas_Snäckmark, http://dbpedia.org/resource/Dan_Lissvik, http://dbpedia.org/resource/Magnus_Lidehäll, http://dbpedia.org/resource/The_Confusions, http://dbpedia.org/resource/Friday_Bridge, http://dbpedia.org/resource/Bonafide_(band), http://dbpedia.org/resource/Erika_Selin, http://dbpedia.org/resource/Inja_(band), http://dbpedia.org/resource/The_Fallen_Empires, http://dbpedia.org/resource/Arsenite_(band), http://dbpedia.org/resource/Melanie_is_demented, http://dbpedia.org/resource/Dökött, http://dbpedia.org/resource/Kindbergs, http://dbpedia.org/resource/Donnez, http://dbpedia.org/resource/Dial_M_for_Murder!, http://dbpedia.org/resource/Asha_Ali, http://dbpedia.org/resource/System_Shock_(band), http://dbpedia.org/resource/Adrian_Lux, http://dbpedia.org/resource/Anaïs_Lameche, http://dbpedia.org/resource/Army_of_Lovers, http://dbpedia.org/resource/Imperial_State_Electric, http://dbpedia.org/resource/Jay_Orpin, http://dbpedia.org/resource/The_Hellacopters, http://dbpedia.org/resource/The_Hydromatics, http://dbpedia.org/resource/The_Solution_(band), http://dbpedia.org/resource/The_Spotnicks, http://dbpedia.org/resource/Avicii, http://dbpedia.org/resource/Per_Magnusson, http://dbpedia.org/resource/Yinka_Davies, http://dbpedia.org/resource/Ace_of_Base, http://dbpedia.org/resource/Linn_Berggren, http://dbpedia.org/resource/The_Cardigans, http://dbpedia.org/resource/Tove_Lo, http://dbpedia.org/resource/Dream_Evil, http://dbpedia.org/resource/Eric_Prydz, http://dbpedia.org/resource/Erik_Grönwall, http://dbpedia.org/resource/Funeral_Mist, http://dbpedia.org/resource/Hypocrisy_(band), http://dbpedia.org/resource/Pernilla_Wahlgren, http://dbpedia.org/resource/Deathstars, http://dbpedia.org/resource/Kleerup, http://dbpedia.org/resource/Lindemann_(band), http://dbpedia.org/resource/Meja, http://dbpedia.org/resource/Teddybears, http://dbpedia.org/resource/Bloodbath, http://dbpedia.org/resource/Fredrik_Nordström, http://dbpedia.org/resource/Fredrik_Åkesson, http://dbpedia.org/resource/Per_Wiberg, http://dbpedia.org/resource/Machinae_Supremacy, http://dbpedia.org/resource/Emperor_Magus_Caligula, http://dbpedia.org/resource/Falconer_(band), http://dbpedia.org/resource/Mithotyn, http://dbpedia.org/resource/Månegarm, http://dbpedia.org/resource/Nocturnal_Rites, http://dbpedia.org/resource/Labyrint_(band), http://dbpedia.org/resource/Lazee, http://dbpedia.org/resource/ALPHA_60, http://dbpedia.org/resource/Mister_Kite, http://dbpedia.org/resource/Talisman_(band), http://dbpedia.org/resource/Aerial_(Swedish_band), http://dbpedia.org/resource/Acid_House_Kings, http://dbpedia.org/resource/Bloodbound, http://dbpedia.org/resource/Count_Raven, http://dbpedia.org/resource/Majestic_(band), http://dbpedia.org/resource/Morifade, http://dbpedia.org/resource/Peter_Bjorn_and_John, http://dbpedia.org/resource/Steel_Attack, http://dbpedia.org/resource/Twice_a_Man, http://dbpedia.org/resource/Mefisto_(band), http://dbpedia.org/resource/Alf_Svensson_(guitarist), http://dbpedia.org/resource/Alice_in_Videoland, http://dbpedia.org/resource/Amanda_Mair, http://dbpedia.org/resource/Antiloop, http://dbpedia.org/resource/Arcana_(Swedish_band), http://dbpedia.org/resource/Astral_Doors, http://dbpedia.org/resource/Basic_Element_(music_group), http://dbpedia.org/resource/Caramell, http://dbpedia.org/resource/Carnal_Forge, http://dbpedia.org/resource/Club_8, http://dbpedia.org/resource/Dawn_(metal_band), http://dbpedia.org/resource/Den_Fule, http://dbpedia.org/resource/Diablo_Swing_Orchestra, http://dbpedia.org/resource/Dimension_Zero_(Swedish_band), http://dbpedia.org/resource/Division_of_Laura_Lee, http://dbpedia.org/resource/Dragonland, http://dbpedia.org/resource/Eldkvarn_(band), http://dbpedia.org/resource/Engel_(band), http://dbpedia.org/resource/Friska_Viljor, http://dbpedia.org/resource/Gert_Jonnys, http://dbpedia.org/resource/Henrik_Danhage, http://dbpedia.org/resource/Hoven_Droven, http://dbpedia.org/resource/I'm_from_Barcelona, http://dbpedia.org/resource/Joel_Alme, http://dbpedia.org/resource/Jonas_Ekdahl, http://dbpedia.org/resource/Khoma, http://dbpedia.org/resource/Lili_&_Susie, http://dbpedia.org/resource/Linda_Bengtzing, http://dbpedia.org/resource/Lisa_Ekdahl, http://dbpedia.org/resource/Lisa_Nilsson, http://dbpedia.org/resource/Love_Is_All_(band), http://dbpedia.org/resource/Lowe_(band), http://dbpedia.org/resource/Marit_Bergman, http://dbpedia.org/resource/Martin_Larsson_(guitarist), http://dbpedia.org/resource/Moonlight_Agony, http://dbpedia.org/resource/Movits!, http://dbpedia.org/resource/NEXX, http://dbpedia.org/resource/Nationalteatern, http://dbpedia.org/resource/Oh_Laura, http://dbpedia.org/resource/Pelle_Carlberg, http://dbpedia.org/resource/Sandy_Mouche, http://dbpedia.org/resource/Secret_Service_(band), http://dbpedia.org/resource/Silent_Border, http://dbpedia.org/resource/Suburban_Kids_with_Biblical_Names, http://dbpedia.org/resource/Terror_2000, http://dbpedia.org/resource/The_Concretes, http://dbpedia.org/resource/Truckfighters, http://dbpedia.org/resource/Violent_Silence, http://dbpedia.org/resource/Whipped_Cream_(band), http://dbpedia.org/resource/Face_Down_(band), http://dbpedia.org/resource/J._C._Schütz, http://dbpedia.org/resource/John_Zwetsloot, http://dbpedia.org/resource/Samir_&_Viktor, http://dbpedia.org/resource/Sarah_Dawn_Finer, http://dbpedia.org/resource/Seance_(band), http://dbpedia.org/resource/Stephen_Simmonds, http://dbpedia.org/resource/The_Resistance_(Swedish_band), http://dbpedia.org/resource/Laila_Bagge_Wahlgren, http://dbpedia.org/resource/Visceral_Bleeding, http://dbpedia.org/resource/Caroline_af_Ugglas, http://dbpedia.org/resource/Marching_Band_(band), http://dbpedia.org/resource/Moonbabies, http://dbpedia.org/resource/Renegade_Five, http://dbpedia.org/resource/Royal_Republic, http://dbpedia.org/resource/Surrounded_(band), http://dbpedia.org/resource/Weeping_Willows, http://dbpedia.org/resource/Gas_Lipstick, http://dbpedia.org/resource/Draconian_(band), http://dbpedia.org/resource/Satanic_Slaughter, http://dbpedia.org/resource/Taken_by_Trees, http://dbpedia.org/resource/Stefan_Weinerhall, http://dbpedia.org/resource/Figge_Boström, http://dbpedia.org/resource/Pantokrator_(band), http://dbpedia.org/resource/Dominion_(Swedish_band), http://dbpedia.org/resource/Rebound!, http://dbpedia.org/resource/Comecon_(band), http://dbpedia.org/resource/Antique_(band), http://dbpedia.org/resource/Johan_Lagerlöf, http://dbpedia.org/resource/Killer_Bee_(band), http://dbpedia.org/resource/Freda', http://dbpedia.org/resource/Kayser_(band), http://dbpedia.org/resource/Art_Metal_(band), http://dbpedia.org/resource/Top_Cats, http://dbpedia.org/resource/The_Murder_of_My_Sweet, http://dbpedia.org/resource/Qoph_(band), http://dbpedia.org/resource/Dolly_Style, http://dbpedia.org/resource/Mats_Bergmans, http://dbpedia.org/resource/GES_(band), http://dbpedia.org/resource/Decadence_(band), http://dbpedia.org/resource/Kajsa_Grytt, http://dbpedia.org/resource/Stockholm_Lisboa_Project, http://dbpedia.org/resource/Dundertåget, http://dbpedia.org/resource/Miio, http://dbpedia.org/resource/Det_Vackra_Livet, http://dbpedia.org/resource/The_Headlines, http://dbpedia.org/resource/Stellan_Wahlström_Drift_Band, http://dbpedia.org/resource/Willez, http://dbpedia.org/resource/Oh_My!_(band), http://dbpedia.org/resource/Ditt_Inre, http://dbpedia.org/resource/The_Borderline_Saints, http://dbpedia.org/resource/7_Samurai_(electronic_musicians), http://dbpedia.org/resource/Chords_(musician), http://dbpedia.org/resource/Doctors_&_Dealers, http://dbpedia.org/resource/The_Majority_Says, http://dbpedia.org/resource/Abruptum, http://dbpedia.org/resource/Arckanum, http://dbpedia.org/resource/Craft_(band), http://dbpedia.org/resource/Crimson_Moonlight, http://dbpedia.org/resource/In_Battle, http://dbpedia.org/resource/Nifelheim, http://dbpedia.org/resource/Ofermod, http://dbpedia.org/resource/Ondskapt, http://dbpedia.org/resource/Rednex, http://dbpedia.org/resource/Tiamat_(band), http://dbpedia.org/resource/Watain, http://dbpedia.org/resource/Avatar_(Swedish_band), http://dbpedia.org/resource/Axwell, http://dbpedia.org/resource/Carnage_(band), http://dbpedia.org/resource/Dark_Tranquillity, http://dbpedia.org/resource/God_Macabre, http://dbpedia.org/resource/In_Flames, http://dbpedia.org/resource/Danny_Saucedo, http://dbpedia.org/resource/Fission_(band), http://dbpedia.org/resource/Fredrik_Thordendal, http://dbpedia.org/resource/Freestyle_(Swedish_band), http://dbpedia.org/resource/Pain_of_Salvation, http://dbpedia.org/resource/General_Surgery_(band), http://dbpedia.org/resource/Regurgitate_(band), http://dbpedia.org/resource/Swedish_House_Mafia, http://dbpedia.org/resource/Dave_Lepard, http://dbpedia.org/resource/Felicia_Brandström, http://dbpedia.org/resource/Michael_Amott, http://dbpedia.org/resource/Bo_Andersson_(businessman), http://dbpedia.org/resource/Nina_Persson, http://dbpedia.org/resource/Thyrfing, http://dbpedia.org/resource/Carl_Johan_Grimmark, http://dbpedia.org/resource/Clockwise_(band), http://dbpedia.org/resource/Eric_Saade, http://dbpedia.org/resource/Last_Autumn's_Dream, http://dbpedia.org/resource/Pacific!, http://dbpedia.org/resource/Palle_Danielsson, http://dbpedia.org/resource/She_(band), http://dbpedia.org/resource/Anders_Jormin, http://dbpedia.org/resource/Carola_Häggkvist, http://dbpedia.org/resource/Sabaton_(band), http://dbpedia.org/resource/Grave_(band), http://dbpedia.org/resource/Brainpool_(band), http://dbpedia.org/resource/Christoffer_Lundquist, http://dbpedia.org/resource/Helena_Josefsson, http://dbpedia.org/resource/The_Lonely_Boys, http://dbpedia.org/resource/Anti_Cimex, http://dbpedia.org/resource/Beardfish_(band), http://dbpedia.org/resource/Disfear, http://dbpedia.org/resource/Heavy_Load_(band), http://dbpedia.org/resource/Mob_47, http://dbpedia.org/resource/Peter_Wildoer, http://dbpedia.org/resource/Skitsystem, http://dbpedia.org/resource/Alyson_Avenue, http://dbpedia.org/resource/Linus_Svenning, http://dbpedia.org/resource/Play_(group), http://dbpedia.org/resource/Brainbombs, http://dbpedia.org/resource/Breach_(band), http://dbpedia.org/resource/Ceremonial_Oath, http://dbpedia.org/resource/Dhani_Lennevald, http://dbpedia.org/resource/Drain_STH, http://dbpedia.org/resource/Gardenian, http://dbpedia.org/resource/Herreys, http://dbpedia.org/resource/Johan_Larsson_(musician), http://dbpedia.org/resource/Leila_K, http://dbpedia.org/resource/Lost_Horizon_(band), http://dbpedia.org/resource/No_Fun_at_All, http://dbpedia.org/resource/October_Tide, http://dbpedia.org/resource/Persuader, http://dbpedia.org/resource/Spiritual_Beggars, http://dbpedia.org/resource/The_Crown_(band), http://dbpedia.org/resource/Witchery, http://dbpedia.org/resource/Leif_Elggren, http://dbpedia.org/resource/Pouppée_Fabrikk, http://dbpedia.org/resource/Fame_(duo), http://dbpedia.org/resource/Kristofer_Åström, http://dbpedia.org/resource/Sonic_Syndicate, http://dbpedia.org/resource/Jenny_Silver, http://dbpedia.org/resource/Deutsch_Nepal, http://dbpedia.org/resource/Enforcer_(band), http://dbpedia.org/resource/Overload_(Swedish_band), http://dbpedia.org/resource/Raison_d'être_(band), http://dbpedia.org/resource/Solar_Fields, http://dbpedia.org/resource/Adept_(band), http://dbpedia.org/resource/Sherissa, http://dbpedia.org/resource/Towa_Carson, http://dbpedia.org/resource/Technicoloured_Roses, http://dbpedia.org/resource/Sten_&_Stanley, http://dbpedia.org/resource/Envelopes_(band), http://dbpedia.org/resource/Timeless_Miracle, http://dbpedia.org/resource/Arise_(band), http://dbpedia.org/resource/Chiquita_(band), http://dbpedia.org/resource/Infernal_(Swedish_band), http://dbpedia.org/resource/Hearts_No_Static, http://dbpedia.org/resource/Henrik_Janson, http://dbpedia.org/resource/Beautiful_Sin, http://dbpedia.org/resource/Crystal_Eyes, http://dbpedia.org/resource/The_Nomads_(band), http://dbpedia.org/resource/The_Royal_Concept, http://dbpedia.org/resource/Murlyn_Music_Group, http://dbpedia.org/resource/Selfmindead, http://dbpedia.org/resource/Amber_Oak, http://dbpedia.org/resource/Dead_Man_(band), http://dbpedia.org/resource/Hollow_(band), http://dbpedia.org/resource/Moon_Safari_(band), http://dbpedia.org/resource/The_Duskfall, http://dbpedia.org/resource/Simbi_(band), http://dbpedia.org/resource/Cool_James_and_Black_Teacher, http://dbpedia.org/resource/Anne-Lie_Rydé, http://dbpedia.org/resource/Johanna_Lind, http://dbpedia.org/resource/Twin_(production_team), http://dbpedia.org/resource/Jesper_Dahlbäck, http://dbpedia.org/resource/The_Funeral_Orchestra, http://dbpedia.org/resource/Death_Organ, http://dbpedia.org/resource/The_Mushroom_River_Band, http://dbpedia.org/resource/Victims_(band), http://dbpedia.org/resource/Kikki,_Bettan_&_Lotta, http://dbpedia.org/resource/Satariel_(band), http://dbpedia.org/resource/Sista_Mannen_På_Jorden, http://dbpedia.org/resource/Loch_Vostok, http://dbpedia.org/resource/Lasse_Stefanz, http://dbpedia.org/resource/Snowstorm_(band), http://dbpedia.org/resource/Dan_Berglund, http://dbpedia.org/resource/Human_Clay_(band), http://dbpedia.org/resource/Humanimal_(band), http://dbpedia.org/resource/My_Brother_The_Wind, http://dbpedia.org/resource/Youngblood_(band), http://dbpedia.org/resource/No_Science, http://dbpedia.org/resource/Amanda_Bergman, http://dbpedia.org/resource/Paus_(band), http://dbpedia.org/resource/Loosegoats, http://dbpedia.org/resource/Sture_Nordin, http://dbpedia.org/resource/Cloudberry_Jam, http://dbpedia.org/resource/Hell_on_Wheels_(band), http://dbpedia.org/resource/Pauline_Kamusewu, http://dbpedia.org/resource/Lowrider_(Swedish_band), http://dbpedia.org/resource/Nina_Rochelle, http://dbpedia.org/resource/Awa_Santesson-Sey, http://dbpedia.org/resource/Stonewall_Noise_Orchestra, http://dbpedia.org/resource/Immanu_El, http://dbpedia.org/resource/Komplett_Arnold, http://dbpedia.org/resource/Den_svenska_björnstammen, http://dbpedia.org/resource/Broken_Door, http://dbpedia.org/resource/Millas_Mirakel, http://dbpedia.org/resource/Alfstarz, http://dbpedia.org/resource/Skrömta, http://dbpedia.org/resource/Coerced_Existence, http://dbpedia.org/resource/Eurotix, http://dbpedia.org/resource/Istapp, http://dbpedia.org/resource/Broder_Daniel, http://dbpedia.org/resource/Christian_Liljegren, http://dbpedia.org/resource/Gyllene_Tider, http://dbpedia.org/resource/Jens_Lekman, http://dbpedia.org/resource/Ulrik_Munther, http://dbpedia.org/resource/Europe_(band), http://dbpedia.org/resource/Steve_Angello, http://dbpedia.org/resource/Witchcraft_(band), http://dbpedia.org/resource/Bloodshy_&_Avant, http://dbpedia.org/resource/Kite_(band), http://dbpedia.org/resource/KSMB_(band), http://dbpedia.org/resource/Page_(Swedish_band), http://dbpedia.org/resource/Emma_Härdelin, http://dbpedia.org/resource/Caesars_(band), http://dbpedia.org/resource/Date_(band), http://dbpedia.org/resource/Samla_Mammas_Manna, http://dbpedia.org/resource/Seventh_Wonder, http://dbpedia.org/resource/Birgit_Ridderstedt, http://dbpedia.org/resource/Carbon_Based_Lifeforms, http://dbpedia.org/resource/Za_Frûmi, http://dbpedia.org/resource/Shout_Out_Louds, http://dbpedia.org/resource/Swingfly, http://dbpedia.org/resource/Leviticus_(band), http://dbpedia.org/resource/Maher_Zain, http://dbpedia.org/resource/Robert_Wells_(composer), http://dbpedia.org/resource/Triakel, http://dbpedia.org/resource/Electric_Boys, http://dbpedia.org/resource/Maria_Rolf, http://dbpedia.org/resource/Union_Carbide_Productions, http://dbpedia.org/resource/Anekdoten, http://dbpedia.org/resource/Beseech, http://dbpedia.org/resource/Bubbles_(band), http://dbpedia.org/resource/Cazzette, http://dbpedia.org/resource/Demons_(band), http://dbpedia.org/resource/Deranged_(band), http://dbpedia.org/resource/Goto80, http://dbpedia.org/resource/Hel_(band), http://dbpedia.org/resource/In_Mourning_(band), http://dbpedia.org/resource/Infinite_Mass, http://dbpedia.org/resource/Ison_&_Fille, http://dbpedia.org/resource/John_Dahlbäck, http://dbpedia.org/resource/Kaamos_(Swedish_band), http://dbpedia.org/resource/Koop_(band), http://dbpedia.org/resource/Landberk, http://dbpedia.org/resource/Looptroop_Rockers, http://dbpedia.org/resource/Maskinen, http://dbpedia.org/resource/Minilogue, http://dbpedia.org/resource/Nause, http://dbpedia.org/resource/Necrophobic, http://dbpedia.org/resource/Neverstore, http://dbpedia.org/resource/One_Man_Army_and_the_Undead_Quartet, http://dbpedia.org/resource/Paatos, http://dbpedia.org/resource/Raubtier, http://dbpedia.org/resource/Rebecca_&_Fiona, http://dbpedia.org/resource/Roffe_Ruff, http://dbpedia.org/resource/Sally_Shapiro, http://dbpedia.org/resource/Smile.dk, http://dbpedia.org/resource/Snook_(band), http://dbpedia.org/resource/Soundism, http://dbpedia.org/resource/Takida, http://dbpedia.org/resource/The_Fooo_Conspiracy, http://dbpedia.org/resource/The_Radio_Dept., http://dbpedia.org/resource/Timo_Räisänen, http://dbpedia.org/resource/Valkyrja, http://dbpedia.org/resource/Vomitory_(band), http://dbpedia.org/resource/Krister_Linder, http://dbpedia.org/resource/Zinny_J._Zan, http://dbpedia.org/resource/Ola_&_the_Janglers, http://dbpedia.org/resource/Conny_Nimmersjö, http://dbpedia.org/resource/Niklas_Hellberg, http://dbpedia.org/resource/Pelle_Ossler, http://dbpedia.org/resource/Sällskapet_(band), http://dbpedia.org/resource/Admonish_(band), http://dbpedia.org/resource/Endless_Shame, http://dbpedia.org/resource/Py_Bäckman, http://dbpedia.org/resource/Triad_(band), http://dbpedia.org/resource/Kristian_Wåhlin, http://dbpedia.org/resource/Raymond_&_Maria, http://dbpedia.org/resource/The_Most_(Swedish_band), http://dbpedia.org/resource/Axenstar, http://dbpedia.org/resource/Small_Town_Singers, http://dbpedia.org/resource/Johan_Åberg, http://dbpedia.org/resource/Agent_Side_Grinder, http://dbpedia.org/resource/Colony_5, http://dbpedia.org/resource/Elegant_Machinery, http://dbpedia.org/resource/First_Floor_Power, http://dbpedia.org/resource/Neon_Rose, http://dbpedia.org/resource/Tungevaag_&_Raaban, http://dbpedia.org/resource/Familjen, http://dbpedia.org/resource/Tomas_Andersson, http://dbpedia.org/resource/Caroline_Wennergren, http://dbpedia.org/resource/Sara_Löfgren, http://dbpedia.org/resource/The_Wallstones, http://dbpedia.org/resource/Chattanooga_(band), http://dbpedia.org/resource/Standfast, http://dbpedia.org/resource/Introduction_(rock_band), http://dbpedia.org/resource/Calaisa, http://dbpedia.org/resource/Karjalan_sissit, http://dbpedia.org/resource/The_Göteborg_String_Theory, http://dbpedia.org/resource/Magnus_Öström, http://dbpedia.org/resource/Albatraoz, http://dbpedia.org/resource/Owe_Lingvall, http://dbpedia.org/resource/Silent_Images, http://dbpedia.org/resource/Leif_Bloms, http://dbpedia.org/resource/Atrium_Carceri, http://dbpedia.org/resource/Featherstone_(rock_band), http://dbpedia.org/resource/Rising_Shadows, http://dbpedia.org/resource/Diamond_Dogs_(band), http://dbpedia.org/resource/Mindless_Sinner, http://dbpedia.org/resource/What's_Up!, http://dbpedia.org/resource/Hans_Edler, http://dbpedia.org/resource/Siena_Root, http://dbpedia.org/resource/Twelvestep, http://dbpedia.org/resource/Waves_under_Water, http://dbpedia.org/resource/EJ_(singer), http://dbpedia.org/resource/The_Kooks_(Swedish_band), http://dbpedia.org/resource/MAASAI, http://dbpedia.org/resource/The_Moth_Gatherer, http://dbpedia.org/resource/Blodsrit, http://dbpedia.org/resource/Jenny_Saléns, http://dbpedia.org/resource/Tråd, http://dbpedia.org/resource/Maini_Sorri, http://dbpedia.org/resource/My_heart_is_a_metronome, http://dbpedia.org/resource/El_Perro_del_Mar, http://dbpedia.org/resource/Evergrey, http://dbpedia.org/resource/HammerFall, http://dbpedia.org/resource/José_González_(singer), http://dbpedia.org/resource/Studio_(band), http://dbpedia.org/resource/Supergroupies, http://dbpedia.org/resource/The_Soundtrack_of_Our_Lives, http://dbpedia.org/resource/The_Tough_Alliance, http://dbpedia.org/resource/Erling_Wicklund, http://dbpedia.org/resource/Papa_Dee, http://dbpedia.org/resource/Refused, http://dbpedia.org/resource/Svenska_Akademien_(band), http://dbpedia.org/resource/The_(International)_Noise_Conspiracy, http://dbpedia.org/resource/Amanda_Jenssen, http://dbpedia.org/resource/Dominika_Peczynski, http://dbpedia.org/resource/Ilya_Salmanzadeh, http://dbpedia.org/resource/Tove_Styrke, http://dbpedia.org/resource/Christopher_Amott, http://dbpedia.org/resource/Hoola_Bandoola_Band, http://dbpedia.org/resource/Peter_Dolving, http://dbpedia.org/resource/Jean-Pierre_Barda, http://dbpedia.org/resource/Jonas_Björler, http://dbpedia.org/resource/Mohombi, http://dbpedia.org/resource/Nihilist_(band), http://dbpedia.org/resource/Krux, http://dbpedia.org/resource/Grizfolk, http://dbpedia.org/resource/Tommy_Sparks, http://dbpedia.org/resource/Da_Buzz, http://dbpedia.org/resource/Frida_Hyvönen, http://dbpedia.org/resource/Grönwalls, http://dbpedia.org/resource/Lina_Hedlund, http://dbpedia.org/resource/Nicolai_Dunger, http://dbpedia.org/resource/Sahara_Hotnights, http://dbpedia.org/resource/The_Wannadies, http://dbpedia.org/resource/Vikingarna_(band), http://dbpedia.org/resource/Wilmer_X, http://dbpedia.org/resource/Apostasy_(band), http://dbpedia.org/resource/Dea_Norberg, http://dbpedia.org/resource/Andreas_Axelsson, http://dbpedia.org/resource/Dozer, http://dbpedia.org/resource/Easy_Action_(band), http://dbpedia.org/resource/Pan.Thy.Monium, http://dbpedia.org/resource/Tages_(band), http://dbpedia.org/resource/The_Shanes, http://dbpedia.org/resource/Rigmor_Gustafsson, http://dbpedia.org/resource/Dag_Vag, http://dbpedia.org/resource/Sanne_Karlsson, http://dbpedia.org/resource/By_Night, http://dbpedia.org/resource/Demonoid_(band), http://dbpedia.org/resource/Separation_(band), http://dbpedia.org/resource/Style_(Swedish_band), http://dbpedia.org/resource/The_Same, http://dbpedia.org/resource/Angel_(Swedish_band), http://dbpedia.org/resource/Nordman, http://dbpedia.org/resource/Solid_Base, http://dbpedia.org/resource/The_Storyteller_(band), http://dbpedia.org/resource/Galantis, http://dbpedia.org/resource/John_Martin_(singer), http://dbpedia.org/resource/Max_Elto, http://dbpedia.org/resource/CirKus, http://dbpedia.org/resource/Shine_(project), http://dbpedia.org/resource/All_Ends, http://dbpedia.org/resource/Abhinanda, http://dbpedia.org/resource/Invasionen, http://dbpedia.org/resource/Star_Pilots, http://dbpedia.org/resource/Team_Rockit_(musical_project), http://dbpedia.org/resource/It's_Alive_(band), http://dbpedia.org/resource/Witherscape, http://dbpedia.org/resource/Isole_(band), http://dbpedia.org/resource/Soils_of_Fate, http://dbpedia.org/resource/Dayeene, http://dbpedia.org/resource/Autopsy_Torment, http://dbpedia.org/resource/Deportees_(band), http://dbpedia.org/resource/Godgory, http://dbpedia.org/resource/Amaran, http://dbpedia.org/resource/Ingemar_Backman, http://dbpedia.org/resource/Supreme_Majesty, http://dbpedia.org/resource/Style_of_Eye, http://dbpedia.org/resource/Kobojsarna, http://dbpedia.org/resource/ChansoNoir, http://dbpedia.org/resource/Marigold_(band), http://dbpedia.org/resource/Samuelsons, http://dbpedia.org/resource/Andrea_Myrander, http://dbpedia.org/resource/Revl9n, http://dbpedia.org/resource/Abramis_Brama_(band), http://dbpedia.org/resource/Mike_Perry_(musician), http://dbpedia.org/resource/William_C._Woxlin, http://dbpedia.org/resource/Virna_Lindt, http://dbpedia.org/resource/Sheri_Nowrozi, http://dbpedia.org/resource/Ratata_(band), http://dbpedia.org/resource/Audionom, http://dbpedia.org/resource/Desiderii_Marginis, http://dbpedia.org/resource/Kammarheit, http://dbpedia.org/resource/Lars_Eriksson_(musician), http://dbpedia.org/resource/Minnah_Karlsson, http://dbpedia.org/resource/Sarek_(band), http://dbpedia.org/resource/NoNoNo_(band), http://dbpedia.org/resource/Gustaf_Spetz, http://dbpedia.org/resource/Armagedda, http://dbpedia.org/resource/Charizma_(band), http://dbpedia.org/resource/Supersci, http://dbpedia.org/resource/Universal_Poplab, http://dbpedia.org/resource/Helene_&_gänget, http://dbpedia.org/resource/Country_Road_(band), http://dbpedia.org/resource/Him_Kerosene, http://dbpedia.org/resource/Mimikry, http://dbpedia.org/resource/Be_The_Bear, http://dbpedia.org/resource/Despite_(band), http://dbpedia.org/resource/Melpo_Mene, http://dbpedia.org/resource/Disarm_(band), http://dbpedia.org/resource/Per-Olov_Kindgren, http://dbpedia.org/resource/Rickard_Söderberg, http://dbpedia.org/resource/Le_Kid, http://dbpedia.org/resource/De_Van_(band), http://dbpedia.org/resource/Pools_orkester, http://dbpedia.org/resource/The_Amplifetes, http://dbpedia.org/resource/Tikkle_Me, http://dbpedia.org/resource/Baby_Alice, http://dbpedia.org/resource/Intohimo, http://dbpedia.org/resource/Laser_Inc, http://dbpedia.org/resource/Sir_Reg, http://dbpedia.org/resource/De_Vet_Du, http://dbpedia.org/resource/Midvinter, http://dbpedia.org/resource/Jannez, http://dbpedia.org/resource/Sannex, http://dbpedia.org/resource/Busungarna, http://dbpedia.org/resource/Wahlströms, http://dbpedia.org/resource/Curt_Görans, http://dbpedia.org/resource/Trees_of_Eternity, http://dbpedia.org/resource/Diztord, http://dbpedia.org/resource/Eagle-Eye_Cherry, http://dbpedia.org/resource/Jenny_Berggren, http://dbpedia.org/resource/Nanne_Grönvall, http://dbpedia.org/resource/Veronica_Maggio, http://dbpedia.org/resource/Cronian, http://dbpedia.org/resource/Diabolique_(band), http://dbpedia.org/resource/Dr._Alban, http://dbpedia.org/resource/Hearse_(band), http://dbpedia.org/resource/Hello_Saferide, http://dbpedia.org/resource/NonExist, http://dbpedia.org/resource/Alcazar_(band), http://dbpedia.org/resource/Amy_Diamond, http://dbpedia.org/resource/Fredrik_Johansson_(musician), http://dbpedia.org/resource/Jens_Kidman, http://dbpedia.org/resource/Mikael_Stanne, http://dbpedia.org/resource/Roxette, http://dbpedia.org/resource/Yung_Lean, http://dbpedia.org/resource/The_Flower_Kings, http://dbpedia.org/resource/S.P.O.C.K, http://dbpedia.org/resource/Christer_Lindarw, http://dbpedia.org/resource/Dia_Psalma, http://dbpedia.org/resource/Jumper_(band), http://dbpedia.org/resource/Sejr_Volmer-Sørensen, http://dbpedia.org/resource/Karmakanic, http://dbpedia.org/resource/Martinez_(band), http://dbpedia.org/resource/Carolina_Liar, http://dbpedia.org/resource/Dispatched, http://dbpedia.org/resource/Thunder_Express, http://dbpedia.org/resource/Forbes_(band), http://dbpedia.org/resource/Malta_(band), http://dbpedia.org/resource/Slam_Creepers, http://dbpedia.org/resource/Street_Fighting_Man_(band), http://dbpedia.org/resource/Sunblock_(band), http://dbpedia.org/resource/Bjorn_Englen, http://dbpedia.org/resource/Bombshell_Rocks, http://dbpedia.org/resource/Fireside_(band), http://dbpedia.org/resource/Randy_(band), http://dbpedia.org/resource/Satanic_Surfers, http://dbpedia.org/resource/Tad_Morose, http://dbpedia.org/resource/Arvingarna, http://dbpedia.org/resource/Family_Four, http://dbpedia.org/resource/Afro-dite, http://dbpedia.org/resource/Memento_Mori_(band), http://dbpedia.org/resource/Notre_Dame_(band), http://dbpedia.org/resource/Ribspreader, http://dbpedia.org/resource/Sahlene, http://dbpedia.org/resource/Time_Requiem, http://dbpedia.org/resource/War_(Swedish_band), http://dbpedia.org/resource/59_Times_the_Pain, http://dbpedia.org/resource/Drifters_(Swedish_band), http://dbpedia.org/resource/Silverbullit, http://dbpedia.org/resource/Graveyard_(band), http://dbpedia.org/resource/Yummie, http://dbpedia.org/resource/Idha, http://dbpedia.org/resource/Köttgrottorna, http://dbpedia.org/resource/Nordvargr, http://dbpedia.org/resource/JJ_(Swedish_band), http://dbpedia.org/resource/Sebastian_Karlsson_(singer), http://dbpedia.org/resource/Avalon_(Swedish_group), http://dbpedia.org/resource/Marie_Lindberg_(singer), http://dbpedia.org/resource/Jimmy_Jansson, http://dbpedia.org/resource/Amanda_Fondell, http://dbpedia.org/resource/Daniel_Lindström, http://dbpedia.org/resource/Jay_Smith_(singer), http://dbpedia.org/resource/Markus_Fagervall, http://dbpedia.org/resource/The_Bitter_Twins, http://dbpedia.org/resource/Faith_(band), http://dbpedia.org/resource/Annika_Kjærgaard, http://dbpedia.org/resource/BWO_(band), http://dbpedia.org/resource/Therese_Merkel, http://dbpedia.org/resource/Dogge_Doggelito, http://dbpedia.org/resource/Svenne_and_Lotta, http://dbpedia.org/resource/Nina_Söderquist, http://dbpedia.org/resource/Peter_Tuthill, http://dbpedia.org/resource/Ablaze_My_Sorrow, http://dbpedia.org/resource/Anders_Wendin, http://dbpedia.org/resource/Elin_Lanto, http://dbpedia.org/resource/Flamingokvintetten, http://dbpedia.org/resource/Greenleaf_(band), http://dbpedia.org/resource/Josef_Johansson, http://dbpedia.org/resource/Laakso_(band), http://dbpedia.org/resource/Mattias_Andréasson, http://dbpedia.org/resource/Ragnarök_(Swedish_band), http://dbpedia.org/resource/SALT_(quartet), http://dbpedia.org/resource/The_Attic_(band), http://dbpedia.org/resource/The_Grand_Opening, http://dbpedia.org/resource/Therese_Grankvist, http://dbpedia.org/resource/Twilight_Force, http://dbpedia.org/resource/Yaki-Da, http://dbpedia.org/resource/Quiz_&_Larossi, http://dbpedia.org/resource/Peter_London, http://dbpedia.org/resource/Ebony_Tears, http://dbpedia.org/resource/Demon_Cleaner_(band), http://dbpedia.org/resource/Theory_in_Practice, http://dbpedia.org/resource/Schytts, http://dbpedia.org/resource/Audiovision, http://dbpedia.org/resource/Jonathan_Fritzén, http://dbpedia.org/resource/Lorentz_&_Sakarias, http://dbpedia.org/resource/Majestic_Vanguard, http://dbpedia.org/resource/Sanctifica, http://dbpedia.org/resource/Shadows_of_Paragon, http://dbpedia.org/resource/Skald_in_Veum, http://dbpedia.org/resource/Jacob_Truedson_Demitz, http://dbpedia.org/resource/The_Night_Flight_Orchestra, http://dbpedia.org/resource/Hästpojken, http://dbpedia.org/resource/Rebecca_Zadig, http://dbpedia.org/resource/Katarina_Lilja, http://dbpedia.org/resource/Kishti_Tomita, http://dbpedia.org/resource/Wizex, http://dbpedia.org/resource/Bibleblack, http://dbpedia.org/resource/Tiger_Bell, http://dbpedia.org/resource/Lights_&_Motion, http://dbpedia.org/resource/Will_Gambola_Sing, http://dbpedia.org/resource/ZooL, http://dbpedia.org/resource/The_Honeydrips, http://dbpedia.org/resource/Walk_on_Water_(band), http://dbpedia.org/resource/We_Are_Serenades, http://dbpedia.org/resource/Factory_(band), http://dbpedia.org/resource/State_of_Drama, http://dbpedia.org/resource/Janne_Stark, http://dbpedia.org/resource/Morlocks_(band), http://dbpedia.org/resource/Berth_Idoffs, http://dbpedia.org/resource/Curt_Haagers, http://dbpedia.org/resource/Karl_Martindahl, http://dbpedia.org/resource/Larz-Kristerz, http://dbpedia.org/resource/Matz_Bladhs, http://dbpedia.org/resource/Magnum_Bonum, http://dbpedia.org/resource/CajsaStina_Åkerström, http://dbpedia.org/resource/Lovestoned_(band), http://dbpedia.org/resource/Airbase_(DJ), http://dbpedia.org/resource/Lucy_Street, http://dbpedia.org/resource/H.E.A.T_(band), http://dbpedia.org/resource/Boeoes_Kaelstigen, http://dbpedia.org/resource/Triumphator_(band), http://dbpedia.org/resource/First_Love,_Last_Rites_(musical_group), http://dbpedia.org/resource/Bengt_Hennings, http://dbpedia.org/resource/Jan_Öjlers, http://dbpedia.org/resource/Vildsvin, http://dbpedia.org/resource/Frelow, http://dbpedia.org/resource/Vanbot, http://dbpedia.org/resource/Adiam_Dymott, http://dbpedia.org/resource/E-Type_(musician), http://dbpedia.org/resource/Rebecka_Törnqvist, http://dbpedia.org/resource/The_Bear_Quartet, http://dbpedia.org/resource/Chips_(band), http://dbpedia.org/resource/Jonas_Hellborg, http://dbpedia.org/resource/Kent_(band), http://dbpedia.org/resource/Sami_Sirviö, http://dbpedia.org/resource/Asta_Kask, http://dbpedia.org/resource/Evil_Masquerade, http://dbpedia.org/resource/Sister_Sin, http://dbpedia.org/resource/Blender_(band), http://dbpedia.org/resource/A_Camp, http://dbpedia.org/resource/Stonefunkers, http://dbpedia.org/resource/DJ_Méndez, http://dbpedia.org/resource/Dada_Life, http://dbpedia.org/resource/Loney,_Dear, http://dbpedia.org/resource/Mohammed_Ali_(duo), http://dbpedia.org/resource/The_Perishers_(band), http://dbpedia.org/resource/Casanovas, http://dbpedia.org/resource/Logh_(band), http://dbpedia.org/resource/Barbados_(band), http://dbpedia.org/resource/Eucharist_(band), http://dbpedia.org/resource/Roger_Nilsson, http://dbpedia.org/resource/The_Quill_(band), http://dbpedia.org/resource/A.C.T, http://dbpedia.org/resource/Masquerade_(Swedish_band), http://dbpedia.org/resource/Svartsyn, http://dbpedia.org/resource/E.M.D., http://dbpedia.org/resource/BlackJack_(Swedish_band), http://dbpedia.org/resource/Franky_Lee, http://dbpedia.org/resource/Nouveau_Riche_(Swedish_band), http://dbpedia.org/resource/Kamchatka_(band), http://dbpedia.org/resource/Per_Schelander, http://dbpedia.org/resource/Fejd, http://dbpedia.org/resource/Nostradameus, http://dbpedia.org/resource/Raise_Hell, http://dbpedia.org/resource/Richard_Andersson, http://dbpedia.org/resource/Merciless, http://dbpedia.org/resource/Lorentz_(rapper), http://dbpedia.org/resource/Damn_(band), http://dbpedia.org/resource/Embee, http://dbpedia.org/resource/Magnus_Carlson, http://dbpedia.org/resource/Sound_Factory, http://dbpedia.org/resource/Mr._Death_(band), http://dbpedia.org/resource/Kal_P._Dal, http://dbpedia.org/resource/Lustans_Lakejer, http://dbpedia.org/resource/Trettioåriga_Kriget, http://dbpedia.org/resource/Her_Bright_Skies, http://dbpedia.org/resource/Insision, http://dbpedia.org/resource/Axewitch, http://dbpedia.org/resource/Cloudscape_(band), http://dbpedia.org/resource/Corroded_(band), http://dbpedia.org/resource/Electric_Banana_Band, http://dbpedia.org/resource/Hysterica, http://dbpedia.org/resource/Lillasyster, http://dbpedia.org/resource/Memory_Garden, http://dbpedia.org/resource/Mårran, http://dbpedia.org/resource/Nasty_Idols, http://dbpedia.org/resource/November_(band), http://dbpedia.org/resource/Platitude_(band), http://dbpedia.org/resource/Syron_Vanes, http://dbpedia.org/resource/Amason_(band), http://dbpedia.org/resource/Sad_Day_for_Puppets, http://dbpedia.org/resource/Orup, http://dbpedia.org/resource/Streaplers, http://dbpedia.org/resource/The_Embassy_(band), http://dbpedia.org/resource/Candela_(Swedish_band), http://dbpedia.org/resource/Peter_Mansson]... [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 22, P: 0.277, R: 0.334, macro F1: 0.259, macro F1*: 0.303 -[INFO] QALD Cumulative metrics, sample: 22, P: 0.368, R: 0.334, macro F1: 0.259, QALD macro F1: 0.350 -[INFO] Current time: 10643 ms, average: 4377.45 ms/ques, entity detection: 916.82 ms/ques (linking tool: 820.09 ms/ques), relation detection: 1363.64 ms/ques (similarity: 806.82 ms/ques), EDG generation: 207.82 ms/ques +[INFO] Cumulative metrics, sample: 22, P: 0.277, R: 0.334, macro F1: 0.303 +[INFO] QALD Cumulative metrics, sample: 22, P: 0.368, R: 0.334, average F1: 0.259, QALD macro F1: 0.350 +[INFO] Current question consumed time: 9818 ms, average: 4130.91 ms/ques, entity detection: 840.14 ms/ques (linking tool: 751.77 ms/ques), relation detection: 1318.55 ms/ques (similarity: 819.05 ms/ques), EDG generation: 213.45 ms/ques [INFO] Entity Linking P: 0.13636363636363635, R: 0.13636363636363635, micro F1: 0.13636363636363635, macro F1: 0.13636363636363635 [INFO] Relation Linking P: 0.18181818181818182, R: 0.18181818181818182, micro F1: 0.18181818181818182, macro F1: 0.18181818181818182 [INFO] Type Linking P: 0.9090909090909091, R: 0.9090909090909091, micro F1: 0.9090909090909091, macro F1: 0.9090909090909091 @@ -337,9 +337,9 @@ Question 22: Which monarchs of the United Kingdom were married to a German? [INFO] Golden Answer: [http://dbpedia.org/resource/William_IV_of_the_United_Kingdom] [INFO] Predicted Answer: [] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 23, P: 0.265, R: 0.320, macro F1: 0.248, macro F1*: 0.290 -[INFO] QALD Cumulative metrics, sample: 23, P: 0.395, R: 0.320, macro F1: 0.248, QALD macro F1: 0.354 -[INFO] Current time: 19633 ms, average: 5040.74 ms/ques, entity detection: 1111.22 ms/ques (linking tool: 1013.04 ms/ques), relation detection: 1588.61 ms/ques (similarity: 865.57 ms/ques), EDG generation: 199.91 ms/ques +[INFO] Cumulative metrics, sample: 23, P: 0.265, R: 0.320, macro F1: 0.290 +[INFO] QALD Cumulative metrics, sample: 23, P: 0.395, R: 0.320, average F1: 0.248, QALD macro F1: 0.354 +[INFO] Current question consumed time: 17572 ms, average: 4715.30 ms/ques, entity detection: 1017.22 ms/ques (linking tool: 927.13 ms/ques), relation detection: 1515.30 ms/ques (similarity: 876.87 ms/ques), EDG generation: 205.57 ms/ques [INFO] Entity Linking P: 0.17391304347826086, R: 0.17391304347826086, micro F1: 0.17391304347826086, macro F1: 0.17391304347826086 [INFO] Relation Linking P: 0.21739130434782608, R: 0.21739130434782608, micro F1: 0.21739130434782608, macro F1: 0.21739130434782608 [INFO] Type Linking P: 0.9130434782608695, R: 0.9130434782608695, micro F1: 0.9130434782608695, macro F1: 0.9130434782608695 @@ -352,9 +352,9 @@ Question 23: Give me all Argentine films. [INFO] Golden Answer: [http://dbpedia.org/resource/The_Warrior_and_the_Sorceress, http://dbpedia.org/resource/Air_Force,_Incorporated, http://dbpedia.org/resource/Los_ricos_no_piden_permiso, http://dbpedia.org/resource/Culpable_(film), http://dbpedia.org/resource/The_German_Friend, http://dbpedia.org/resource/Géminis, http://dbpedia.org/resource/Left_for_Dead_(2007_Western_film), http://dbpedia.org/resource/Angel_Face_(1998_film), http://dbpedia.org/resource/Cacería, http://dbpedia.org/resource/No_sos_vos,_soy_yo, http://dbpedia.org/resource/A_Cuban_in_Spain, http://dbpedia.org/resource/Buenos_Aires_a_la_vista, http://dbpedia.org/resource/Off_to_Havana_I_Go, http://dbpedia.org/resource/Chile_672, http://dbpedia.org/resource/Blessed_by_Fire, http://dbpedia.org/resource/La_Aventura_explosiva, http://dbpedia.org/resource/Ashes_to_the_Wind, http://dbpedia.org/resource/Tire_dié, http://dbpedia.org/resource/Argentina_revolucionaria, http://dbpedia.org/resource/Bajo_el_signo_de_la_patria, http://dbpedia.org/resource/Bodas_de_cristal, http://dbpedia.org/resource/Campo_arado, http://dbpedia.org/resource/Cantaniño_cuenta_un_cuento, http://dbpedia.org/resource/Casi_un_sueño, http://dbpedia.org/resource/Cerro_Guanaco, http://dbpedia.org/resource/Chiche_bombón, http://dbpedia.org/resource/Las_Aventuras_de_Pikín, http://dbpedia.org/resource/Una_Atrevida_aventurita, http://dbpedia.org/resource/Rodencia_y_el_Diente_de_la_Princesa, http://dbpedia.org/resource/Animal_(2001_film), http://dbpedia.org/resource/Orquesta_Tipica, http://dbpedia.org/resource/The_Tango_Lesson, http://dbpedia.org/resource/Familia_rodante, http://dbpedia.org/resource/Whisky_Romeo_Zulu, http://dbpedia.org/resource/Cama_Adentro, http://dbpedia.org/resource/Todas_las_azafatas_van_al_cielo, http://dbpedia.org/resource/The_Curious_Dr._Humpp, http://dbpedia.org/resource/Immigrants_(1948_film), http://dbpedia.org/resource/Herederos_de_una_venganza, http://dbpedia.org/resource/Monobloc_(film), http://dbpedia.org/resource/Las_furias, http://dbpedia.org/resource/Lost_Embrace, http://dbpedia.org/resource/Waiting_for_the_Messiah, http://dbpedia.org/resource/Amigos_para_La_Aventura, http://dbpedia.org/resource/La_Amada_Inmóvil, http://dbpedia.org/resource/The_Soul_of_the_Children, http://dbpedia.org/resource/Aquello_que_Amamos, http://dbpedia.org/resource/Help_Me_to_Live, http://dbpedia.org/resource/The_Circus_Cavalcade, http://dbpedia.org/resource/Joven,_viuda_y_estanciera_(1970_film), http://dbpedia.org/resource/Así_es_Buenos_Aires, http://dbpedia.org/resource/A_Chrysanthemum_Bursts_in_Cincoesquinas, http://dbpedia.org/resource/Asunto_terminado, http://dbpedia.org/resource/Rapado, http://dbpedia.org/resource/Balada_para_un_mochilero, http://dbpedia.org/resource/El_caso_Matías, http://dbpedia.org/resource/Path_to_Peace, http://dbpedia.org/resource/El_Camino_de_San_Diego, http://dbpedia.org/resource/Son_of_the_Bride, http://dbpedia.org/resource/Foolish_Heart_(1998_film), http://dbpedia.org/resource/Monday's_Child_(film), http://dbpedia.org/resource/Asesinato_en_el_Senado_de_la_Nación, http://dbpedia.org/resource/Love_Is_a_Fat_Woman, http://dbpedia.org/resource/Palo_y_hueso, http://dbpedia.org/resource/Verónico_Cruz_(film), http://dbpedia.org/resource/Patagonia_(film), http://dbpedia.org/resource/Alma_Mía, http://dbpedia.org/resource/The_Marihuana_Story, http://dbpedia.org/resource/Maradona,_the_Hand_of_God, http://dbpedia.org/resource/Amor_prohibido_(film), http://dbpedia.org/resource/Amor_último_modelo, http://dbpedia.org/resource/Excitante, http://dbpedia.org/resource/Parapalos, http://dbpedia.org/resource/El_Amarillo, http://dbpedia.org/resource/Abuso_de_confianza, http://dbpedia.org/resource/Amor_a_primera_vista, http://dbpedia.org/resource/Canario_rojo, http://dbpedia.org/resource/La_Procesión, http://dbpedia.org/resource/The_Age_of_Love_(1954_film), http://dbpedia.org/resource/Cuidado_Con_Las_Mujeres, http://dbpedia.org/resource/Las_Aventuras_de_Tremendo, http://dbpedia.org/resource/With_the_Music_in_my_Soul, http://dbpedia.org/resource/Anteojito_y_Antifaz,_mil_intentos_y_un_invento, http://dbpedia.org/resource/El_Mar_de_Lucas, http://dbpedia.org/resource/Harassed_(1964_film), http://dbpedia.org/resource/Diez_canciones_de_Gardel, http://dbpedia.org/resource/Tetro, http://dbpedia.org/resource/The_Motorcycle_Diaries_(film), http://dbpedia.org/resource/Snuff_(film), http://dbpedia.org/resource/Snuff_102, http://dbpedia.org/resource/El_Profesor_Hippie, http://dbpedia.org/resource/The_Supporter, http://dbpedia.org/resource/Hawaii_(2013_film), http://dbpedia.org/resource/Testosterone_(film), http://dbpedia.org/resource/XXY_(film), http://dbpedia.org/resource/Peludópolis, http://dbpedia.org/resource/High_School_Musical:_El_Desafío, http://dbpedia.org/resource/Social_Genocide, http://dbpedia.org/resource/Alambrado, http://dbpedia.org/resource/White_Elephant_(2012_film), http://dbpedia.org/resource/Arriba_Juventud, http://dbpedia.org/resource/Chiquititas:_Rincón_de_luz, http://dbpedia.org/resource/Rincón_de_Luz, http://dbpedia.org/resource/Apasionados, http://dbpedia.org/resource/Alma_Pirata, http://dbpedia.org/resource/El_Asalto, http://dbpedia.org/resource/Valentina_(2008_film), http://dbpedia.org/resource/Amor_libre_(film), http://dbpedia.org/resource/Las_Apariencias_engañan, http://dbpedia.org/resource/Brigada_en_acción, http://dbpedia.org/resource/Good_Life_Delivery, http://dbpedia.org/resource/Plaza_de_Almas, http://dbpedia.org/resource/The_Quispe_Girls, http://dbpedia.org/resource/A_Sangre_Fría, http://dbpedia.org/resource/Aconcagua_(film), http://dbpedia.org/resource/Arrabalera, http://dbpedia.org/resource/Closed_Door, http://dbpedia.org/resource/The_Two_Waters, http://dbpedia.org/resource/Accident_703, http://dbpedia.org/resource/Amalio_Reyes,_un_hombre, http://dbpedia.org/resource/La_Quintrala_(film), http://dbpedia.org/resource/Babilonia, http://dbpedia.org/resource/Captura_recomendada, http://dbpedia.org/resource/3_Américas, http://dbpedia.org/resource/Return_to_Bolivia, http://dbpedia.org/resource/An_American_in_Buenos_Aires, http://dbpedia.org/resource/Roma_(2004_film), http://dbpedia.org/resource/Plumíferos, http://dbpedia.org/resource/Eversmile,_New_Jersey, http://dbpedia.org/resource/Gaturro_(film), http://dbpedia.org/resource/Waiting_for_the_Hearse, http://dbpedia.org/resource/Boquitas_pintadas, http://dbpedia.org/resource/Johny_Tolengo,_el_majestuoso, http://dbpedia.org/resource/Los_caballeros_de_la_cama_redonda, http://dbpedia.org/resource/Un_Argentino_en_New_York, http://dbpedia.org/resource/Love_in_Flight, http://dbpedia.org/resource/Noites_Cariocas, http://dbpedia.org/resource/Al_Compás_de_tu_Mentira, http://dbpedia.org/resource/Buenas_noches,_Buenos_Aires, http://dbpedia.org/resource/The_Soul_of_a_Tango, http://dbpedia.org/resource/Family_Law_(film), http://dbpedia.org/resource/La_vida_por_Perón, http://dbpedia.org/resource/The_Hand_in_the_Trap, http://dbpedia.org/resource/The_Blonds, http://dbpedia.org/resource/The_Girl_from_Florida_Street, http://dbpedia.org/resource/La_barra_de_Taponazo, http://dbpedia.org/resource/La_vía_de_oro, http://dbpedia.org/resource/Yira,_yira, http://dbpedia.org/resource/Native_Son_(1951_film), http://dbpedia.org/resource/Academia_El_Tango_Argentino, http://dbpedia.org/resource/Assassination_Tango, http://dbpedia.org/resource/La_Raulito, http://dbpedia.org/resource/Muñequitas_porteñas, http://dbpedia.org/resource/The_Accused_(1960_film), http://dbpedia.org/resource/Alias_Gardelito, http://dbpedia.org/resource/Where_the_Wind_Dies, http://dbpedia.org/resource/Burnt_Money, http://dbpedia.org/resource/A_King_and_His_Movie, http://dbpedia.org/resource/Bailoretto,_la_aventura_de_un_rebelde, http://dbpedia.org/resource/The_ABC_of_Love, http://dbpedia.org/resource/The_Minder, http://dbpedia.org/resource/Corazón_de_criolla, http://dbpedia.org/resource/El_Andador, http://dbpedia.org/resource/La_Gaucha, http://dbpedia.org/resource/Las_Aventuras_del_Capitán_Piluso_en_el_Castillo_del_Terror, http://dbpedia.org/resource/Perdón,_viejita, http://dbpedia.org/resource/Anni_Ribelli, http://dbpedia.org/resource/The_Appeared, http://dbpedia.org/resource/Así_No_Hay_Cama_Que_Aguante, http://dbpedia.org/resource/Mujeres_de_nadie, http://dbpedia.org/resource/Los_Cuentos_de_Borges:_El_Sur, http://dbpedia.org/resource/A_Buddha, http://dbpedia.org/resource/Francis:_Pray_for_Me, http://dbpedia.org/resource/Eva_Doesn't_Sleep, http://dbpedia.org/resource/Bólidos_de_acero, http://dbpedia.org/resource/With_Life_and_Soul, http://dbpedia.org/resource/La_Fuga, http://dbpedia.org/resource/El_perro, http://dbpedia.org/resource/Allá_en_el_Norte, http://dbpedia.org/resource/Andrea_(film), http://dbpedia.org/resource/La_Ciénaga_(film), http://dbpedia.org/resource/Una_noche_con_Sabrina_Love, http://dbpedia.org/resource/Common_Places, http://dbpedia.org/resource/El_Bonaerense, http://dbpedia.org/resource/Un_Oso_Rojo, http://dbpedia.org/resource/The_Grandfather_(1954_film), http://dbpedia.org/resource/Agua_(film), http://dbpedia.org/resource/Only_Human_(2004_film), http://dbpedia.org/resource/Tocar_el_cielo, http://dbpedia.org/resource/Adiós,_Roberto, http://dbpedia.org/resource/Adiós_problemas, http://dbpedia.org/resource/Buenos_Aires,_Ciudad_de_Ensueño, http://dbpedia.org/resource/Magic_Bay, http://dbpedia.org/resource/Adiós_pampa_mía, http://dbpedia.org/resource/Ven_mi_corazón_te_llama, http://dbpedia.org/resource/Adam_and_the_Serpent, http://dbpedia.org/resource/Searching_for_Monica, http://dbpedia.org/resource/La_Fidelidad, http://dbpedia.org/resource/Las_Campanas_de_Teresa, http://dbpedia.org/resource/Whisky_(film), http://dbpedia.org/resource/A_Place_in_the_World_(film), http://dbpedia.org/resource/Delirium_(2014_film), http://dbpedia.org/resource/Ana_and_the_Others, http://dbpedia.org/resource/Crane_World, http://dbpedia.org/resource/Mala_Época, http://dbpedia.org/resource/God_Reward_You, http://dbpedia.org/resource/Rosaura_at_10_O'Clock, http://dbpedia.org/resource/Deathstalker_(film), http://dbpedia.org/resource/Top_Cat:_The_Movie, http://dbpedia.org/resource/El_Satario, http://dbpedia.org/resource/Cocaine_Wars, http://dbpedia.org/resource/Nobleza_gaucha_(1915_film), http://dbpedia.org/resource/Soy_gitano, http://dbpedia.org/resource/Las_Acacias_(film), http://dbpedia.org/resource/Así_O_De_Otra_Manera, http://dbpedia.org/resource/El_Amor_–_primera_parte, http://dbpedia.org/resource/The_Crossroads_(1952_film), http://dbpedia.org/resource/Cubitos_de_hielo, http://dbpedia.org/resource/The_Man_Who_Owed_a_Death, http://dbpedia.org/resource/Centauros_del_pasado, http://dbpedia.org/resource/Eclipse_of_the_Sun_(film), http://dbpedia.org/resource/Encadenado, http://dbpedia.org/resource/Avivato, http://dbpedia.org/resource/Deathstalker_II, http://dbpedia.org/resource/Chinese_Take-Away, http://dbpedia.org/resource/Perdido_por_perdido_(film), http://dbpedia.org/resource/Esa_Maldita_Costilla, http://dbpedia.org/resource/The_Last_Train_(2002_film), http://dbpedia.org/resource/Adiós_muchachos, http://dbpedia.org/resource/Amor_y_un_poco_más, http://dbpedia.org/resource/The_Secret_Agents_Against_Green_Glove, http://dbpedia.org/resource/The_Yacht_Isabel_Arrived_This_Afternoon, http://dbpedia.org/resource/La_Muchacha_del_arrabal, http://dbpedia.org/resource/Los_amores_de_Laurita, http://dbpedia.org/resource/Departamento_compartido, http://dbpedia.org/resource/La_Antena, http://dbpedia.org/resource/Humo_de_Marihuana, http://dbpedia.org/resource/El_Amor_infiel, http://dbpedia.org/resource/El_Amor_y_el_Espanto, http://dbpedia.org/resource/Kill_and_Be_Killed, http://dbpedia.org/resource/The_Bohemian_Soul, http://dbpedia.org/resource/The_Loves_of_Kafka, http://dbpedia.org/resource/Una_Jaula_no_tiene_secretos] [INFO] Predicted Answer: [] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 24, P: 0.254, R: 0.307, macro F1: 0.238, macro F1*: 0.278 -[INFO] QALD Cumulative metrics, sample: 24, P: 0.421, R: 0.307, macro F1: 0.238, QALD macro F1: 0.355 -[INFO] Current time: 251 ms, average: 4841.17 ms/ques, entity detection: 1064.96 ms/ques (linking tool: 970.83 ms/ques), relation detection: 1522.42 ms/ques (similarity: 829.50 ms/ques), EDG generation: 191.92 ms/ques +[INFO] Cumulative metrics, sample: 24, P: 0.254, R: 0.307, macro F1: 0.278 +[INFO] QALD Cumulative metrics, sample: 24, P: 0.421, R: 0.307, average F1: 0.238, QALD macro F1: 0.355 +[INFO] Current question consumed time: 216 ms, average: 4527.88 ms/ques, entity detection: 974.83 ms/ques (linking tool: 888.50 ms/ques), relation detection: 1452.17 ms/ques (similarity: 840.33 ms/ques), EDG generation: 197.29 ms/ques [INFO] Entity Linking P: 0.20833333333333334, R: 0.20833333333333334, micro F1: 0.20833333333333334, macro F1: 0.20833333333333334 [INFO] Relation Linking P: 0.25, R: 0.25, micro F1: 0.25, macro F1: 0.25 [INFO] Type Linking P: 0.9166666666666666, R: 0.9166666666666666, micro F1: 0.9166666666666666, macro F1: 0.9166666666666666 @@ -367,9 +367,9 @@ Question 24: How did Michael Jackson die? [INFO] Golden Answer: [Homicide] [INFO] Predicted Answer: [Homicide] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 25, P: 0.284, R: 0.334, macro F1: 0.268, macro F1*: 0.307 -[INFO] QALD Cumulative metrics, sample: 25, P: 0.444, R: 0.334, macro F1: 0.268, QALD macro F1: 0.381 -[INFO] Current time: 2355 ms, average: 4741.72 ms/ques, entity detection: 1042.68 ms/ques (linking tool: 947.80 ms/ques), relation detection: 1497.36 ms/ques (similarity: 815.32 ms/ques), EDG generation: 184.60 ms/ques +[INFO] Cumulative metrics, sample: 25, P: 0.284, R: 0.334, macro F1: 0.307 +[INFO] QALD Cumulative metrics, sample: 25, P: 0.444, R: 0.334, average F1: 0.268, QALD macro F1: 0.381 +[INFO] Current question consumed time: 2290 ms, average: 4438.36 ms/ques, entity detection: 956.00 ms/ques (linking tool: 868.52 ms/ques), relation detection: 1428.68 ms/ques (similarity: 825.60 ms/ques), EDG generation: 189.72 ms/ques [INFO] Entity Linking P: 0.2, R: 0.2, micro F1: 0.2, macro F1: 0.20000000000000004 [INFO] Relation Linking P: 0.24, R: 0.24, micro F1: 0.24, macro F1: 0.24 [INFO] Type Linking P: 0.92, R: 0.92, micro F1: 0.92, macro F1: 0.92 @@ -382,9 +382,9 @@ Question 25: Which U.S. state has been admitted latest? [INFO] Golden Answer: [http://dbpedia.org/resource/Hawaii] [INFO] Predicted Answer: [] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 26, P: 0.273, R: 0.321, macro F1: 0.258, macro F1*: 0.295 -[INFO] QALD Cumulative metrics, sample: 26, P: 0.465, R: 0.321, macro F1: 0.258, QALD macro F1: 0.380 -[INFO] Current time: 1765 ms, average: 4627.23 ms/ques, entity detection: 1025.88 ms/ques (linking tool: 932.19 ms/ques), relation detection: 1443.42 ms/ques (similarity: 786.31 ms/ques), EDG generation: 178.00 ms/ques +[INFO] Cumulative metrics, sample: 26, P: 0.273, R: 0.321, macro F1: 0.295 +[INFO] QALD Cumulative metrics, sample: 26, P: 0.465, R: 0.321, average F1: 0.258, QALD macro F1: 0.380 +[INFO] Current question consumed time: 1597 ms, average: 4329.08 ms/ques, entity detection: 940.23 ms/ques (linking tool: 853.92 ms/ques), relation detection: 1377.08 ms/ques (similarity: 796.12 ms/ques), EDG generation: 182.88 ms/ques [INFO] Entity Linking P: 0.23076923076923078, R: 0.23076923076923078, micro F1: 0.23076923076923078, macro F1: 0.23076923076923078 [INFO] Relation Linking P: 0.2692307692307692, R: 0.2692307692307692, micro F1: 0.2692307692307692, macro F1: 0.2692307692307692 [INFO] Type Linking P: 0.9230769230769231, R: 0.9230769230769231, micro F1: 0.9230769230769231, macro F1: 0.9230769230769231 @@ -397,9 +397,9 @@ Question 26: Which classes does the Millepede belong to? [INFO] Golden Answer: [Diplopoda] [INFO] Predicted Answer: [http://dbpedia.org/resource/Pill_millipede, http://dbpedia.org/resource/Harpaphe_haydeniana, http://dbpedia.org/resource/Mammamia, http://dbpedia.org/resource/Hiltonius, http://dbpedia.org/resource/Desmoxytes, http://dbpedia.org/resource/Orthoporus_ornatus, http://dbpedia.org/resource/Bollmania, http://dbpedia.org/resource/Spirobolidae, http://dbpedia.org/resource/Tylobolus, http://dbpedia.org/resource/Procyliosoma, http://dbpedia.org/resource/Causeyella_causeyae, http://dbpedia.org/resource/Trigoniulus_corallinus, http://dbpedia.org/resource/Archispirostreptus, http://dbpedia.org/resource/Archispirostreptus_gigas, http://dbpedia.org/resource/Callipodida, http://dbpedia.org/resource/Causeyella, http://dbpedia.org/resource/Chordeumatida, http://dbpedia.org/resource/Glomerida, http://dbpedia.org/resource/Glomeridesmida, http://dbpedia.org/resource/Illacme, http://dbpedia.org/resource/Julida, http://dbpedia.org/resource/Juliformia, http://dbpedia.org/resource/Julus_(genus), http://dbpedia.org/resource/Motyxia, http://dbpedia.org/resource/Ommatoiulus_moreleti, http://dbpedia.org/resource/Orthoporus, http://dbpedia.org/resource/Pentazonia, http://dbpedia.org/resource/Platydesmida, http://dbpedia.org/resource/Polydesmida, http://dbpedia.org/resource/Polyxenida, http://dbpedia.org/resource/Polyxenus_fasciculatus, http://dbpedia.org/resource/Polyzoniida, http://dbpedia.org/resource/Psammodesmus_bryophorus, http://dbpedia.org/resource/Siphonocryptida, http://dbpedia.org/resource/Siphonophorida, http://dbpedia.org/resource/Sphaerotheriida, http://dbpedia.org/resource/Spirobolida, http://dbpedia.org/resource/Spirostreptida, http://dbpedia.org/resource/Trichopetalum_whitei, http://dbpedia.org/resource/Xenobolus_carnifex, http://dbpedia.org/resource/Ommatoiulus_avatar, http://dbpedia.org/resource/Californiulus, http://dbpedia.org/resource/Chamberlineptus, http://dbpedia.org/resource/Glomeridae, http://dbpedia.org/resource/Brachyiulus_apfelbecki, http://dbpedia.org/resource/Brachyiulus_bagnalli, http://dbpedia.org/resource/Brachyiulus_jawlowskii, http://dbpedia.org/resource/Brachyiulus_klisurensis, http://dbpedia.org/resource/Brachyiulus_lusitanus, http://dbpedia.org/resource/Brachyiulus_stuxbergi, http://dbpedia.org/resource/Brachyiulus_varibolinus, http://dbpedia.org/resource/Floridobolus, http://dbpedia.org/resource/Nyssodesmus, http://dbpedia.org/resource/Nyssodesmus_python, http://dbpedia.org/resource/Narceus, http://dbpedia.org/resource/Apheloria, http://dbpedia.org/resource/Brachoria, http://dbpedia.org/resource/Xystodesmidae, http://dbpedia.org/resource/Julidae, http://dbpedia.org/resource/Titanophyllum, http://dbpedia.org/resource/Trachysphaera, http://dbpedia.org/resource/Alloporus, http://dbpedia.org/resource/Anastreptus, http://dbpedia.org/resource/Anethoporus, http://dbpedia.org/resource/Calostreptus, http://dbpedia.org/resource/Phyllogonostreptus_nigrolabiatus, http://dbpedia.org/resource/Spirostreptus, http://dbpedia.org/resource/Telodeinopus, http://dbpedia.org/resource/Triaenostreptus, http://dbpedia.org/resource/Polyxenidae, http://dbpedia.org/resource/Polyxenus_(genus), http://dbpedia.org/resource/Sphaeromimus, http://dbpedia.org/resource/Brachyiulus, http://dbpedia.org/resource/Cylindroiulus, http://dbpedia.org/resource/Seychelles_giant_millipede, http://dbpedia.org/resource/Glomeris_marginata, http://dbpedia.org/resource/Tachypodoiulus_niger, http://dbpedia.org/resource/Spinotarsus_caboverdus, http://dbpedia.org/resource/Glomeris, http://dbpedia.org/resource/Trichopetalum_(millipede), http://dbpedia.org/resource/Hirudisomatidae, http://dbpedia.org/resource/Synxenidae, http://dbpedia.org/resource/Brachycybe, http://dbpedia.org/resource/Gosodesmus, http://dbpedia.org/resource/Apheloria_tigana, http://dbpedia.org/resource/Mardonius_(genus), http://dbpedia.org/resource/Spirostreptidae, http://dbpedia.org/resource/Harpaphe, http://dbpedia.org/resource/Procyliosoma_tuberculatum, http://dbpedia.org/resource/Sphaerotherium, http://dbpedia.org/resource/Proteroiulus_broelemanni, http://dbpedia.org/resource/Proteroiulus_fuscus, http://dbpedia.org/resource/Proteroiulus_hispanus, http://dbpedia.org/resource/Narceus_gordanus, http://dbpedia.org/resource/Aprosphylosoma, http://dbpedia.org/resource/Californiulus_euphanus, http://dbpedia.org/resource/Californiulus_yosemitensis, http://dbpedia.org/resource/Paeromopus, http://dbpedia.org/resource/Paeromopus_angusticeps, http://dbpedia.org/resource/Paeromopus_paniculus, http://dbpedia.org/resource/Pleurojulidae, http://dbpedia.org/resource/Narceus_americanus, http://dbpedia.org/resource/Myriacantherpestes, http://dbpedia.org/resource/Ammodesmidae, http://dbpedia.org/resource/Cyrtodesmidae, http://dbpedia.org/resource/Desmoxytes_rhinoceros, http://dbpedia.org/resource/Desmoxytes_rhinoparva, http://dbpedia.org/resource/Eutrichodesmus, http://dbpedia.org/resource/Gomphodesmidae, http://dbpedia.org/resource/Haplodesmidae, http://dbpedia.org/resource/Leptodesmidea, http://dbpedia.org/resource/Platyrhacidae, http://dbpedia.org/resource/Polydesmidae, http://dbpedia.org/resource/Tridontomidae, http://dbpedia.org/resource/Glomeris_connexa, http://dbpedia.org/resource/Glomeris_klugii, http://dbpedia.org/resource/Condexenus, http://dbpedia.org/resource/Phryssonotus, http://dbpedia.org/resource/Schindalmonotus, http://dbpedia.org/resource/Colobodesmus_crucis, http://dbpedia.org/resource/Sphaeriodesmus_filamentosus, http://dbpedia.org/resource/Trichomorpha_esulcata, http://dbpedia.org/resource/Trichomorpha_gracilis, http://dbpedia.org/resource/Julus_terrestris, http://dbpedia.org/resource/Ommatoiulus_sabulosus, http://dbpedia.org/resource/Polyxenus_lagurus, http://dbpedia.org/resource/Desmoxytoides, http://dbpedia.org/resource/Blaniulidae, http://dbpedia.org/resource/Cylindroiulus_punctatus, http://dbpedia.org/resource/Nemasomatidae, http://dbpedia.org/resource/Pachyiulus, http://dbpedia.org/resource/Paeromopodidae, http://dbpedia.org/resource/Parajulidae, http://dbpedia.org/resource/Stemmiulidae, http://dbpedia.org/resource/Anadenobolus_monilicornis, http://dbpedia.org/resource/Crurifarcimen, http://dbpedia.org/resource/Eucarlia, http://dbpedia.org/resource/Messicobolidae, http://dbpedia.org/resource/Rhinocricidae, http://dbpedia.org/resource/Doratogonus, http://dbpedia.org/resource/Ommatoiulus, http://dbpedia.org/resource/Sinocallipus, http://dbpedia.org/resource/Parafontaria, http://dbpedia.org/resource/Xystocheir, http://dbpedia.org/resource/Amplinus_bituberculosus, http://dbpedia.org/resource/Cyrtodesmus_depressus, http://dbpedia.org/resource/Cyrtodesmus_humerosus, http://dbpedia.org/resource/Cyrtodesmus_lobatus, http://dbpedia.org/resource/Blaniulus, http://dbpedia.org/resource/Boreoiulus, http://dbpedia.org/resource/Proteroiulus, http://dbpedia.org/resource/Blaniulus_guttulatus, http://dbpedia.org/resource/Blaniulus_lichtensteini, http://dbpedia.org/resource/Blaniulus_lorifer, http://dbpedia.org/resource/Blaniulus_mayeti, http://dbpedia.org/resource/Blaniulus_orientalis, http://dbpedia.org/resource/Blaniulus_troglobius, http://dbpedia.org/resource/Blaniulus_troglodites, http://dbpedia.org/resource/Boreoiulus_dollfusi, http://dbpedia.org/resource/Julus_curvicornis, http://dbpedia.org/resource/Julus_scandinavius, http://dbpedia.org/resource/Julus_scanicus, http://dbpedia.org/resource/Pachyiulus_asiaeminoris, http://dbpedia.org/resource/Pachyiulus_cattarensis, http://dbpedia.org/resource/Pachyiulus_dentiger, http://dbpedia.org/resource/Pachyiulus_humicola, http://dbpedia.org/resource/Pachyiulus_hungaricus, http://dbpedia.org/resource/Pachyiulus_marmoratus, http://dbpedia.org/resource/Pachyiulus_silvestrii, http://dbpedia.org/resource/Pachyiulus_speciosus, http://dbpedia.org/resource/Pachyiulus_varius, http://dbpedia.org/resource/Psammodesmus, http://dbpedia.org/resource/Xenobolus, http://dbpedia.org/resource/Aulodesmus, http://dbpedia.org/resource/Aulodesmus_mossambicus] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 27, P: 0.263, R: 0.310, macro F1: 0.248, macro F1*: 0.284 -[INFO] QALD Cumulative metrics, sample: 27, P: 0.448, R: 0.310, macro F1: 0.248, QALD macro F1: 0.366 -[INFO] Current time: 894 ms, average: 4488.96 ms/ques, entity detection: 991.44 ms/ques (linking tool: 900.41 ms/ques), relation detection: 1402.48 ms/ques (similarity: 769.33 ms/ques), EDG generation: 171.81 ms/ques +[INFO] Cumulative metrics, sample: 27, P: 0.263, R: 0.310, macro F1: 0.284 +[INFO] QALD Cumulative metrics, sample: 27, P: 0.448, R: 0.310, average F1: 0.248, QALD macro F1: 0.366 +[INFO] Current question consumed time: 835 ms, average: 4199.67 ms/ques, entity detection: 908.63 ms/ques (linking tool: 824.81 ms/ques), relation detection: 1337.96 ms/ques (similarity: 778.63 ms/ques), EDG generation: 176.52 ms/ques [INFO] Entity Linking P: 0.2222222222222222, R: 0.2222222222222222, micro F1: 0.2222222222222222, macro F1: 0.2222222222222222 [INFO] Relation Linking P: 0.25925925925925924, R: 0.25925925925925924, micro F1: 0.25925925925925924, macro F1: 0.25925925925925924 [INFO] Type Linking P: 0.9259259259259259, R: 0.9259259259259259, micro F1: 0.9259259259259259, macro F1: 0.9259259259259259 @@ -412,9 +412,9 @@ Question 27: Give me the homepage of Forbes. [INFO] Golden Answer: [http://forbes.com] [INFO] Predicted Answer: [http://forbes.com] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 28, P: 0.289, R: 0.334, macro F1: 0.275, macro F1*: 0.310 -[INFO] QALD Cumulative metrics, sample: 28, P: 0.468, R: 0.334, macro F1: 0.275, QALD macro F1: 0.390 -[INFO] Current time: 1646 ms, average: 4387.43 ms/ques, entity detection: 966.82 ms/ques (linking tool: 875.64 ms/ques), relation detection: 1370.04 ms/ques (similarity: 750.82 ms/ques), EDG generation: 166.07 ms/ques +[INFO] Cumulative metrics, sample: 28, P: 0.289, R: 0.334, macro F1: 0.310 +[INFO] QALD Cumulative metrics, sample: 28, P: 0.468, R: 0.334, average F1: 0.275, QALD macro F1: 0.390 +[INFO] Current question consumed time: 1464 ms, average: 4102.00 ms/ques, entity detection: 886.11 ms/ques (linking tool: 802.04 ms/ques), relation detection: 1306.64 ms/ques (similarity: 759.71 ms/ques), EDG generation: 170.57 ms/ques [INFO] Entity Linking P: 0.21428571428571427, R: 0.21428571428571427, micro F1: 0.21428571428571427, macro F1: 0.21428571428571427 [INFO] Relation Linking P: 0.25, R: 0.25, micro F1: 0.25, macro F1: 0.25 [INFO] Type Linking P: 0.9285714285714286, R: 0.9285714285714286, micro F1: 0.9285714285714286, macro F1: 0.9285714285714286 @@ -427,9 +427,9 @@ Question 28: Who is the husband of Amanda Palmer? [INFO] Golden Answer: [http://dbpedia.org/resource/Neil_Gaiman] [INFO] Predicted Answer: [http://dbpedia.org/resource/Neil_Gaiman] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 29, P: 0.314, R: 0.357, macro F1: 0.300, macro F1*: 0.334 -[INFO] QALD Cumulative metrics, sample: 29, P: 0.486, R: 0.357, macro F1: 0.300, QALD macro F1: 0.412 -[INFO] Current time: 1124 ms, average: 4274.90 ms/ques, entity detection: 940.28 ms/ques (linking tool: 849.38 ms/ques), relation detection: 1336.93 ms/ques (similarity: 733.31 ms/ques), EDG generation: 160.66 ms/ques +[INFO] Cumulative metrics, sample: 29, P: 0.314, R: 0.357, macro F1: 0.334 +[INFO] QALD Cumulative metrics, sample: 29, P: 0.486, R: 0.357, average F1: 0.300, QALD macro F1: 0.412 +[INFO] Current question consumed time: 1008 ms, average: 3995.31 ms/ques, entity detection: 861.41 ms/ques (linking tool: 777.83 ms/ques), relation detection: 1275.14 ms/ques (similarity: 741.86 ms/ques), EDG generation: 165.10 ms/ques [INFO] Entity Linking P: 0.20689655172413793, R: 0.20689655172413793, micro F1: 0.20689655172413793, macro F1: 0.20689655172413793 [INFO] Relation Linking P: 0.2413793103448276, R: 0.2413793103448276, micro F1: 0.2413793103448276, macro F1: 0.2413793103448276 [INFO] Type Linking P: 0.9310344827586207, R: 0.9310344827586207, micro F1: 0.9310344827586207, macro F1: 0.9310344827586207 @@ -442,9 +442,9 @@ Question 29: Who painted The Storm on the Sea of Galilee? [INFO] Golden Answer: [http://dbpedia.org/resource/Rembrandt_van_Rijn] [INFO] Predicted Answer: [http://dbpedia.org/resource/Rembrandt_van_Rijn] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 30, P: 0.336, R: 0.379, macro F1: 0.323, macro F1*: 0.356 -[INFO] QALD Cumulative metrics, sample: 30, P: 0.503, R: 0.379, macro F1: 0.323, QALD macro F1: 0.432 -[INFO] Current time: 1471 ms, average: 4181.43 ms/ques, entity detection: 918.73 ms/ques (linking tool: 828.87 ms/ques), relation detection: 1297.80 ms/ques (similarity: 712.30 ms/ques), EDG generation: 158.20 ms/ques +[INFO] Cumulative metrics, sample: 30, P: 0.336, R: 0.379, macro F1: 0.356 +[INFO] QALD Cumulative metrics, sample: 30, P: 0.503, R: 0.379, average F1: 0.323, QALD macro F1: 0.432 +[INFO] Current question consumed time: 1300 ms, average: 3905.47 ms/ques, entity detection: 841.70 ms/ques (linking tool: 759.20 ms/ques), relation detection: 1238.00 ms/ques (similarity: 720.70 ms/ques), EDG generation: 162.27 ms/ques [INFO] Entity Linking P: 0.2, R: 0.2, micro F1: 0.2, macro F1: 0.20000000000000004 [INFO] Relation Linking P: 0.23333333333333334, R: 0.23333333333333334, micro F1: 0.23333333333333334, macro F1: 0.23333333333333334 [INFO] Type Linking P: 0.9333333333333333, R: 0.9333333333333333, micro F1: 0.9333333333333333, macro F1: 0.9333333333333333 @@ -457,9 +457,9 @@ Question 30: What is the highest place of Karakoram? [INFO] Golden Answer: [K2] [INFO] Predicted Answer: [K2] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 31, P: 0.358, R: 0.399, macro F1: 0.345, macro F1*: 0.377 -[INFO] QALD Cumulative metrics, sample: 31, P: 0.519, R: 0.399, macro F1: 0.345, QALD macro F1: 0.451 -[INFO] Current time: 864 ms, average: 4074.42 ms/ques, entity detection: 894.94 ms/ques (linking tool: 805.77 ms/ques), relation detection: 1266.65 ms/ques (similarity: 695.58 ms/ques), EDG generation: 153.61 ms/ques +[INFO] Cumulative metrics, sample: 31, P: 0.358, R: 0.399, macro F1: 0.377 +[INFO] QALD Cumulative metrics, sample: 31, P: 0.519, R: 0.399, average F1: 0.345, QALD macro F1: 0.451 +[INFO] Current question consumed time: 805 ms, average: 3805.45 ms/ques, entity detection: 820.13 ms/ques (linking tool: 738.16 ms/ques), relation detection: 1208.26 ms/ques (similarity: 703.74 ms/ques), EDG generation: 157.45 ms/ques [INFO] Entity Linking P: 0.1935483870967742, R: 0.1935483870967742, micro F1: 0.1935483870967742, macro F1: 0.1935483870967742 [INFO] Relation Linking P: 0.22580645161290322, R: 0.22580645161290322, micro F1: 0.22580645161290322, macro F1: 0.22580645161290322 [INFO] Type Linking P: 0.9354838709677419, R: 0.9354838709677419, micro F1: 0.9354838709677419, macro F1: 0.9354838709677419 @@ -472,9 +472,9 @@ Question 31: When did Finland join the EU? [INFO] Golden Answer: [1995-01-01] [INFO] Predicted Answer: [1809-03-29, 1917-12-06] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 32, P: 0.347, R: 0.386, macro F1: 0.334, macro F1*: 0.365 -[INFO] QALD Cumulative metrics, sample: 32, P: 0.503, R: 0.386, macro F1: 0.334, QALD macro F1: 0.437 -[INFO] Current time: 6961 ms, average: 4164.62 ms/ques, entity detection: 908.16 ms/ques (linking tool: 817.66 ms/ques), relation detection: 1305.28 ms/ques (similarity: 712.09 ms/ques), EDG generation: 149.06 ms/ques +[INFO] Cumulative metrics, sample: 32, P: 0.347, R: 0.386, macro F1: 0.365 +[INFO] QALD Cumulative metrics, sample: 32, P: 0.503, R: 0.386, average F1: 0.334, QALD macro F1: 0.437 +[INFO] Current question consumed time: 6635 ms, average: 3893.88 ms/ques, entity detection: 832.12 ms/ques (linking tool: 748.38 ms/ques), relation detection: 1246.53 ms/ques (similarity: 720.25 ms/ques), EDG generation: 152.78 ms/ques [INFO] Entity Linking P: 0.1875, R: 0.1875, micro F1: 0.1875, macro F1: 0.1875 [INFO] Relation Linking P: 0.21875, R: 0.21875, micro F1: 0.21875, macro F1: 0.21875 [INFO] Type Linking P: 0.9375, R: 0.9375, micro F1: 0.9375, macro F1: 0.9375 @@ -487,9 +487,9 @@ Question 32: Which actors play in Big Bang Theory? [INFO] Golden Answer: [http://dbpedia.org/resource/Johnny_Galecki, http://dbpedia.org/resource/Kunal_Nayyar, http://dbpedia.org/resource/Melissa_Rauch, http://dbpedia.org/resource/Simon_Helberg, http://dbpedia.org/resource/Kaley_Cuoco, http://dbpedia.org/resource/Mayim_Bialik, http://dbpedia.org/resource/Jim_Parsons, http://dbpedia.org/resource/Sara_Gilbert, http://dbpedia.org/resource/Laura_Spencer_(actress), http://dbpedia.org/resource/Kevin_Sussman] [INFO] Predicted Answer: [http://dbpedia.org/resource/Styx_(band)] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 33, P: 0.336, R: 0.374, macro F1: 0.324, macro F1*: 0.354 -[INFO] QALD Cumulative metrics, sample: 33, P: 0.488, R: 0.374, macro F1: 0.324, QALD macro F1: 0.424 -[INFO] Current time: 1126 ms, average: 4072.55 ms/ques, entity detection: 886.30 ms/ques (linking tool: 796.94 ms/ques), relation detection: 1274.79 ms/ques (similarity: 700.33 ms/ques), EDG generation: 144.79 ms/ques +[INFO] Cumulative metrics, sample: 33, P: 0.336, R: 0.374, macro F1: 0.354 +[INFO] QALD Cumulative metrics, sample: 33, P: 0.488, R: 0.374, average F1: 0.324, QALD macro F1: 0.424 +[INFO] Current question consumed time: 1179 ms, average: 3811.61 ms/ques, entity detection: 812.76 ms/ques (linking tool: 729.91 ms/ques), relation detection: 1218.36 ms/ques (similarity: 708.42 ms/ques), EDG generation: 148.45 ms/ques [INFO] Entity Linking P: 0.18181818181818182, R: 0.18181818181818182, micro F1: 0.18181818181818182, macro F1: 0.18181818181818182 [INFO] Relation Linking P: 0.21212121212121213, R: 0.21212121212121213, micro F1: 0.21212121212121213, macro F1: 0.21212121212121215 [INFO] Type Linking P: 0.9393939393939394, R: 0.9393939393939394, micro F1: 0.9393939393939394, macro F1: 0.9393939393939394 @@ -502,9 +502,9 @@ Question 33: Which computer scientist won an oscar? [INFO] Golden Answer: [http://dbpedia.org/resource/Edwin_Catmull] [INFO] Predicted Answer: [http://dbpedia.org/resource/Ramesh_Raskar, http://dbpedia.org/resource/Dinesh_Manocha, http://dbpedia.org/resource/Henry_Fuchs, http://dbpedia.org/resource/Ming_C._Lin, http://dbpedia.org/resource/John_Canny, http://dbpedia.org/resource/Michael_Reiter, http://dbpedia.org/resource/Maneesh_Agrawala, http://dbpedia.org/resource/Diane_Pozefsky] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 34, P: 0.326, R: 0.363, macro F1: 0.315, macro F1*: 0.344 -[INFO] QALD Cumulative metrics, sample: 34, P: 0.473, R: 0.363, macro F1: 0.315, QALD macro F1: 0.411 -[INFO] Current time: 1568 ms, average: 3998.88 ms/ques, entity detection: 867.38 ms/ques (linking tool: 776.65 ms/ques), relation detection: 1253.18 ms/ques (similarity: 695.74 ms/ques), EDG generation: 140.82 ms/ques +[INFO] Cumulative metrics, sample: 34, P: 0.326, R: 0.363, macro F1: 0.344 +[INFO] QALD Cumulative metrics, sample: 34, P: 0.473, R: 0.363, average F1: 0.315, QALD macro F1: 0.411 +[INFO] Current question consumed time: 1682 ms, average: 3748.97 ms/ques, entity detection: 796.97 ms/ques (linking tool: 711.88 ms/ques), relation detection: 1199.00 ms/ques (similarity: 703.21 ms/ques), EDG generation: 144.38 ms/ques [INFO] Entity Linking P: 0.17647058823529413, R: 0.17647058823529413, micro F1: 0.17647058823529413, macro F1: 0.17647058823529413 [INFO] Relation Linking P: 0.20588235294117646, R: 0.20588235294117646, micro F1: 0.20588235294117646, macro F1: 0.20588235294117646 [INFO] Type Linking P: 0.9411764705882353, R: 0.9411764705882353, micro F1: 0.9411764705882353, macro F1: 0.9411764705882353 @@ -517,9 +517,9 @@ Question 34: Who wrote Harry Potter? [INFO] Golden Answer: [http://dbpedia.org/resource/J._K._Rowling] [INFO] Predicted Answer: [http://dbpedia.org/resource/J._K._Rowling] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 35, P: 0.346, R: 0.382, macro F1: 0.334, macro F1*: 0.363 -[INFO] QALD Cumulative metrics, sample: 35, P: 0.488, R: 0.382, macro F1: 0.334, QALD macro F1: 0.428 -[INFO] Current time: 1325 ms, average: 3922.49 ms/ques, entity detection: 850.11 ms/ques (linking tool: 760.06 ms/ques), relation detection: 1232.51 ms/ques (similarity: 684.69 ms/ques), EDG generation: 137.03 ms/ques +[INFO] Cumulative metrics, sample: 35, P: 0.346, R: 0.382, macro F1: 0.363 +[INFO] QALD Cumulative metrics, sample: 35, P: 0.488, R: 0.382, average F1: 0.334, QALD macro F1: 0.428 +[INFO] Current question consumed time: 1374 ms, average: 3681.11 ms/ques, entity detection: 782.09 ms/ques (linking tool: 697.34 ms/ques), relation detection: 1180.03 ms/ques (similarity: 691.83 ms/ques), EDG generation: 140.40 ms/ques [INFO] Entity Linking P: 0.17142857142857143, R: 0.17142857142857143, micro F1: 0.17142857142857143, macro F1: 0.17142857142857143 [INFO] Relation Linking P: 0.2, R: 0.2, micro F1: 0.2, macro F1: 0.20000000000000004 [INFO] Type Linking P: 0.9428571428571428, R: 0.9428571428571428, micro F1: 0.9428571428571428, macro F1: 0.9428571428571428 @@ -532,9 +532,9 @@ Question 35: Give me all writers that won the Nobel Prize in literature. [INFO] Golden Answer: [http://dbpedia.org/resource/Alice_Munro, http://dbpedia.org/resource/Pablo_Neruda, http://dbpedia.org/resource/William_Golding, http://dbpedia.org/resource/Ernest_Hemingway, http://dbpedia.org/resource/Hermann_Hesse, http://dbpedia.org/resource/John_Steinbeck, http://dbpedia.org/resource/José_Saramago, http://dbpedia.org/resource/T._S._Eliot, http://dbpedia.org/resource/Thomas_Mann, http://dbpedia.org/resource/Eugene_O'Neill, http://dbpedia.org/resource/Toni_Morrison, http://dbpedia.org/resource/Aleksandr_Solzhenitsyn, http://dbpedia.org/resource/Czesław_Miłosz, http://dbpedia.org/resource/Seamus_Heaney, http://dbpedia.org/resource/Harold_Pinter, http://dbpedia.org/resource/Juan_Ramón_Jiménez, http://dbpedia.org/resource/Svetlana_Alexievich, http://dbpedia.org/resource/Joseph_Brodsky, http://dbpedia.org/resource/Patrick_Modiano, http://dbpedia.org/resource/Wisława_Szymborska, http://dbpedia.org/resource/Gerhart_Hauptmann] [INFO] Predicted Answer: [http://dbpedia.org/resource/Alice_Munro, http://dbpedia.org/resource/Pablo_Neruda, http://dbpedia.org/resource/William_Golding, http://dbpedia.org/resource/Bob_Dylan, http://dbpedia.org/resource/Ernest_Hemingway, http://dbpedia.org/resource/Hermann_Hesse, http://dbpedia.org/resource/John_Steinbeck, http://dbpedia.org/resource/José_Saramago, http://dbpedia.org/resource/T._S._Eliot, http://dbpedia.org/resource/Thomas_Mann, http://dbpedia.org/resource/Eugene_O'Neill, http://dbpedia.org/resource/Toni_Morrison, http://dbpedia.org/resource/Aleksandr_Solzhenitsyn, http://dbpedia.org/resource/Czesław_Miłosz, http://dbpedia.org/resource/Seamus_Heaney, http://dbpedia.org/resource/Harold_Pinter, http://dbpedia.org/resource/Ivo_Andrić, http://dbpedia.org/resource/Juan_Ramón_Jiménez, http://dbpedia.org/resource/Svetlana_Alexievich, http://dbpedia.org/resource/Joseph_Brodsky, http://dbpedia.org/resource/Patrick_Modiano, http://dbpedia.org/resource/Wisława_Szymborska, http://dbpedia.org/resource/Gerhart_Hauptmann, http://dbpedia.org/resource/List_of_awards_and_nominations_received_by_Bob_Dylan] [INFO] P: 0.875, R: 1.0, F: 0.9333333333333333 -[INFO] Cumulative metrics, sample: 36, P: 0.360, R: 0.399, macro F1: 0.351, macro F1*: 0.379 -[INFO] QALD Cumulative metrics, sample: 36, P: 0.499, R: 0.399, macro F1: 0.351, QALD macro F1: 0.443 -[INFO] Current time: 1731 ms, average: 3861.61 ms/ques, entity detection: 836.14 ms/ques (linking tool: 748.08 ms/ques), relation detection: 1207.83 ms/ques (similarity: 675.33 ms/ques), EDG generation: 133.86 ms/ques +[INFO] Cumulative metrics, sample: 36, P: 0.360, R: 0.399, macro F1: 0.379 +[INFO] QALD Cumulative metrics, sample: 36, P: 0.499, R: 0.399, average F1: 0.351, QALD macro F1: 0.443 +[INFO] Current question consumed time: 1677 ms, average: 3625.44 ms/ques, entity detection: 769.36 ms/ques (linking tool: 686.50 ms/ques), relation detection: 1156.42 ms/ques (similarity: 682.19 ms/ques), EDG generation: 136.92 ms/ques [INFO] Entity Linking P: 0.16666666666666666, R: 0.16666666666666666, micro F1: 0.16666666666666666, macro F1: 0.16666666666666666 [INFO] Relation Linking P: 0.19444444444444445, R: 0.19444444444444445, micro F1: 0.19444444444444445, macro F1: 0.19444444444444445 [INFO] Type Linking P: 0.9444444444444444, R: 0.9444444444444444, micro F1: 0.9444444444444444, macro F1: 0.9444444444444444 @@ -547,9 +547,9 @@ Question 36: Give me English actors starring in Lovesick. [INFO] Golden Answer: [http://dbpedia.org/resource/Dudley_Moore, http://dbpedia.org/resource/Alec_Guinness] [INFO] Predicted Answer: [http://dbpedia.org/resource/Ashley_Williams_(actress), http://dbpedia.org/resource/Chevy_Chase, http://dbpedia.org/resource/Matt_LeBlanc, http://dbpedia.org/resource/Rachael_Harris, http://dbpedia.org/resource/Ali_Larter, http://dbpedia.org/resource/Kristen_Johnston] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 37, P: 0.351, R: 0.388, macro F1: 0.341, macro F1*: 0.368 -[INFO] QALD Cumulative metrics, sample: 37, P: 0.486, R: 0.388, macro F1: 0.341, QALD macro F1: 0.431 -[INFO] Current time: 1222 ms, average: 3790.27 ms/ques, entity detection: 817.92 ms/ques (linking tool: 730.00 ms/ques), relation detection: 1184.14 ms/ques (similarity: 667.05 ms/ques), EDG generation: 132.57 ms/ques +[INFO] Cumulative metrics, sample: 37, P: 0.351, R: 0.388, macro F1: 0.368 +[INFO] QALD Cumulative metrics, sample: 37, P: 0.486, R: 0.388, average F1: 0.341, QALD macro F1: 0.431 +[INFO] Current question consumed time: 1111 ms, average: 3557.49 ms/ques, entity detection: 752.62 ms/ques (linking tool: 669.86 ms/ques), relation detection: 1134.43 ms/ques (similarity: 673.81 ms/ques), EDG generation: 133.59 ms/ques [INFO] Entity Linking P: 0.16216216216216217, R: 0.16216216216216217, micro F1: 0.16216216216216217, macro F1: 0.16216216216216217 [INFO] Relation Linking P: 0.1891891891891892, R: 0.1891891891891892, micro F1: 0.1891891891891892, macro F1: 0.18918918918918917 [INFO] Type Linking P: 0.9459459459459459, R: 0.9459459459459459, micro F1: 0.9459459459459459, macro F1: 0.9459459459459459 @@ -562,9 +562,9 @@ Question 37: What is the nick name of Baghdad? [INFO] Golden Answer: ['The City of Peace'@en] [INFO] Predicted Answer: ['The City of Peace'@en] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 38, P: 0.368, R: 0.404, macro F1: 0.359, macro F1*: 0.385 -[INFO] QALD Cumulative metrics, sample: 38, P: 0.499, R: 0.404, macro F1: 0.359, QALD macro F1: 0.447 -[INFO] Current time: 4017 ms, average: 3796.24 ms/ques, entity detection: 811.45 ms/ques (linking tool: 721.68 ms/ques), relation detection: 1203.76 ms/ques (similarity: 679.95 ms/ques), EDG generation: 129.39 ms/ques +[INFO] Cumulative metrics, sample: 38, P: 0.368, R: 0.404, macro F1: 0.385 +[INFO] QALD Cumulative metrics, sample: 38, P: 0.499, R: 0.404, average F1: 0.359, QALD macro F1: 0.447 +[INFO] Current question consumed time: 4008 ms, average: 3569.34 ms/ques, entity detection: 747.76 ms/ques (linking tool: 662.95 ms/ques), relation detection: 1156.03 ms/ques (similarity: 686.50 ms/ques), EDG generation: 130.39 ms/ques [INFO] Entity Linking P: 0.15789473684210525, R: 0.15789473684210525, micro F1: 0.15789473684210525, macro F1: 0.15789473684210525 [INFO] Relation Linking P: 0.18421052631578946, R: 0.18421052631578946, micro F1: 0.18421052631578946, macro F1: 0.18421052631578946 [INFO] Type Linking P: 0.9473684210526315, R: 0.9473684210526315, micro F1: 0.9473684210526315, macro F1: 0.9473684210526315 @@ -577,9 +577,9 @@ Question 38: In which city was the president of Montenegro born? [INFO] Golden Answer: [http://dbpedia.org/resource/Belgrade] [INFO] Predicted Answer: [http://dbpedia.org/resource/Belgrade] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 39, P: 0.384, R: 0.419, macro F1: 0.375, macro F1*: 0.401 -[INFO] QALD Cumulative metrics, sample: 39, P: 0.512, R: 0.419, macro F1: 0.375, QALD macro F1: 0.461 -[INFO] Current time: 1959 ms, average: 3749.13 ms/ques, entity detection: 798.13 ms/ques (linking tool: 710.21 ms/ques), relation detection: 1187.90 ms/ques (similarity: 678.33 ms/ques), EDG generation: 126.31 ms/ques +[INFO] Cumulative metrics, sample: 39, P: 0.384, R: 0.419, macro F1: 0.401 +[INFO] QALD Cumulative metrics, sample: 39, P: 0.512, R: 0.419, average F1: 0.375, QALD macro F1: 0.461 +[INFO] Current question consumed time: 1993 ms, average: 3528.92 ms/ques, entity detection: 736.05 ms/ques (linking tool: 652.97 ms/ques), relation detection: 1141.64 ms/ques (similarity: 684.67 ms/ques), EDG generation: 127.31 ms/ques [INFO] Entity Linking P: 0.15384615384615385, R: 0.15384615384615385, micro F1: 0.15384615384615385, macro F1: 0.15384615384615385 [INFO] Relation Linking P: 0.1794871794871795, R: 0.1794871794871795, micro F1: 0.1794871794871795, macro F1: 0.1794871794871795 [INFO] Type Linking P: 0.9230769230769231, R: 0.9230769230769231, micro F1: 0.9230769230769231, macro F1: 0.9230769230769231 @@ -592,9 +592,9 @@ Question 39: Which U.S. state has the highest population density? [INFO] Golden Answer: [http://dbpedia.org/resource/New_Jersey] [INFO] Predicted Answer: [] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 40, P: 0.374, R: 0.409, macro F1: 0.366, macro F1*: 0.391 -[INFO] QALD Cumulative metrics, sample: 40, P: 0.524, R: 0.409, macro F1: 0.366, QALD macro F1: 0.459 -[INFO] Current time: 5304 ms, average: 3788.00 ms/ques, entity detection: 821.38 ms/ques (linking tool: 732.25 ms/ques), relation detection: 1176.08 ms/ques (similarity: 662.95 ms/ques), EDG generation: 123.40 ms/ques +[INFO] Cumulative metrics, sample: 40, P: 0.374, R: 0.409, macro F1: 0.391 +[INFO] QALD Cumulative metrics, sample: 40, P: 0.524, R: 0.409, average F1: 0.366, QALD macro F1: 0.459 +[INFO] Current question consumed time: 5308 ms, average: 3573.40 ms/ques, entity detection: 760.02 ms/ques (linking tool: 675.83 ms/ques), relation detection: 1131.28 ms/ques (similarity: 669.27 ms/ques), EDG generation: 124.35 ms/ques [INFO] Entity Linking P: 0.175, R: 0.175, micro F1: 0.175, macro F1: 0.175 [INFO] Relation Linking P: 0.2, R: 0.2, micro F1: 0.2, macro F1: 0.20000000000000004 [INFO] Type Linking P: 0.925, R: 0.925, micro F1: 0.925, macro F1: 0.925 @@ -607,9 +607,9 @@ Question 40: What is the longest river in China? [INFO] Golden Answer: [http://dbpedia.org/resource/Yangtze] [INFO] Predicted Answer: [http://dbpedia.org/resource/Salween_River, http://dbpedia.org/resource/Xinfeng_River, http://dbpedia.org/resource/Parlung_Tsangpo, http://dbpedia.org/resource/Ghaghara, http://dbpedia.org/resource/Thao_River, http://dbpedia.org/resource/N'Mai_River, http://dbpedia.org/resource/Black_River_(Asia), http://dbpedia.org/resource/Karakash_River, http://dbpedia.org/resource/Second_Songhua_River, http://dbpedia.org/resource/Caopo_River, http://dbpedia.org/resource/Daxi_Creek, http://dbpedia.org/resource/Liuchong_River, http://dbpedia.org/resource/Liuxihe_River, http://dbpedia.org/resource/Loushui_River, http://dbpedia.org/resource/Muli_River, http://dbpedia.org/resource/Nanya_River, http://dbpedia.org/resource/Shiguan_River, http://dbpedia.org/resource/Xitiao_River, http://dbpedia.org/resource/Langqên_Zangbo, http://dbpedia.org/resource/Renhe_River, http://dbpedia.org/resource/Ili_River, http://dbpedia.org/resource/Irtysh_River, http://dbpedia.org/resource/Beipan_River, http://dbpedia.org/resource/Gui_River, http://dbpedia.org/resource/Yulong_River, http://dbpedia.org/resource/Gar_Tsangpo, http://dbpedia.org/resource/Sênggê_Zangbo, http://dbpedia.org/resource/Shweli_River, http://dbpedia.org/resource/Qin_River, http://dbpedia.org/resource/Manas_River, http://dbpedia.org/resource/Nam_Hka, http://dbpedia.org/resource/Qingshui_River_(Guizhou)] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 41, P: 0.365, R: 0.399, macro F1: 0.357, macro F1*: 0.381 -[INFO] QALD Cumulative metrics, sample: 41, P: 0.511, R: 0.399, macro F1: 0.357, QALD macro F1: 0.448 -[INFO] Current time: 8758 ms, average: 3909.22 ms/ques, entity detection: 848.46 ms/ques (linking tool: 759.07 ms/ques), relation detection: 1235.32 ms/ques (similarity: 687.85 ms/ques), EDG generation: 120.68 ms/ques +[INFO] Cumulative metrics, sample: 41, P: 0.365, R: 0.399, macro F1: 0.381 +[INFO] QALD Cumulative metrics, sample: 41, P: 0.511, R: 0.399, average F1: 0.357, QALD macro F1: 0.448 +[INFO] Current question consumed time: 8488 ms, average: 3693.27 ms/ques, entity detection: 785.32 ms/ques (linking tool: 701.22 ms/ques), relation detection: 1191.44 ms/ques (similarity: 694.27 ms/ques), EDG generation: 121.61 ms/ques [INFO] Entity Linking P: 0.17073170731707318, R: 0.17073170731707318, micro F1: 0.17073170731707318, macro F1: 0.17073170731707318 [INFO] Relation Linking P: 0.1951219512195122, R: 0.1951219512195122, micro F1: 0.1951219512195122, macro F1: 0.1951219512195122 [INFO] Type Linking P: 0.926829268292683, R: 0.926829268292683, micro F1: 0.926829268292683, macro F1: 0.926829268292683 @@ -622,9 +622,9 @@ Question 41: What is the area code of Berlin? [INFO] Golden Answer: [030] [INFO] Predicted Answer: [030] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 42, P: 0.380, R: 0.413, macro F1: 0.372, macro F1*: 0.396 -[INFO] QALD Cumulative metrics, sample: 42, P: 0.523, R: 0.413, macro F1: 0.372, QALD macro F1: 0.462 -[INFO] Current time: 6112 ms, average: 3961.67 ms/ques, entity detection: 854.50 ms/ques (linking tool: 765.45 ms/ques), relation detection: 1273.83 ms/ques (similarity: 707.12 ms/ques), EDG generation: 118.10 ms/ques +[INFO] Cumulative metrics, sample: 42, P: 0.380, R: 0.413, macro F1: 0.396 +[INFO] QALD Cumulative metrics, sample: 42, P: 0.523, R: 0.413, average F1: 0.372, QALD macro F1: 0.462 +[INFO] Current question consumed time: 6043 ms, average: 3749.21 ms/ques, entity detection: 792.21 ms/ques (linking tool: 708.14 ms/ques), relation detection: 1231.43 ms/ques (similarity: 713.29 ms/ques), EDG generation: 118.98 ms/ques [INFO] Entity Linking P: 0.16666666666666666, R: 0.16666666666666666, micro F1: 0.16666666666666666, macro F1: 0.16666666666666666 [INFO] Relation Linking P: 0.19047619047619047, R: 0.19047619047619047, micro F1: 0.19047619047619047, macro F1: 0.19047619047619047 [INFO] Type Linking P: 0.9285714285714286, R: 0.9285714285714286, micro F1: 0.9285714285714286, macro F1: 0.9285714285714286 @@ -637,9 +637,9 @@ Question 42: How many scientists graduated from an Ivy League university? [INFO] Golden Answer: [403] [INFO] Predicted Answer: [1, 9] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 43, P: 0.371, R: 0.404, macro F1: 0.364, macro F1*: 0.387 -[INFO] QALD Cumulative metrics, sample: 43, P: 0.511, R: 0.404, macro F1: 0.364, QALD macro F1: 0.451 -[INFO] Current time: 1632 ms, average: 3907.49 ms/ques, entity detection: 839.23 ms/ques (linking tool: 751.74 ms/ques), relation detection: 1254.47 ms/ques (similarity: 701.05 ms/ques), EDG generation: 115.74 ms/ques +[INFO] Cumulative metrics, sample: 43, P: 0.371, R: 0.404, macro F1: 0.387 +[INFO] QALD Cumulative metrics, sample: 43, P: 0.511, R: 0.404, average F1: 0.364, QALD macro F1: 0.451 +[INFO] Current question consumed time: 1519 ms, average: 3697.35 ms/ques, entity detection: 777.98 ms/ques (linking tool: 695.42 ms/ques), relation detection: 1212.72 ms/ques (similarity: 707.00 ms/ques), EDG generation: 116.58 ms/ques [INFO] Entity Linking P: 0.16279069767441862, R: 0.16279069767441862, micro F1: 0.16279069767441862, macro F1: 0.16279069767441862 [INFO] Relation Linking P: 0.18604651162790697, R: 0.18604651162790697, micro F1: 0.18604651162790697, macro F1: 0.18604651162790695 [INFO] Type Linking P: 0.9302325581395349, R: 0.9302325581395349, micro F1: 0.9302325581395349, macro F1: 0.9302325581395349 @@ -652,9 +652,9 @@ Question 43: Which professional surfers were born in Australia? [INFO] Golden Answer: [http://dbpedia.org/resource/Ian_Cairns, http://dbpedia.org/resource/Koby_Abberton] [INFO] Predicted Answer: [http://dbpedia.org/resource/Aaron_Heslehurst, http://dbpedia.org/resource/Andrew_Peacock, http://dbpedia.org/resource/Barry_O'Farrell, http://dbpedia.org/resource/Dylan_Lewis, http://dbpedia.org/resource/Emma_Brown_Garett, http://dbpedia.org/resource/George_Bailey_(cricketer,_born_1982), http://dbpedia.org/resource/Kyle_Sandilands, http://dbpedia.org/resource/Phil_Cleary, http://dbpedia.org/resource/Richard_Roxburgh, http://dbpedia.org/resource/Rod_Eddington, http://dbpedia.org/resource/Shane_Watson, http://dbpedia.org/resource/Tania_Zaetta, http://dbpedia.org/resource/Cosima_De_Vito, http://dbpedia.org/resource/Frank_Woodley, http://dbpedia.org/resource/Gerald_O'Collins, http://dbpedia.org/resource/Jason_Clare, http://dbpedia.org/resource/Mitchell_Starc, http://dbpedia.org/resource/Paul_Hester, http://dbpedia.org/resource/Pete_Murray, http://dbpedia.org/resource/Richard_McGregor, http://dbpedia.org/resource/Sally_Seltmann, http://dbpedia.org/resource/Stephen_Hatton, http://dbpedia.org/resource/Bryan_Brown, http://dbpedia.org/resource/Deidre_Brock, http://dbpedia.org/resource/E._J._Brady, http://dbpedia.org/resource/Johnny_Logan_(singer), http://dbpedia.org/resource/Judith_Wright, http://dbpedia.org/resource/Les_Murray_(poet), http://dbpedia.org/resource/Mary_Gilmore, http://dbpedia.org/resource/Robert_Adamson_(poet), http://dbpedia.org/resource/Adam_Zampa, http://dbpedia.org/resource/Albert_Fuller_Ellis, http://dbpedia.org/resource/Brett_Lee, http://dbpedia.org/resource/Frank_Reid_Murphy, http://dbpedia.org/resource/Gary_Sutton, http://dbpedia.org/resource/George_Treweek, http://dbpedia.org/resource/Paul_Mercurio, http://dbpedia.org/resource/Philip_Nitschke, http://dbpedia.org/resource/Ross_Emerson, http://dbpedia.org/resource/Tom_Moody, http://dbpedia.org/resource/Emmy_Kubainski, http://dbpedia.org/resource/James_McTeigue, http://dbpedia.org/resource/Julian_Savulescu, http://dbpedia.org/resource/Roy_De_Maistre, http://dbpedia.org/resource/Simon_Westaway, http://dbpedia.org/resource/Simon_Wincer, http://dbpedia.org/resource/Alice_Garner, http://dbpedia.org/resource/Atrios, http://dbpedia.org/resource/Gregor_Jordan, http://dbpedia.org/resource/Jocelyn_Moorhouse, http://dbpedia.org/resource/Liam_McIntyre, http://dbpedia.org/resource/Paddy_Bedford, http://dbpedia.org/resource/Queenie_McKenzie, http://dbpedia.org/resource/Rover_Thomas, http://dbpedia.org/resource/Sammy_Woods, http://dbpedia.org/resource/Shane_Jacobson, http://dbpedia.org/resource/Adam_Spreadbury-Maher, http://dbpedia.org/resource/Bob_Irwin, http://dbpedia.org/resource/Daniel_Keene, http://dbpedia.org/resource/Jared_Graves, http://dbpedia.org/resource/Ross_Milne, http://dbpedia.org/resource/Aurelio_Vidmar, http://dbpedia.org/resource/Jody_Rigby, http://dbpedia.org/resource/Shelley_Craft, http://dbpedia.org/resource/Dan_Kelly_(bushranger), http://dbpedia.org/resource/Kirsty_Sword_Gusmão, http://dbpedia.org/resource/Lawrence_Wells, http://dbpedia.org/resource/Wayne_Pygram, http://dbpedia.org/resource/David_Holmgren, http://dbpedia.org/resource/Dorothy_Cumming, http://dbpedia.org/resource/Jill_Bilcock, http://dbpedia.org/resource/Mike_Bradley_(politician), http://dbpedia.org/resource/Veronika_Jenet, http://dbpedia.org/resource/Richard_L._Hunter, http://dbpedia.org/resource/William_Smith_(shearer), http://dbpedia.org/resource/Barbara_Stone, http://dbpedia.org/resource/Cornthwaite_John_Hector, http://dbpedia.org/resource/Mat_Mladin, http://dbpedia.org/resource/Alfred_Gibson, http://dbpedia.org/resource/Neal_Blewett, http://dbpedia.org/resource/Peter_James_(cinematographer), http://dbpedia.org/resource/Corey_Page, http://dbpedia.org/resource/Kerry_Anne_Wells, http://dbpedia.org/resource/Ronald_Sydney_Nyholm, http://dbpedia.org/resource/Alfred_J._Goulding, http://dbpedia.org/resource/Bill_Alley, http://dbpedia.org/resource/Michael_Beckley, http://dbpedia.org/resource/Kram_(musician), http://dbpedia.org/resource/Richard_Grimsdale, http://dbpedia.org/resource/Wilberforce_Eaves, http://dbpedia.org/resource/Bukkcity, http://dbpedia.org/resource/C._Montague_Shaw, http://dbpedia.org/resource/Zora_Cross, http://dbpedia.org/resource/Blake_Williams, http://dbpedia.org/resource/Cam_Sinclair, http://dbpedia.org/resource/Ken_Freeman_(astronomer), http://dbpedia.org/resource/Malcolm_Milne, http://dbpedia.org/resource/Sianoa_Smit-McPhee, http://dbpedia.org/resource/Talia_Zucker, http://dbpedia.org/resource/Amanda_Drury, http://dbpedia.org/resource/Jim_Phillips, http://dbpedia.org/resource/Norman_Brearley, http://dbpedia.org/resource/Priscilla_Hon, http://dbpedia.org/resource/Stanley_Burbury, http://dbpedia.org/resource/Anna_Hruby, http://dbpedia.org/resource/Deborah_Kennedy, http://dbpedia.org/resource/Lisa_Crittenden, http://dbpedia.org/resource/Rebecca_Rigg, http://dbpedia.org/resource/Tony_Albert, http://dbpedia.org/resource/Andrew_Dent, http://dbpedia.org/resource/Luke_O'Dea_(footballer), http://dbpedia.org/resource/Mortimer_Menpes, http://dbpedia.org/resource/Owen_Dixon, http://dbpedia.org/resource/Ronald_Francis, http://dbpedia.org/resource/Bronwyn_Oliver, http://dbpedia.org/resource/Clara_Southern, http://dbpedia.org/resource/Robert_Klippel, http://dbpedia.org/resource/William_Piguenit, http://dbpedia.org/resource/Dani_Thompson, http://dbpedia.org/resource/Matt_Day, http://dbpedia.org/resource/Nathaniel_Dean, http://dbpedia.org/resource/Peter_Barter, http://dbpedia.org/resource/Peter_Parker_(umpire), http://dbpedia.org/resource/Alan_Griffin, http://dbpedia.org/resource/Charles_Taylor_(Queensland_politician), http://dbpedia.org/resource/Daryl_Melham, http://dbpedia.org/resource/Geoff_Buckland, http://dbpedia.org/resource/Gordon_Reid_(governor), http://dbpedia.org/resource/Harry_Quick, http://dbpedia.org/resource/Helen_Kroger, http://dbpedia.org/resource/Janice_Crosio, http://dbpedia.org/resource/Jill_Hall, http://dbpedia.org/resource/Jocelyn_Newman, http://dbpedia.org/resource/Kay_Elson, http://dbpedia.org/resource/Kay_Hull, http://dbpedia.org/resource/Robert_Hill_(Australian_politician), http://dbpedia.org/resource/Christine_Stephen-Daly, http://dbpedia.org/resource/Cindy_Watson, http://dbpedia.org/resource/Tracey_Gaudry, http://dbpedia.org/resource/Jay_Watson, http://dbpedia.org/resource/Johnny_Jarrett, http://dbpedia.org/resource/Matthew_Newton, http://dbpedia.org/resource/Aaron_Downes, http://dbpedia.org/resource/Chris_Duggan, http://dbpedia.org/resource/Rachel_Khedoori, http://dbpedia.org/resource/Robert_Stambolziev, http://dbpedia.org/resource/Rodney_Martin_(squash_player), http://dbpedia.org/resource/Alex_Perry, http://dbpedia.org/resource/Andrew_Supanz, http://dbpedia.org/resource/Fletcher_Humphrys, http://dbpedia.org/resource/Heather_Mitchell, http://dbpedia.org/resource/Jack_Campbell_(actor), http://dbpedia.org/resource/Jenni_Baird, http://dbpedia.org/resource/Jessica_Gomes, http://dbpedia.org/resource/Luke_Pegler, http://dbpedia.org/resource/Rupert_Myer, http://dbpedia.org/resource/Nick_Rizzo, http://dbpedia.org/resource/John_Cooper_(tennis), http://dbpedia.org/resource/Ian_Pratt_(politician), http://dbpedia.org/resource/Lloyd_Mash, http://dbpedia.org/resource/Stephen_Milosz, http://dbpedia.org/resource/Andrew_Fraser_(Queensland_politician), http://dbpedia.org/resource/Craig_Laundy, http://dbpedia.org/resource/David_Coleman_(Australian_politician), http://dbpedia.org/resource/Irene_Crespin, http://dbpedia.org/resource/Jack_Metcalfe, http://dbpedia.org/resource/Kate_Jones, http://dbpedia.org/resource/Kevin_Newman_(politician), http://dbpedia.org/resource/Michael_Brand, http://dbpedia.org/resource/Nickolas_Varvaris, http://dbpedia.org/resource/Paul_McLeay, http://dbpedia.org/resource/Sue_Napier, http://dbpedia.org/resource/Ben_Judge, http://dbpedia.org/resource/Glen_Wilkinson, http://dbpedia.org/resource/Rod_Hardy, http://dbpedia.org/resource/Steve_Kilburn, http://dbpedia.org/resource/Steve_Mifsud, http://dbpedia.org/resource/Craig_Porteils, http://dbpedia.org/resource/Gary_Eck, http://dbpedia.org/resource/Jamie_Aditya, http://dbpedia.org/resource/Maike_Evers, http://dbpedia.org/resource/Peter_Hendy_(politician), http://dbpedia.org/resource/Belinda_Luscombe, http://dbpedia.org/resource/Bob_Joshua, http://dbpedia.org/resource/Frank_Winterstein, http://dbpedia.org/resource/Hannah_Campbell-Pegg, http://dbpedia.org/resource/Matt_Parish, http://dbpedia.org/resource/Rebecca_Brown_(swimmer), http://dbpedia.org/resource/Belinda_Wollaston, http://dbpedia.org/resource/Brett_Climo, http://dbpedia.org/resource/Graham_Harris, http://dbpedia.org/resource/Josephine_Mitchell, http://dbpedia.org/resource/Leila_Hayes, http://dbpedia.org/resource/Paul_Gibson_(politician), http://dbpedia.org/resource/Jim_Houghton_(politician), http://dbpedia.org/resource/Hugh_McClure_Smith, http://dbpedia.org/resource/Leif_Podhajsky, http://dbpedia.org/resource/Brad_Maloney, http://dbpedia.org/resource/Buddy_Farah, http://dbpedia.org/resource/Matthew_Davies_(footballer), http://dbpedia.org/resource/Dominic_del_Rosario, http://dbpedia.org/resource/Wendy_Kingston, http://dbpedia.org/resource/David_Hunter_(New_South_Wales_politician), http://dbpedia.org/resource/Gareth_Yuen, http://dbpedia.org/resource/Greg_Mail, http://dbpedia.org/resource/Ron_Allum, http://dbpedia.org/resource/Tom_Hackett, http://dbpedia.org/resource/Tom_Hornsey, http://dbpedia.org/resource/Max_Oldmeadow, http://dbpedia.org/resource/Andrew_Scott_(cricketer), http://dbpedia.org/resource/Bryanne_Stewart, http://dbpedia.org/resource/Daniel_Pollock, http://dbpedia.org/resource/Elias_C._Laycock, http://dbpedia.org/resource/Keith_Sullivan, http://dbpedia.org/resource/Tai_McIsaac, http://dbpedia.org/resource/Craig_Fong, http://dbpedia.org/resource/Joseph_Heller_(zoologist), http://dbpedia.org/resource/Linda_Marlowe, http://dbpedia.org/resource/Richard_Face, http://dbpedia.org/resource/Chloe_McConville, http://dbpedia.org/resource/Felicia_Atkins, http://dbpedia.org/resource/Carolyn_Male, http://dbpedia.org/resource/Chris_Cummins_(politician), http://dbpedia.org/resource/Chris_Foley_(Australian_politician), http://dbpedia.org/resource/Christine_Smith_(politician), http://dbpedia.org/resource/Darryl_Briskey, http://dbpedia.org/resource/Dianne_Reilly, http://dbpedia.org/resource/Gary_Fenlon, http://dbpedia.org/resource/Jacob_Butler, http://dbpedia.org/resource/Jan_Jarratt, http://dbpedia.org/resource/Jason_O'Brien, http://dbpedia.org/resource/John_English_(Australian_politician), http://dbpedia.org/resource/Karen_Struthers, http://dbpedia.org/resource/Liddy_Clark, http://dbpedia.org/resource/Lindy_Nelson-Carr, http://dbpedia.org/resource/Marc_Rowell, http://dbpedia.org/resource/Neil_Roberts_(politician), http://dbpedia.org/resource/Nita_Cunningham, http://dbpedia.org/resource/Pat_Purcell, http://dbpedia.org/resource/Paul_Hoolihan, http://dbpedia.org/resource/Peta-Kaye_Croft, http://dbpedia.org/resource/Phil_Reeves_(politician), http://dbpedia.org/resource/Rachel_Nolan, http://dbpedia.org/resource/Stuart_Copeland, http://dbpedia.org/resource/Ted_Malone_(Australian_politician), http://dbpedia.org/resource/Terry_Sullivan_(Australian_politician), http://dbpedia.org/resource/Jack_Clisby, http://dbpedia.org/resource/Julie_Rrap, http://dbpedia.org/resource/Kim_So-hyun, http://dbpedia.org/resource/Aaron_Kemps, http://dbpedia.org/resource/Carly_Wilson, http://dbpedia.org/resource/Dellys_Starr, http://dbpedia.org/resource/Ellen_Pettitt, http://dbpedia.org/resource/Erika_Yamasaki, http://dbpedia.org/resource/Erin_Carroll, http://dbpedia.org/resource/Jeremy_Roff, http://dbpedia.org/resource/Jobie_Dajka, http://dbpedia.org/resource/Lisa_Corrigan, http://dbpedia.org/resource/Mark_Jamieson, http://dbpedia.org/resource/Natasha_Barker, http://dbpedia.org/resource/Prashanth_Sellathurai, http://dbpedia.org/resource/Peter_Browne_(Australian_politician), http://dbpedia.org/resource/Daniel_Nilsson, http://dbpedia.org/resource/Holly_Brisley, http://dbpedia.org/resource/Iya_Villania, http://dbpedia.org/resource/Jonathan_Little_(composer), http://dbpedia.org/resource/Nicole_Schmitz, http://dbpedia.org/resource/Scott_Burgess_(actor), http://dbpedia.org/resource/Samantha_Lane, http://dbpedia.org/resource/Kristine_Bayley, http://dbpedia.org/resource/Liana_Cornell, http://dbpedia.org/resource/Mandy_Johnstone, http://dbpedia.org/resource/Miles_Scotson, http://dbpedia.org/resource/Cassi_Van_Den_Dungen, http://dbpedia.org/resource/Gemma_Sanderson, http://dbpedia.org/resource/George_Lambadaridis, http://dbpedia.org/resource/Kelsey_Martinovich, http://dbpedia.org/resource/Samantha_Downie, http://dbpedia.org/resource/Sophie_Van_Den_Akker, http://dbpedia.org/resource/Richard_Gray_(director), http://dbpedia.org/resource/1989_Davao_hostage_crisis, http://dbpedia.org/resource/Albert_Planta, http://dbpedia.org/resource/Alex_Read, http://dbpedia.org/resource/Jasmine_Curtis-Smith, http://dbpedia.org/resource/Andrew_Dykes_(cricketer), http://dbpedia.org/resource/Grace_Harris, http://dbpedia.org/resource/Nicole_Bolton, http://dbpedia.org/resource/Sarah_Elliott_(cricketer), http://dbpedia.org/resource/Bianca_Reddy, http://dbpedia.org/resource/Norma_Plummer, http://dbpedia.org/resource/Rel_Hunt, http://dbpedia.org/resource/Stephen_Hoiles, http://dbpedia.org/resource/Carla_Ryan, http://dbpedia.org/resource/Carlee_Taylor, http://dbpedia.org/resource/Kimberley_Wells_(cyclist), http://dbpedia.org/resource/Jack_Hill_(cricketer), http://dbpedia.org/resource/Jeff_Hammond_(cricketer), http://dbpedia.org/resource/Jeff_Moss_(cricketer), http://dbpedia.org/resource/Kain_O'Keeffe, http://dbpedia.org/resource/Kevin_Wright, http://dbpedia.org/resource/Len_Johnson_(cricketer), http://dbpedia.org/resource/Tony_Taylor_(GC), http://dbpedia.org/resource/Esther_Bottomley, http://dbpedia.org/resource/Norman_Greenwood, http://dbpedia.org/resource/Paul_Reid_(footballer,_born_1979), http://dbpedia.org/resource/Matt_Doran_(journalist), http://dbpedia.org/resource/Norm_Ryan, http://dbpedia.org/resource/Raymond_Hogan, http://dbpedia.org/resource/Bill_Lickiss, http://dbpedia.org/resource/Ben_Griffin_(footballer), http://dbpedia.org/resource/Bill_Baumann, http://dbpedia.org/resource/Bob_Dollin, http://dbpedia.org/resource/Bob_Harper_(politician), http://dbpedia.org/resource/Brendon_Šantalab, http://dbpedia.org/resource/Brian_Littleproud, http://dbpedia.org/resource/Bruce_Davidson_(Australian_politician), http://dbpedia.org/resource/Christine_Scott, http://dbpedia.org/resource/Corey_Brown_(footballer), http://dbpedia.org/resource/Doug_Slack, http://dbpedia.org/resource/Graham_Healy, http://dbpedia.org/resource/Grant_Musgrove, http://dbpedia.org/resource/John_Goss_(politician), http://dbpedia.org/resource/Ken_McElligott, http://dbpedia.org/resource/Len_Stephan, http://dbpedia.org/resource/Luke_Woolmer, http://dbpedia.org/resource/Lyn_Warwick, http://dbpedia.org/resource/Mark_Stoneman_(politician), http://dbpedia.org/resource/Merri_Rose, http://dbpedia.org/resource/Nick_Kruger, http://dbpedia.org/resource/Ray_Connor, http://dbpedia.org/resource/Rob_Mitchell_(Queensland_politician), http://dbpedia.org/resource/Robert_Poole_(politician), http://dbpedia.org/resource/Ted_Radke, http://dbpedia.org/resource/Tony_Elliott_(politician), http://dbpedia.org/resource/Trevor_Strong, http://dbpedia.org/resource/Vince_Lester, http://dbpedia.org/resource/William_Baragwanath, http://dbpedia.org/resource/Beryce_Nelson, http://dbpedia.org/resource/Bob_Gardner_(Queensland_politician), http://dbpedia.org/resource/Doug_Sherrington, http://dbpedia.org/resource/Fred_Campbell_(Australian_politician), http://dbpedia.org/resource/George_Marriott, http://dbpedia.org/resource/Rob_Akers, http://dbpedia.org/resource/Alec_Urosevski, http://dbpedia.org/resource/Anja_Nissen, http://dbpedia.org/resource/Dan_Feuerriegel, http://dbpedia.org/resource/Kaden_Hensel, http://dbpedia.org/resource/Nathan_Hall_(football_coach), http://dbpedia.org/resource/Spase_Dilevski, http://dbpedia.org/resource/Yvonne_Chapman, http://dbpedia.org/resource/Clive_Berghofer, http://dbpedia.org/resource/John_Warner_(Australian_politician), http://dbpedia.org/resource/Leah_Poulton, http://dbpedia.org/resource/Peter_Wood_(Australian_politician), http://dbpedia.org/resource/Ray_Bousen, http://dbpedia.org/resource/Rocky_Visconte, http://dbpedia.org/resource/Víctor_García_(Spanish_singer), http://dbpedia.org/resource/Daniel_Falzon, http://dbpedia.org/resource/Joel_Parsons, http://dbpedia.org/resource/Kyle_Nix, http://dbpedia.org/resource/Mark_Brusasco, http://dbpedia.org/resource/Benjamin_Day_(cyclist), http://dbpedia.org/resource/David_McKenzie_(cyclist), http://dbpedia.org/resource/Bill_Chaffey, http://dbpedia.org/resource/David_Grant_(cricketer), http://dbpedia.org/resource/Frank_Mitchell_(sportsman_born_1922), http://dbpedia.org/resource/George_Keegan, http://dbpedia.org/resource/Greg_Warren_(politician), http://dbpedia.org/resource/Peter_Toms, http://dbpedia.org/resource/Steve_Kamper, http://dbpedia.org/resource/Xu_Rongmao, http://dbpedia.org/resource/Rex_Cramphorn, http://dbpedia.org/resource/Clive_Boughton, http://dbpedia.org/resource/Brenna_Harding, http://dbpedia.org/resource/George_Golding, http://dbpedia.org/resource/Steve_Cooney, http://dbpedia.org/resource/Tara_Rushton, http://dbpedia.org/resource/Chloe_Hosking, http://dbpedia.org/resource/Christopher_Elliott_(actor), http://dbpedia.org/resource/James_Lemke, http://dbpedia.org/resource/Bryan_Staring, http://dbpedia.org/resource/James_Carlton_(athlete), http://dbpedia.org/resource/Chris_Brooks_(guitarist), http://dbpedia.org/resource/Brad_Knowles, http://dbpedia.org/resource/Chris_Duval, http://dbpedia.org/resource/Darren_McNees, http://dbpedia.org/resource/Darren_Wates, http://dbpedia.org/resource/Josh_Mangan, http://dbpedia.org/resource/Mark_Cameron_(cricketer), http://dbpedia.org/resource/Nathan_Wegman, http://dbpedia.org/resource/Rhett_Lockyear, http://dbpedia.org/resource/Alan_Grant_(darts_player), http://dbpedia.org/resource/Glenn_Scott, http://dbpedia.org/resource/Horrie_Seden, http://dbpedia.org/resource/Kevin_White_(darts_player), http://dbpedia.org/resource/Chris_Clearihan, http://dbpedia.org/resource/Gerald_Arthur, http://dbpedia.org/resource/Belinda_Goss, http://dbpedia.org/resource/Catherine_Brown_(footballer), http://dbpedia.org/resource/Julie_Sutton_(mayor), http://dbpedia.org/resource/Phillip_Thuaux, http://dbpedia.org/resource/Edward_Ronald_Walker, http://dbpedia.org/resource/Dean_Windsor, http://dbpedia.org/resource/Emma_Mackie, http://dbpedia.org/resource/Emma_Rickards, http://dbpedia.org/resource/Rawdon_Dalrymple, http://dbpedia.org/resource/Christina_Collard, http://dbpedia.org/resource/Joel_O'Keeffe, http://dbpedia.org/resource/Ryan_O'Keeffe, http://dbpedia.org/resource/Constance_Elizabeth_D'Arcy, http://dbpedia.org/resource/Danielle_Prince, http://dbpedia.org/resource/Peter_Henderson_(cricketer), http://dbpedia.org/resource/Alfred_Johns, http://dbpedia.org/resource/Arch_Nicholson, http://dbpedia.org/resource/Reece_Caira, http://dbpedia.org/resource/Arthur_Watson_(umpire), http://dbpedia.org/resource/Johnny_Bulunbulun, http://dbpedia.org/resource/Paul_Carroll_(volleyball), http://dbpedia.org/resource/John_Raymond_Garrett, http://dbpedia.org/resource/Danny_McLauchlan, http://dbpedia.org/resource/Matt_Johnston, http://dbpedia.org/resource/John_McNeill_(speedway_rider), http://dbpedia.org/resource/László_Varga_(footballer), http://dbpedia.org/resource/Paul_Davies_(artist), http://dbpedia.org/resource/Thomas_Lacey, http://dbpedia.org/resource/Ciara_Conway_(footballer), http://dbpedia.org/resource/Dean_Herbert_(developer), http://dbpedia.org/resource/Ellvana_Curo, http://dbpedia.org/resource/Bill_Bryson_(politician), http://dbpedia.org/resource/Peter_Baines_(footballer), http://dbpedia.org/resource/Alex_Dewar, http://dbpedia.org/resource/Bill_Gunn_(Queensland_politician), http://dbpedia.org/resource/Clive_Hughes_(Queensland_politician), http://dbpedia.org/resource/Frank_Roberts_(Australian_politician), http://dbpedia.org/resource/Harold_Hartley_(Australian_politician), http://dbpedia.org/resource/James_Clark_(Australian_politician), http://dbpedia.org/resource/James_Crawford_(Australian_politician), http://dbpedia.org/resource/James_Fry_(Australian_politician), http://dbpedia.org/resource/Jim_Burrows_(Queensland_politician), http://dbpedia.org/resource/John_Donnelly_(Australian_politician), http://dbpedia.org/resource/John_Hayes_(Queensland_politician), http://dbpedia.org/resource/John_Kessell_(politician), http://dbpedia.org/resource/John_Row_(Australian_politician), http://dbpedia.org/resource/Kerry_Copley, http://dbpedia.org/resource/Ted_Harris_(Queensland_politician), http://dbpedia.org/resource/Andrew_Pasterfield, http://dbpedia.org/resource/Francesco_Stella_(footballer), http://dbpedia.org/resource/Grace_Gill, http://dbpedia.org/resource/Jenna_McCormick, http://dbpedia.org/resource/Michael_Simcoe, http://dbpedia.org/resource/Nickoletta_Flannery, http://dbpedia.org/resource/Trudy_Burke, http://dbpedia.org/resource/Steve_Hall_(Canadian_football), http://dbpedia.org/resource/Dan_Macarthur, http://dbpedia.org/resource/Julie_Stockton, http://dbpedia.org/resource/Karen_Price_(cricketer), http://dbpedia.org/resource/William_Cooper_(Australian_politician), http://dbpedia.org/resource/Harry_Lambert, http://dbpedia.org/resource/Eddie_Illingworth, http://dbpedia.org/resource/Goulburn_Watsford, http://dbpedia.org/resource/Jeff_Hallebone, http://dbpedia.org/resource/Paul_Garlick_(Australian_cricketer), http://dbpedia.org/resource/Aaron_Frankcomb, http://dbpedia.org/resource/Britt_Simmons, http://dbpedia.org/resource/Donna_Cockayne, http://dbpedia.org/resource/Enza_Barilla, http://dbpedia.org/resource/Georgia_Yeoman-Dale, http://dbpedia.org/resource/Isabel_Hodgson, http://dbpedia.org/resource/Kim_Carroll_(footballer), http://dbpedia.org/resource/Lauren_Steer, http://dbpedia.org/resource/Natalie_Tobin, http://dbpedia.org/resource/Olivia_Price_(footballer), http://dbpedia.org/resource/Rachel_Alonso, http://dbpedia.org/resource/Tenneille_Boaler, http://dbpedia.org/resource/Tiarn_Powell, http://dbpedia.org/resource/Phil_Gray_(politician), http://dbpedia.org/resource/Shanrah_Wakefield, http://dbpedia.org/resource/David_Austin_(actor), http://dbpedia.org/resource/Peta_Mullens, http://dbpedia.org/resource/Brooke_Starrett, http://dbpedia.org/resource/Emma_Stewart, http://dbpedia.org/resource/Karina_Roweth, http://dbpedia.org/resource/Cameron_Merchant, http://dbpedia.org/resource/Hayley_McElhinney, http://dbpedia.org/resource/Helen_Scott_(actress), http://dbpedia.org/resource/Angela_Pulvirenti, http://dbpedia.org/resource/Doug_Hodgson, http://dbpedia.org/resource/Riley_Nottingham, http://dbpedia.org/resource/Charlotte_Nicdao, http://dbpedia.org/resource/Richard_Campbell_(water_polo), http://dbpedia.org/resource/Nicole_Hunt, http://dbpedia.org/resource/Erika_Elze, http://dbpedia.org/resource/Basil_Hansen, http://dbpedia.org/resource/Daniela_Di_Bartolo, http://dbpedia.org/resource/Matthew_Lehoczky, http://dbpedia.org/resource/Terry_Prindiville, http://dbpedia.org/resource/Brandon_Lundy, http://dbpedia.org/resource/Olly_Simpson, http://dbpedia.org/resource/Death_of_Luke_Borusiewicz, http://dbpedia.org/resource/Thom_Crawford, http://dbpedia.org/resource/Joe_Lesina, http://dbpedia.org/resource/Paul_Worsteling, http://dbpedia.org/resource/Basenji_(producer), http://dbpedia.org/resource/Claire_Elizabeth_Parker, http://dbpedia.org/resource/Osmond_Charles_Fuhrman, http://dbpedia.org/resource/Merv_Mueller, http://dbpedia.org/resource/Fergus_Yeates, http://dbpedia.org/resource/Keith_Ziebell, http://dbpedia.org/resource/Tyler_Martin, http://dbpedia.org/resource/Clements_Frederick_Vivian_Jackson, http://dbpedia.org/resource/Jada_Mathyssen-Whyman, http://dbpedia.org/resource/Justin_Truloff, http://dbpedia.org/resource/Robert_Puca, http://dbpedia.org/resource/Ben_Spillane, http://dbpedia.org/resource/Dave_Dillewaard, http://dbpedia.org/resource/Kerry_Norton-Smyser, http://dbpedia.org/resource/Nicole_Rogerson, http://dbpedia.org/resource/Tracie_McGovern, http://dbpedia.org/resource/Stephen_Benwell, http://dbpedia.org/resource/Vince_Vozzo, http://dbpedia.org/resource/Geoffrey_Harley_Mewton, http://dbpedia.org/resource/Graham_Rowley_(cyclist), http://dbpedia.org/resource/Laurie_Venn, http://dbpedia.org/resource/Terry_Hammond, http://dbpedia.org/resource/Henry_Walkerden, http://dbpedia.org/resource/Paul_Zschorn, http://dbpedia.org/resource/Jarryd_Hayne__Jarryd_Hayne__1, http://dbpedia.org/resource/John_Purdy_(chess_player)__John_Purdy__1, http://dbpedia.org/resource/Adam_Gilchrist, http://dbpedia.org/resource/Andy_Bichel, http://dbpedia.org/resource/Angus_Bethune_(politician), http://dbpedia.org/resource/Ashton_Agar, http://dbpedia.org/resource/Clive_Churchill, http://dbpedia.org/resource/Damien_Martyn, http://dbpedia.org/resource/Darren_Lehmann, http://dbpedia.org/resource/Eric_Bana, http://dbpedia.org/resource/Glenn_Rogers, http://dbpedia.org/resource/Ian_Healy, http://dbpedia.org/resource/Ian_Ross_(newsreader), http://dbpedia.org/resource/Jack_Gregory_(cricketer), http://dbpedia.org/resource/Jeff_Thomson, http://dbpedia.org/resource/John_Truscott, http://dbpedia.org/resource/Johnny_Raper, http://dbpedia.org/resource/Jonathon_Dutton, http://dbpedia.org/resource/Ken_Muggleston, http://dbpedia.org/resource/Kerry_Packer, http://dbpedia.org/resource/Malcolm_Fraser, http://dbpedia.org/resource/Mark_Waugh, http://dbpedia.org/resource/Matthew_Hayden, http://dbpedia.org/resource/Maxwell_Fuller, http://dbpedia.org/resource/Merv_Hughes, http://dbpedia.org/resource/Monty_Noble, http://dbpedia.org/resource/P._L._Travers, http://dbpedia.org/resource/Peter_Singer, http://dbpedia.org/resource/Ricky_Ponting, http://dbpedia.org/resource/Robbie_Middleby, http://dbpedia.org/resource/Shawn_Mackay, http://dbpedia.org/resource/Steve_Waugh, http://dbpedia.org/resource/Stuart_O'Grady, http://dbpedia.org/resource/Vali_Myers, http://dbpedia.org/resource/Victor_Trumper, http://dbpedia.org/resource/Warwick_Armstrong, http://dbpedia.org/resource/William_Wentworth, http://dbpedia.org/resource/Bennelong, http://dbpedia.org/resource/Bert_Oldfield, http://dbpedia.org/resource/Clement_Meadmore, http://dbpedia.org/resource/Craig_Revel_Horwood, http://dbpedia.org/resource/David_Chalmers, http://dbpedia.org/resource/David_Johnston_(politician), http://dbpedia.org/resource/David_Williamson, http://dbpedia.org/resource/Dianne_Fromholtz, http://dbpedia.org/resource/Harry_Bridges, http://dbpedia.org/resource/Ian_Harvey, http://dbpedia.org/resource/J._L._Mackie, http://dbpedia.org/resource/Jessica_Mauboy, http://dbpedia.org/resource/John_Williams_(guitarist), http://dbpedia.org/resource/Luke_Mitchell, http://dbpedia.org/resource/Mark_Milligan, http://dbpedia.org/resource/Nick_Kyrgios, http://dbpedia.org/resource/Paul_Reiffel, http://dbpedia.org/resource/Phil_Jamieson, http://dbpedia.org/resource/Philip_K._Chapman, http://dbpedia.org/resource/Rachael_Finch, http://dbpedia.org/resource/Reg_Lindsay, http://dbpedia.org/resource/Richard_Johnson_(footballer), http://dbpedia.org/resource/Roger_Donaldson, http://dbpedia.org/resource/Rolf_Harris, http://dbpedia.org/resource/Talitha_Cummins, http://dbpedia.org/resource/Ted_a'Beckett, http://dbpedia.org/resource/Yemmerrawanne, http://dbpedia.org/resource/Arthur_Chesterfield-Evans, http://dbpedia.org/resource/David_Winderlich, http://dbpedia.org/resource/Lionel_Murphy, http://dbpedia.org/resource/Mal_Colston, http://dbpedia.org/resource/V._Gordon_Childe, http://dbpedia.org/resource/Ainslie_Meares, http://dbpedia.org/resource/Alfred_Winslow_Jones, http://dbpedia.org/resource/John_Peter_Russell, http://dbpedia.org/resource/Richard_Butler_(diplomat), http://dbpedia.org/resource/Robbie_Farah, http://dbpedia.org/resource/Bruce_Reid, http://dbpedia.org/resource/Charli_Robinson, http://dbpedia.org/resource/Christopher_Egan, http://dbpedia.org/resource/Darrell_Hair, http://dbpedia.org/resource/David_Hawker, http://dbpedia.org/resource/David_Marr_(journalist), http://dbpedia.org/resource/Dean_Jones_(cricketer), http://dbpedia.org/resource/Etienne_L._de_Mestre, http://dbpedia.org/resource/Frank_McEncroe, http://dbpedia.org/resource/Frank_Reys, http://dbpedia.org/resource/Freda_Du_Faur, http://dbpedia.org/resource/Graeme_Lloyd, http://dbpedia.org/resource/Graeme_Rutjes, http://dbpedia.org/resource/Greg_Coffey, http://dbpedia.org/resource/Greg_Dyer, http://dbpedia.org/resource/Greg_Page_(musician), http://dbpedia.org/resource/John_Cutts_(jockey), http://dbpedia.org/resource/John_Edwards_(producer), http://dbpedia.org/resource/John_Lill_(cricketer), http://dbpedia.org/resource/Jon_Finlayson, http://dbpedia.org/resource/Ken_Irvine, http://dbpedia.org/resource/Lloyd_Rees, http://dbpedia.org/resource/Mary,_Crown_Princess_of_Denmark, http://dbpedia.org/resource/Matthew_Nable, http://dbpedia.org/resource/Michael_Beauchamp, http://dbpedia.org/resource/Michael_Joseph_Savage, http://dbpedia.org/resource/Mike_Hollands, http://dbpedia.org/resource/Napoleon_Perdis, http://dbpedia.org/resource/Patricia_Glassop, http://dbpedia.org/resource/Peter_St._Albans, http://dbpedia.org/resource/Richard_Lowenstein, http://dbpedia.org/resource/Rodney_Hogg, http://dbpedia.org/resource/Steve_Vizard, http://dbpedia.org/resource/Ufuk_Talay, http://dbpedia.org/resource/Veronica_Seton-Williams, http://dbpedia.org/resource/Wally_Grout, http://dbpedia.org/resource/Wayne_Gardner, http://dbpedia.org/resource/Yael_Stone, http://dbpedia.org/resource/Jaynie_Seal, http://dbpedia.org/resource/Scott_Stephenson, http://dbpedia.org/resource/Alf_Goullet, http://dbpedia.org/resource/Axle_Whitehead, http://dbpedia.org/resource/Brian_Carlton, http://dbpedia.org/resource/Danny_Clark_(cyclist), http://dbpedia.org/resource/Jack_Bobridge, http://dbpedia.org/resource/Rohan_Dennis, http://dbpedia.org/resource/Ryan_Bayley, http://dbpedia.org/resource/Sheila_Fitzpatrick, http://dbpedia.org/resource/Jordan_Lyden, http://dbpedia.org/resource/Lyndall_Hobbs, http://dbpedia.org/resource/Virgil_Donati, http://dbpedia.org/resource/Arthur_Mailey, http://dbpedia.org/resource/Bruce_Small, http://dbpedia.org/resource/Joel_Clinton, http://dbpedia.org/resource/John_McCarthy_(ambassador_born_1947), http://dbpedia.org/resource/Luke_Kennedy, http://dbpedia.org/resource/Lyndley_Craven, http://dbpedia.org/resource/Matt_Holmes, http://dbpedia.org/resource/Steven_Tandy, http://dbpedia.org/resource/Stuart_Robert, http://dbpedia.org/resource/Sydney_Long, http://dbpedia.org/resource/Tod_Strike, http://dbpedia.org/resource/Arabanoo, http://dbpedia.org/resource/Curtis_Stone, http://dbpedia.org/resource/Ian_Fleming_(actor), http://dbpedia.org/resource/John_Carrick_(Australian_politician), http://dbpedia.org/resource/Ken_Jasper, http://dbpedia.org/resource/Marjorie_Tipping, http://dbpedia.org/resource/Ola_Cohn, http://dbpedia.org/resource/Pete_Evans, http://dbpedia.org/resource/Peter_Gilmore_(chef), http://dbpedia.org/resource/Ron_McLean, http://dbpedia.org/resource/Struan_Sutherland, http://dbpedia.org/resource/Andrew_McAuley, http://dbpedia.org/resource/Damien_Atkins, http://dbpedia.org/resource/E._Phillips_Fox, http://dbpedia.org/resource/Grace_Cossington_Smith, http://dbpedia.org/resource/Jeff_Rowley, http://dbpedia.org/resource/John_Brack, http://dbpedia.org/resource/John_Marsden_(writer), http://dbpedia.org/resource/John_Perceval, http://dbpedia.org/resource/Joy_Nichols, http://dbpedia.org/resource/Michael_Caton, http://dbpedia.org/resource/Michèle_Renouf, http://dbpedia.org/resource/Ollie_Hoskins_(rugby_union), http://dbpedia.org/resource/Pamela_Freeman, http://dbpedia.org/resource/Raelee_Hill, http://dbpedia.org/resource/Timothy_Potts, http://dbpedia.org/resource/Bryan_Gaensler, http://dbpedia.org/resource/Gordon_J._Garradd, http://dbpedia.org/resource/Kenneth_Dougall, http://dbpedia.org/resource/Ted_Ringwood, http://dbpedia.org/resource/Dunc_Gray, http://dbpedia.org/resource/Alex_Scott_(actor), http://dbpedia.org/resource/Basil_Onyons, http://dbpedia.org/resource/Greg_Evans_(television_host), http://dbpedia.org/resource/Kristy_Hinze, http://dbpedia.org/resource/Josh_Hazlewood, http://dbpedia.org/resource/Josh_Mitchell, http://dbpedia.org/resource/Kylie_Gillies, http://dbpedia.org/resource/Peter_Blakeley, http://dbpedia.org/resource/Robert_Clyde_Packer, http://dbpedia.org/resource/Shravasti_Dhammika, http://dbpedia.org/resource/Troy_Hearfield, http://dbpedia.org/resource/Vincent_Ball, http://dbpedia.org/resource/Billy_Dib, http://dbpedia.org/resource/David_Morgan_(businessman), http://dbpedia.org/resource/Jeremy_O'Brien, http://dbpedia.org/resource/Keith_Aickin, http://dbpedia.org/resource/Robin_Scott_(politician), http://dbpedia.org/resource/Samuel_L._Braunstein, http://dbpedia.org/resource/Tony_Sheehan, http://dbpedia.org/resource/Benjamin_Maio_Mackay, http://dbpedia.org/resource/Marion_Lennox, http://dbpedia.org/resource/Meredith_Webber, http://dbpedia.org/resource/Andrew_Wilson_(businessman), http://dbpedia.org/resource/Emil_Minty, http://dbpedia.org/resource/Henry_Ernest_Searle, http://dbpedia.org/resource/Kwame_Yeboah, http://dbpedia.org/resource/Matthew_Wilson_(cyclist), http://dbpedia.org/resource/Nick_Gates, http://dbpedia.org/resource/Sally_Peers, http://dbpedia.org/resource/Stephen_Mayne, http://dbpedia.org/resource/Andrew_McKaige, http://dbpedia.org/resource/Bill_Heatley, http://dbpedia.org/resource/Geoffrey_W._Hoffmann, http://dbpedia.org/resource/Jack_Miller_(motorcycle_racer), http://dbpedia.org/resource/Jason_Dasey, http://dbpedia.org/resource/Leigh_Diffey, http://dbpedia.org/resource/Len_Martin, http://dbpedia.org/resource/Matthew_White_(journalist), http://dbpedia.org/resource/Michael_Schiavello, http://dbpedia.org/resource/Rosemary_Menkens, http://dbpedia.org/resource/Stephen_Quartermain, http://dbpedia.org/resource/Yvonne_Sampson, http://dbpedia.org/resource/Blair_McDonough, http://dbpedia.org/resource/David_Byrne_(Australian_politician), http://dbpedia.org/resource/Helen_Smith_(fencer), http://dbpedia.org/resource/John_Hurley_(New_South_Wales_politician_born_1844), http://dbpedia.org/resource/Aaron_Finch, http://dbpedia.org/resource/Cherie_Nowlan, http://dbpedia.org/resource/Colin_McDonald_(Australian_cricketer), http://dbpedia.org/resource/David_Saker, http://dbpedia.org/resource/Frederick_Pirani, http://dbpedia.org/resource/Glenn_Butcher, http://dbpedia.org/resource/James_Pattinson, http://dbpedia.org/resource/Joel_Paris, http://dbpedia.org/resource/Kane_Richardson, http://dbpedia.org/resource/Mark_Trevorrow, http://dbpedia.org/resource/Matthew_Wade, http://dbpedia.org/resource/Peter_Gouldthorpe, http://dbpedia.org/resource/Sibylla_Budd, http://dbpedia.org/resource/Tanc_Sade, http://dbpedia.org/resource/Vimala_Raman, http://dbpedia.org/resource/Rodger_Corser, http://dbpedia.org/resource/Bebejan, http://dbpedia.org/resource/Dougie_Young, http://dbpedia.org/resource/George_Assang, http://dbpedia.org/resource/Jon_Sullivan, http://dbpedia.org/resource/Paul_Trimboli, http://dbpedia.org/resource/Roger_Knox, http://dbpedia.org/resource/Bruce_R._Davis, http://dbpedia.org/resource/Lauren_Nourse, http://dbpedia.org/resource/Michael_Alaalatoa, http://dbpedia.org/resource/Peter_Samu, http://dbpedia.org/resource/Rosalie_Ham, http://dbpedia.org/resource/Warwick_Stevenson, http://dbpedia.org/resource/Barry_Dancer, http://dbpedia.org/resource/Caroline_Buchanan, http://dbpedia.org/resource/Edward_Ashley-Cooper, http://dbpedia.org/resource/John_Roe_(rugby_union), http://dbpedia.org/resource/Kate_Gordon, http://dbpedia.org/resource/Luke_Bracey, http://dbpedia.org/resource/Marcus_Westbury, http://dbpedia.org/resource/Rob_Simmons_(rugby_union), http://dbpedia.org/resource/Sam_Cordingley, http://dbpedia.org/resource/Sean_Hardman, http://dbpedia.org/resource/Shane_Kelly, http://dbpedia.org/resource/Vicki_Wilson, http://dbpedia.org/resource/David_Reid_(politician), http://dbpedia.org/resource/Fatai, http://dbpedia.org/resource/Kimble_Rendall, http://dbpedia.org/resource/Renato_Bartolomei, http://dbpedia.org/resource/Richard_Davies_(Australian_actor), http://dbpedia.org/resource/Sophie_Hensser, http://dbpedia.org/resource/Andrew_Cibej, http://dbpedia.org/resource/Damian_De_Montemas, http://dbpedia.org/resource/Diana_Glenn, http://dbpedia.org/resource/Jack_Little_(politician), http://dbpedia.org/resource/Jessica_Gower, http://dbpedia.org/resource/Josh_Waters, http://dbpedia.org/resource/Shannon_Cole, http://dbpedia.org/resource/Lex_Lasry, http://dbpedia.org/resource/Celia_Pacquola, http://dbpedia.org/resource/Kevin_Heinze, http://dbpedia.org/resource/Nazeem_Hussain, http://dbpedia.org/resource/Norma_Redpath, http://dbpedia.org/resource/Percy_Leason, http://dbpedia.org/resource/Russell_Dykstra, http://dbpedia.org/resource/Stevie_Nicholson, http://dbpedia.org/resource/Tom_Gleeson, http://dbpedia.org/resource/Ashley_Fisher, http://dbpedia.org/resource/Doris_Younane, http://dbpedia.org/resource/Kristy_Moore, http://dbpedia.org/resource/David_Anthony_Smith, http://dbpedia.org/resource/James_Tobin_(presenter), http://dbpedia.org/resource/Samantha_Kerr, http://dbpedia.org/resource/Wendy_Machin, http://dbpedia.org/resource/John_Wood_(Australian_actor,_1909–1965), http://dbpedia.org/resource/Terry_Bourke, http://dbpedia.org/resource/Benjamin_Hardy, http://dbpedia.org/resource/John_Rogers_(cricketer,_born_1987), http://dbpedia.org/resource/Colin_Drybrough, http://dbpedia.org/resource/Kaarle_McCulloch, http://dbpedia.org/resource/Kayne_Tremills, http://dbpedia.org/resource/Kerrie_Meares, http://dbpedia.org/resource/Madeleine_Orr, http://dbpedia.org/resource/Jan_Mitchell, http://dbpedia.org/resource/Mark_D._McDonnell, http://dbpedia.org/resource/Patrick_Quinlan_(cricketer), http://dbpedia.org/resource/Cliff_Ho, http://dbpedia.org/resource/Hilton_Clarke, http://dbpedia.org/resource/John_Bylsma, http://dbpedia.org/resource/Charlie_Yankos, http://dbpedia.org/resource/Demelza_McCloud, http://dbpedia.org/resource/George_Houvardas, http://dbpedia.org/resource/Paul_Foster_(footballer), http://dbpedia.org/resource/Reshad_Strik, http://dbpedia.org/resource/Seldon_Hunt, http://dbpedia.org/resource/Roger_Rayson, http://dbpedia.org/resource/Charles_Clark_(Australian_politician), http://dbpedia.org/resource/Leigh_Howard, http://dbpedia.org/resource/Renee_Rollason, http://dbpedia.org/resource/Tim_Brown_(darts_player), http://dbpedia.org/resource/James_Paull_(musician), http://dbpedia.org/resource/Gary_Phillips_(Australian_footballer), http://dbpedia.org/resource/Kevin_Rozzoli, http://dbpedia.org/resource/Philip_Bowman, http://dbpedia.org/resource/Robert_Zadow, http://dbpedia.org/resource/Syd_Miller_(cartoonist), http://dbpedia.org/resource/Tom_Morey_(politician), http://dbpedia.org/resource/Felix_Dean, http://dbpedia.org/resource/Lauren_Wells_(athlete), http://dbpedia.org/resource/Mick_Veivers, http://dbpedia.org/resource/Angela_Fimmano, http://dbpedia.org/resource/Brendan_Hamill_(footballer), http://dbpedia.org/resource/Casey_Dumont, http://dbpedia.org/resource/Emily_Dunn_(footballer), http://dbpedia.org/resource/Jenna_Kingsley, http://dbpedia.org/resource/Jennifer_Bisset, http://dbpedia.org/resource/Kerem_Bulut, http://dbpedia.org/resource/Loren_Mahoney, http://dbpedia.org/resource/Matt_Acton, http://dbpedia.org/resource/Mitchell_Mallia, http://dbpedia.org/resource/Morgan_Griffin, http://dbpedia.org/resource/Petar_Franjic, http://dbpedia.org/resource/Ruth_Blackburn, http://dbpedia.org/resource/Sam_Gallaway, http://dbpedia.org/resource/Ursula_Hughson, http://dbpedia.org/resource/Caitlin_Munoz, http://dbpedia.org/resource/Don_McLennan, http://dbpedia.org/resource/Joanne_Burgess, http://dbpedia.org/resource/Kriv_Stenders, http://dbpedia.org/resource/Serin_Murray, http://dbpedia.org/resource/Ylona_Garcia, http://dbpedia.org/resource/Adam_van_Dommele, http://dbpedia.org/resource/Ben_Buckley, http://dbpedia.org/resource/Monique_Iannella, http://dbpedia.org/resource/Chris_Lucas, http://dbpedia.org/resource/Nick_Allanby, http://dbpedia.org/resource/Steve_Rodgers, http://dbpedia.org/resource/Sydney_Stubbs, http://dbpedia.org/resource/Edward_Oldfield, http://dbpedia.org/resource/John_Atkinson_(actor), http://dbpedia.org/resource/Paul_Pantano, http://dbpedia.org/resource/Peter_Johnston_(tennis), http://dbpedia.org/resource/Adam_Melonas, http://dbpedia.org/resource/Jessica_McNamee, http://dbpedia.org/resource/Phibs, http://dbpedia.org/resource/Tom_Willis, http://dbpedia.org/resource/Dylan_Caton, http://dbpedia.org/resource/Jason_Ballantine, http://dbpedia.org/resource/Kliment_Taseski, http://dbpedia.org/resource/Tracey_Hannah, http://dbpedia.org/resource/Rachael_Robertson_(writer), http://dbpedia.org/resource/Taren_Stinebrickner-Kauffman, http://dbpedia.org/resource/Andrew_Zesers, http://dbpedia.org/resource/Glenn_Bishop, http://dbpedia.org/resource/Ken_Hough, http://dbpedia.org/resource/Adam_Polkinghorne, http://dbpedia.org/resource/Chelsea_Locke, http://dbpedia.org/resource/Laura_von_Bertouch, http://dbpedia.org/resource/Levi_Miller, http://dbpedia.org/resource/Mo'onia_Gerrard, http://dbpedia.org/resource/Scott_Kremerskothen, http://dbpedia.org/resource/Christopher_Williams_(cricketer), http://dbpedia.org/resource/Michael_Maranta, http://dbpedia.org/resource/Mick_Miller_(police_officer), http://dbpedia.org/resource/Tim_Ward_(skater), http://dbpedia.org/resource/Andy_Clockwise, http://dbpedia.org/resource/Elisabeth_White, http://dbpedia.org/resource/Eric_Gaven, http://dbpedia.org/resource/Colin_Bennett_(footballer), http://dbpedia.org/resource/John_Englart, http://dbpedia.org/resource/Joshua_Thomas_Bell, http://dbpedia.org/resource/Betty_Maker, http://dbpedia.org/resource/Charles_Boxshall, http://dbpedia.org/resource/Frank_Kerr_(cricketer), http://dbpedia.org/resource/Harry_Williams_(Australian_soccer_player), http://dbpedia.org/resource/Howard_Alloo, http://dbpedia.org/resource/Judy_Masters, http://dbpedia.org/resource/Julie_Dolan, http://dbpedia.org/resource/Miranda_Chase, http://dbpedia.org/resource/Reg_Date, http://dbpedia.org/resource/Robert_Menzies_(cricketer), http://dbpedia.org/resource/Steven_O'Dor, http://dbpedia.org/resource/Terri-Amber_Carlson, http://dbpedia.org/resource/Ashley_Cooper_(racing_driver), http://dbpedia.org/resource/Caitlin_Cooper, http://dbpedia.org/resource/Jane_Zemiro, http://dbpedia.org/resource/Brendan_McArdle, http://dbpedia.org/resource/Ashley_Spina, http://dbpedia.org/resource/Chris_Tadrosse, http://dbpedia.org/resource/Tara_Andrews, http://dbpedia.org/resource/Guboo_Ted_Thomas, http://dbpedia.org/resource/Una_Lucy_Fielding, http://dbpedia.org/resource/Catherine_Mack-Hancock, http://dbpedia.org/resource/Damien_Garvey, http://dbpedia.org/resource/Ellison_Harvie, http://dbpedia.org/resource/Catherine_Cannuli, http://dbpedia.org/resource/Rosemary_Goldie, http://dbpedia.org/resource/Sasha_McDonnell, http://dbpedia.org/resource/David_Zeidler, http://dbpedia.org/resource/Wayne_Weening, http://dbpedia.org/resource/Allison_Shreeve, http://dbpedia.org/resource/Antoni_Trimboli, http://dbpedia.org/resource/Chris_Sullivan_(Australian_association_footballer), http://dbpedia.org/resource/Joe_Mullen_(footballer), http://dbpedia.org/resource/Sebastian_Gregory, http://dbpedia.org/resource/Meg_McLaughlin, http://dbpedia.org/resource/Michael_Leighton_(politician), http://dbpedia.org/resource/Ted_Baldwin_(Queensland_politician), http://dbpedia.org/resource/William_Kent_(Australian_politician), http://dbpedia.org/resource/Alexander_Hill_(rower), http://dbpedia.org/resource/John_Rowe_(author), http://dbpedia.org/resource/Victoria_Brown_(water_polo), http://dbpedia.org/resource/Ben_Glatzer, http://dbpedia.org/resource/Nathan_Roberts_(volleyball), http://dbpedia.org/resource/Rebecca_Reeve, http://dbpedia.org/resource/Sime_Kovacevic, http://dbpedia.org/resource/Mitchell_Clegg, http://dbpedia.org/resource/Alex_Cudlin, http://dbpedia.org/resource/Don_Meij, http://dbpedia.org/resource/Jordan_Zamora, http://dbpedia.org/resource/Max_Croker, http://dbpedia.org/resource/Trudy_Camilleri, http://dbpedia.org/resource/Paul_Hilton_(Queensland_politician), http://dbpedia.org/resource/Pat_Orreal, http://dbpedia.org/resource/Norman_Fisher_(public_servant), http://dbpedia.org/resource/Samuel_Wilson_(ice_hockey), http://dbpedia.org/resource/Styalz_Fuego, http://dbpedia.org/resource/Andy_Belsak, http://dbpedia.org/resource/Arthur_Betts_(cricketer), http://dbpedia.org/resource/Ashley_Facy, http://dbpedia.org/resource/Ben_Targett, http://dbpedia.org/resource/Bertie_Brownlow, http://dbpedia.org/resource/Brian_Robinson_(cricketer), http://dbpedia.org/resource/Bruce_Cruse, http://dbpedia.org/resource/Bruce_Neill, http://dbpedia.org/resource/Chris_Broadby, http://dbpedia.org/resource/Christopher_Dell_(cricketer), http://dbpedia.org/resource/David_Castle_(cricketer), http://dbpedia.org/resource/Dennis_Baker, http://dbpedia.org/resource/Ernest_Free, http://dbpedia.org/resource/Gary_Whitney, http://dbpedia.org/resource/Gerry_Connor, http://dbpedia.org/resource/Graeme_Cunningham, http://dbpedia.org/resource/Gregory_Wilson_(cricketer), http://dbpedia.org/resource/Harold_Hale, http://dbpedia.org/resource/Ian_Connell, http://dbpedia.org/resource/John_Lord_(cricketer), http://dbpedia.org/resource/John_Saint, http://dbpedia.org/resource/Josef_Holyman, http://dbpedia.org/resource/Keith_Linney, http://dbpedia.org/resource/Keith_Westbrook, http://dbpedia.org/resource/Lynton_Rowlands, http://dbpedia.org/resource/Mark_Hill_(cricketer), http://dbpedia.org/resource/Mark_N._Atkinson, http://dbpedia.org/resource/Mark_Scholes, http://dbpedia.org/resource/Michael_Farrell_(cricketer), http://dbpedia.org/resource/Michael_Norman_(Australian_cricketer), http://dbpedia.org/resource/Michael_Tame, http://dbpedia.org/resource/Nicholas_Courtney_(cricketer), http://dbpedia.org/resource/Nigel_Brookes, http://dbpedia.org/resource/Norman_Dodds_(cricketer), http://dbpedia.org/resource/Rex_Davidson, http://dbpedia.org/resource/Richard_Soule, http://dbpedia.org/resource/Robert_Gartrell, http://dbpedia.org/resource/Robert_Knight_(cricketer), http://dbpedia.org/resource/Rowan_Sherriff]... [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 44, P: 0.363, R: 0.394, macro F1: 0.355, macro F1*: 0.378 -[INFO] QALD Cumulative metrics, sample: 44, P: 0.499, R: 0.394, macro F1: 0.355, QALD macro F1: 0.441 -[INFO] Current time: 14988 ms, average: 4159.32 ms/ques, entity detection: 906.48 ms/ques (linking tool: 817.36 ms/ques), relation detection: 1339.43 ms/ques (similarity: 727.59 ms/ques), EDG generation: 113.39 ms/ques +[INFO] Cumulative metrics, sample: 44, P: 0.363, R: 0.394, macro F1: 0.378 +[INFO] QALD Cumulative metrics, sample: 44, P: 0.499, R: 0.394, average F1: 0.355, QALD macro F1: 0.441 +[INFO] Current question consumed time: 13480 ms, average: 3919.68 ms/ques, entity detection: 832.61 ms/ques (linking tool: 748.95 ms/ques), relation detection: 1290.82 ms/ques (similarity: 732.95 ms/ques), EDG generation: 114.23 ms/ques [INFO] Entity Linking P: 0.1590909090909091, R: 0.1590909090909091, micro F1: 0.1590909090909091, macro F1: 0.1590909090909091 [INFO] Relation Linking P: 0.18181818181818182, R: 0.18181818181818182, micro F1: 0.18181818181818182, macro F1: 0.18181818181818182 [INFO] Type Linking P: 0.9318181818181818, R: 0.9318181818181818, micro F1: 0.9318181818181818, macro F1: 0.9318181818181818 @@ -667,9 +667,9 @@ Question 44: Give me all Dutch parties. [INFO] Golden Answer: [http://dbpedia.org/resource/Anti-Revolutionary_Party, http://dbpedia.org/resource/Frisian_National_Party, http://dbpedia.org/resource/Article_50_(political_party), http://dbpedia.org/resource/People's_Party_for_Freedom_and_Democracy, http://dbpedia.org/resource/Trots_op_Nederland, http://dbpedia.org/resource/Christian_Democratic_Appeal, http://dbpedia.org/resource/Reformed_Political_Party, http://dbpedia.org/resource/Free_Union_(anarchist_organisation), http://dbpedia.org/resource/Onafhankelijke_Burger_Partij, http://dbpedia.org/resource/GroenLinks, http://dbpedia.org/resource/Democratic_Political_Turning_Point] [INFO] Predicted Answer: [] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 45, P: 0.355, R: 0.386, macro F1: 0.347, macro F1*: 0.370 -[INFO] QALD Cumulative metrics, sample: 45, P: 0.510, R: 0.386, macro F1: 0.347, QALD macro F1: 0.439 -[INFO] Current time: 934 ms, average: 4087.64 ms/ques, entity detection: 888.20 ms/ques (linking tool: 800.00 ms/ques), relation detection: 1314.49 ms/ques (similarity: 714.29 ms/ques), EDG generation: 111.09 ms/ques +[INFO] Cumulative metrics, sample: 45, P: 0.355, R: 0.386, macro F1: 0.370 +[INFO] QALD Cumulative metrics, sample: 45, P: 0.510, R: 0.386, average F1: 0.347, QALD macro F1: 0.439 +[INFO] Current question consumed time: 844 ms, average: 3851.33 ms/ques, entity detection: 815.73 ms/ques (linking tool: 733.07 ms/ques), relation detection: 1266.76 ms/ques (similarity: 719.56 ms/ques), EDG generation: 111.84 ms/ques [INFO] Entity Linking P: 0.17777777777777778, R: 0.17777777777777778, micro F1: 0.17777777777777778, macro F1: 0.17777777777777778 [INFO] Relation Linking P: 0.2, R: 0.2, micro F1: 0.2, macro F1: 0.20000000000000004 [INFO] Type Linking P: 0.9333333333333333, R: 0.9333333333333333, micro F1: 0.9333333333333333, macro F1: 0.9333333333333333 @@ -682,9 +682,9 @@ Question 45: How many moons does Mars have? [INFO] Golden Answer: [2] [INFO] Predicted Answer: [1] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 46, P: 0.347, R: 0.377, macro F1: 0.340, macro F1*: 0.362 -[INFO] QALD Cumulative metrics, sample: 46, P: 0.499, R: 0.377, macro F1: 0.340, QALD macro F1: 0.430 -[INFO] Current time: 2219 ms, average: 4047.02 ms/ques, entity detection: 874.96 ms/ques (linking tool: 786.74 ms/ques), relation detection: 1302.50 ms/ques (similarity: 715.83 ms/ques), EDG generation: 108.87 ms/ques +[INFO] Cumulative metrics, sample: 46, P: 0.347, R: 0.377, macro F1: 0.362 +[INFO] QALD Cumulative metrics, sample: 46, P: 0.499, R: 0.377, average F1: 0.340, QALD macro F1: 0.430 +[INFO] Current question consumed time: 2101 ms, average: 3813.28 ms/ques, entity detection: 803.72 ms/ques (linking tool: 720.98 ms/ques), relation detection: 1255.26 ms/ques (similarity: 721.02 ms/ques), EDG generation: 109.61 ms/ques [INFO] Entity Linking P: 0.17391304347826086, R: 0.17391304347826086, micro F1: 0.17391304347826086, macro F1: 0.17391304347826086 [INFO] Relation Linking P: 0.1956521739130435, R: 0.1956521739130435, micro F1: 0.1956521739130435, macro F1: 0.1956521739130435 [INFO] Type Linking P: 0.9347826086956522, R: 0.9347826086956522, micro F1: 0.9347826086956522, macro F1: 0.9347826086956522 @@ -697,9 +697,9 @@ Question 46: What is Batman's real name? [INFO] Golden Answer: [Bruce Wayne@en] [INFO] Predicted Answer: [Batman@en, Bruce Wayne@en] [INFO] P: 0.5, R: 1.0, F: 0.6666666666666666 -[INFO] Cumulative metrics, sample: 47, P: 0.350, R: 0.391, macro F1: 0.347, macro F1*: 0.369 -[INFO] QALD Cumulative metrics, sample: 47, P: 0.499, R: 0.391, macro F1: 0.347, QALD macro F1: 0.438 -[INFO] Current time: 2159 ms, average: 4006.85 ms/ques, entity detection: 869.36 ms/ques (linking tool: 779.00 ms/ques), relation detection: 1286.04 ms/ques (similarity: 706.62 ms/ques), EDG generation: 106.72 ms/ques +[INFO] Cumulative metrics, sample: 47, P: 0.350, R: 0.391, macro F1: 0.369 +[INFO] QALD Cumulative metrics, sample: 47, P: 0.499, R: 0.391, average F1: 0.347, QALD macro F1: 0.438 +[INFO] Current question consumed time: 1958 ms, average: 3773.81 ms/ques, entity detection: 797.89 ms/ques (linking tool: 713.68 ms/ques), relation detection: 1239.19 ms/ques (similarity: 711.70 ms/ques), EDG generation: 107.45 ms/ques [INFO] Entity Linking P: 0.1702127659574468, R: 0.1702127659574468, micro F1: 0.1702127659574468, macro F1: 0.1702127659574468 [INFO] Relation Linking P: 0.19148936170212766, R: 0.19148936170212766, micro F1: 0.19148936170212766, macro F1: 0.19148936170212763 [INFO] Type Linking P: 0.9361702127659575, R: 0.9361702127659575, micro F1: 0.9361702127659575, macro F1: 0.9361702127659575 @@ -712,9 +712,9 @@ Question 47: What is Elon Musk famous for? [INFO] Golden Answer: [http://dbpedia.org/resource/SolarCity, http://dbpedia.org/resource/Tesla_Motors, http://dbpedia.org/resource/PayPal, http://dbpedia.org/resource/OpenAI, http://dbpedia.org/resource/SpaceX, http://dbpedia.org/resource/Hyperloop] [INFO] Predicted Answer: [http://dbpedia.org/resource/SolarCity, http://dbpedia.org/resource/Tesla_Motors, http://dbpedia.org/resource/PayPal, http://dbpedia.org/resource/OpenAI, http://dbpedia.org/resource/SpaceX, http://dbpedia.org/resource/Hyperloop] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 48, P: 0.364, R: 0.403, macro F1: 0.360, macro F1*: 0.383 -[INFO] QALD Cumulative metrics, sample: 48, P: 0.510, R: 0.403, macro F1: 0.360, QALD macro F1: 0.450 -[INFO] Current time: 1084 ms, average: 3945.96 ms/ques, entity detection: 854.92 ms/ques (linking tool: 765.35 ms/ques), relation detection: 1268.27 ms/ques (similarity: 697.29 ms/ques), EDG generation: 104.73 ms/ques +[INFO] Cumulative metrics, sample: 48, P: 0.364, R: 0.403, macro F1: 0.383 +[INFO] QALD Cumulative metrics, sample: 48, P: 0.510, R: 0.403, average F1: 0.360, QALD macro F1: 0.450 +[INFO] Current question consumed time: 1013 ms, average: 3716.29 ms/ques, entity detection: 784.73 ms/ques (linking tool: 701.08 ms/ques), relation detection: 1222.15 ms/ques (similarity: 702.25 ms/ques), EDG generation: 105.42 ms/ques [INFO] Entity Linking P: 0.16666666666666666, R: 0.16666666666666666, micro F1: 0.16666666666666666, macro F1: 0.16666666666666666 [INFO] Relation Linking P: 0.1875, R: 0.1875, micro F1: 0.1875, macro F1: 0.1875 [INFO] Type Linking P: 0.9375, R: 0.9375, micro F1: 0.9375, macro F1: 0.9375 @@ -727,9 +727,9 @@ Question 48: Who is the author of WikiLeaks? [INFO] Golden Answer: [http://dbpedia.org/resource/Julian_Assange] [INFO] Predicted Answer: [http://dbpedia.org/resource/Julian_Assange] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 49, P: 0.377, R: 0.415, macro F1: 0.373, macro F1*: 0.395 -[INFO] QALD Cumulative metrics, sample: 49, P: 0.520, R: 0.415, macro F1: 0.373, QALD macro F1: 0.462 -[INFO] Current time: 1056 ms, average: 3886.98 ms/ques, entity detection: 841.59 ms/ques (linking tool: 752.31 ms/ques), relation detection: 1251.98 ms/ques (similarity: 688.39 ms/ques), EDG generation: 102.80 ms/ques +[INFO] Cumulative metrics, sample: 49, P: 0.377, R: 0.415, macro F1: 0.395 +[INFO] QALD Cumulative metrics, sample: 49, P: 0.520, R: 0.415, average F1: 0.373, QALD macro F1: 0.462 +[INFO] Current question consumed time: 941 ms, average: 3659.65 ms/ques, entity detection: 772.10 ms/ques (linking tool: 688.90 ms/ques), relation detection: 1206.10 ms/ques (similarity: 693.29 ms/ques), EDG generation: 103.47 ms/ques [INFO] Entity Linking P: 0.16326530612244897, R: 0.16326530612244897, micro F1: 0.16326530612244897, macro F1: 0.16326530612244897 [INFO] Relation Linking P: 0.1836734693877551, R: 0.1836734693877551, micro F1: 0.1836734693877551, macro F1: 0.18367346938775508 [INFO] Type Linking P: 0.9387755102040817, R: 0.9387755102040817, micro F1: 0.9387755102040817, macro F1: 0.9387755102040817 @@ -742,9 +742,9 @@ Question 49: Sean Parnell was the governor of which U.S. state? [INFO] Golden Answer: [http://dbpedia.org/resource/Alaska] [INFO] Predicted Answer: [http://dbpedia.org/resource/Dan_Sullivan_(U.S._Senator), http://dbpedia.org/resource/Mead_Treadwell, http://dbpedia.org/resource/Craig_Campbell_(politician), http://dbpedia.org/resource/Michael_Geraghty, http://dbpedia.org/resource/Sarah_Palin] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 50, P: 0.369, R: 0.407, macro F1: 0.366, macro F1*: 0.387 -[INFO] QALD Cumulative metrics, sample: 50, P: 0.509, R: 0.407, macro F1: 0.366, QALD macro F1: 0.453 -[INFO] Current time: 2792 ms, average: 3865.08 ms/ques, entity detection: 841.46 ms/ques (linking tool: 750.82 ms/ques), relation detection: 1234.48 ms/ques (similarity: 678.90 ms/ques), EDG generation: 101.08 ms/ques +[INFO] Cumulative metrics, sample: 50, P: 0.369, R: 0.407, macro F1: 0.387 +[INFO] QALD Cumulative metrics, sample: 50, P: 0.509, R: 0.407, average F1: 0.366, QALD macro F1: 0.453 +[INFO] Current question consumed time: 2418 ms, average: 3634.82 ms/ques, entity detection: 770.06 ms/ques (linking tool: 686.32 ms/ques), relation detection: 1188.56 ms/ques (similarity: 683.80 ms/ques), EDG generation: 101.74 ms/ques [INFO] Entity Linking P: 0.16, R: 0.16, micro F1: 0.16, macro F1: 0.16 [INFO] Relation Linking P: 0.18, R: 0.18, micro F1: 0.18, macro F1: 0.18 [INFO] Type Linking P: 0.94, R: 0.94, micro F1: 0.94, macro F1: 0.94 @@ -757,9 +757,9 @@ Question 50: How many years was the Ford Model T manufactured? [INFO] Golden Answer: [19] [INFO] Predicted Answer: [1] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 51, P: 0.362, R: 0.399, macro F1: 0.359, macro F1*: 0.380 -[INFO] QALD Cumulative metrics, sample: 51, P: 0.499, R: 0.399, macro F1: 0.359, QALD macro F1: 0.444 -[INFO] Current time: 1268 ms, average: 3814.16 ms/ques, entity detection: 828.65 ms/ques (linking tool: 738.51 ms/ques), relation detection: 1216.35 ms/ques (similarity: 671.67 ms/ques), EDG generation: 99.35 ms/ques +[INFO] Cumulative metrics, sample: 51, P: 0.362, R: 0.399, macro F1: 0.380 +[INFO] QALD Cumulative metrics, sample: 51, P: 0.499, R: 0.399, average F1: 0.359, QALD macro F1: 0.444 +[INFO] Current question consumed time: 1016 ms, average: 3583.47 ms/ques, entity detection: 757.53 ms/ques (linking tool: 674.59 ms/ques), relation detection: 1170.35 ms/ques (similarity: 676.47 ms/ques), EDG generation: 99.98 ms/ques [INFO] Entity Linking P: 0.1568627450980392, R: 0.1568627450980392, micro F1: 0.1568627450980392, macro F1: 0.1568627450980392 [INFO] Relation Linking P: 0.17647058823529413, R: 0.17647058823529413, micro F1: 0.17647058823529413, macro F1: 0.17647058823529413 [INFO] Type Linking P: 0.9411764705882353, R: 0.9411764705882353, micro F1: 0.9411764705882353, macro F1: 0.9411764705882353 @@ -772,9 +772,9 @@ Question 51: When was Carlo Giuliani shot? [INFO] Golden Answer: [] [INFO] Predicted Answer: [] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 52, P: 0.374, R: 0.411, macro F1: 0.371, macro F1*: 0.392 -[INFO] QALD Cumulative metrics, sample: 52, P: 0.509, R: 0.411, macro F1: 0.371, QALD macro F1: 0.455 -[INFO] Current time: 896 ms, average: 3758.04 ms/ques, entity detection: 815.85 ms/ques (linking tool: 726.60 ms/ques), relation detection: 1197.73 ms/ques (similarity: 661.60 ms/ques), EDG generation: 97.58 ms/ques +[INFO] Cumulative metrics, sample: 52, P: 0.374, R: 0.411, macro F1: 0.392 +[INFO] QALD Cumulative metrics, sample: 52, P: 0.509, R: 0.411, average F1: 0.371, QALD macro F1: 0.455 +[INFO] Current question consumed time: 799 ms, average: 3529.92 ms/ques, entity detection: 745.67 ms/ques (linking tool: 663.52 ms/ques), relation detection: 1152.46 ms/ques (similarity: 666.25 ms/ques), EDG generation: 98.19 ms/ques [INFO] Entity Linking P: 0.17307692307692307, R: 0.17307692307692307, micro F1: 0.17307692307692307, macro F1: 0.17307692307692307 [INFO] Relation Linking P: 0.19230769230769232, R: 0.19230769230769232, micro F1: 0.19230769230769232, macro F1: 0.19230769230769232 [INFO] Type Linking P: 0.9423076923076923, R: 0.9423076923076923, micro F1: 0.9423076923076923, macro F1: 0.9423076923076923 @@ -787,9 +787,9 @@ Question 52: Give me all animals that are extinct. [INFO] Golden Answer: [http://dbpedia.org/resource/California_grizzly_bear, http://dbpedia.org/resource/Mexican_grizzly_bear, http://dbpedia.org/resource/Bernard's_wolf, http://dbpedia.org/resource/Florida_black_wolf, http://dbpedia.org/resource/Stephens_Island_wren, http://dbpedia.org/resource/Alaotra_grebe, http://dbpedia.org/resource/Aldabra_brush_warbler, http://dbpedia.org/resource/Arabian_ostrich, http://dbpedia.org/resource/Ascension_crake, http://dbpedia.org/resource/Ascension_night_heron, http://dbpedia.org/resource/Atitlán_grebe, http://dbpedia.org/resource/Auckland_merganser, http://dbpedia.org/resource/Bar-winged_rail, http://dbpedia.org/resource/Bermuda_night_heron, http://dbpedia.org/resource/Black-fronted_parakeet, http://dbpedia.org/resource/Black_mamo, http://dbpedia.org/resource/Bonin_grosbeak, http://dbpedia.org/resource/Bonin_nankeen_night_heron, http://dbpedia.org/resource/Bonin_thrush, http://dbpedia.org/resource/Brace's_emerald, http://dbpedia.org/resource/Bushwren, http://dbpedia.org/resource/Carolina_parakeet, http://dbpedia.org/resource/Chatham_bellbird, http://dbpedia.org/resource/Chatham_fernbird, http://dbpedia.org/resource/Chatham_penguin, http://dbpedia.org/resource/Chatham_rail, http://dbpedia.org/resource/Choiseul_pigeon, http://dbpedia.org/resource/Colombian_grebe, http://dbpedia.org/resource/Cuban_macaw, http://dbpedia.org/resource/Cyprus_dipper, http://dbpedia.org/resource/Daito_varied_tit, http://dbpedia.org/resource/Daito_wren, http://dbpedia.org/resource/Delalande's_coua, http://dbpedia.org/resource/Dieffenbach's_rail, http://dbpedia.org/resource/Dusky_seaside_sparrow, http://dbpedia.org/resource/Eiao_monarch, http://dbpedia.org/resource/Finsch's_duck, http://dbpedia.org/resource/Garrett's_reed_warbler, http://dbpedia.org/resource/Gould's_emerald, http://dbpedia.org/resource/Grand_Cayman_thrush, http://dbpedia.org/resource/Greater_koa_finch, http://dbpedia.org/resource/Greater_ʻamakihi, http://dbpedia.org/resource/Guadalupe_caracara, http://dbpedia.org/resource/Guadeloupe_amazon, http://dbpedia.org/resource/Guadeloupe_parakeet, http://dbpedia.org/resource/Guam_flycatcher, http://dbpedia.org/resource/Hawaiian_rail, http://dbpedia.org/resource/Hawaiʻi_ʻōʻō, http://dbpedia.org/resource/Hawkins's_rail, http://dbpedia.org/resource/Heath_hen, http://dbpedia.org/resource/Hoopoe_starling, http://dbpedia.org/resource/Huia, http://dbpedia.org/resource/Imber's_petrel, http://dbpedia.org/resource/Kangaroo_Island_emu, http://dbpedia.org/resource/Kauai_palila, http://dbpedia.org/resource/Kauaʻi_ʻakialoa, http://dbpedia.org/resource/Kauaʻi_ʻōʻō, http://dbpedia.org/resource/King_Island_emu, http://dbpedia.org/resource/Kioea, http://dbpedia.org/resource/Kiritimati_sandpiper, http://dbpedia.org/resource/Kona_grosbeak, http://dbpedia.org/resource/Kosrae_crake, http://dbpedia.org/resource/Kosrae_starling, http://dbpedia.org/resource/Kāmaʻo, http://dbpedia.org/resource/Labrador_duck, http://dbpedia.org/resource/Laughing_owl, http://dbpedia.org/resource/Laysan_honeycreeper, http://dbpedia.org/resource/Laysan_millerbird, http://dbpedia.org/resource/Laysan_rail, http://dbpedia.org/resource/Lesser_koa_finch, http://dbpedia.org/resource/Lord_Howe_boobook, http://dbpedia.org/resource/Lord_Howe_gerygone, http://dbpedia.org/resource/Lord_Howe_parakeet, http://dbpedia.org/resource/Lord_Howe_pigeon, http://dbpedia.org/resource/Lord_Howe_starling, http://dbpedia.org/resource/Lord_Howe_swamphen, http://dbpedia.org/resource/Lord_Howe_thrush, http://dbpedia.org/resource/Macquarie_parakeet, http://dbpedia.org/resource/Macquarie_rail, http://dbpedia.org/resource/Madeiran_wood_pigeon, http://dbpedia.org/resource/Mangareva_reed_warbler, http://dbpedia.org/resource/Mariana_mallard, http://dbpedia.org/resource/Marianne_white-eye, http://dbpedia.org/resource/Marquesas_swamphen, http://dbpedia.org/resource/Martinique_amazon, http://dbpedia.org/resource/Mascarene_parrot, http://dbpedia.org/resource/Maupiti_monarch, http://dbpedia.org/resource/Moorea_reed_warbler, http://dbpedia.org/resource/Mukojima_white-eye, http://dbpedia.org/resource/New_Caledonian_gallinule, http://dbpedia.org/resource/New_Zealand_quail, http://dbpedia.org/resource/Newton's_parakeet, http://dbpedia.org/resource/Norfolk_boobook, http://dbpedia.org/resource/Norfolk_kaka, http://dbpedia.org/resource/North_Island_piopio, http://dbpedia.org/resource/North_Island_snipe, http://dbpedia.org/resource/North_Island_takahē, http://dbpedia.org/resource/Nuku_Hiva_monarch, http://dbpedia.org/resource/Oceanic_eclectus_parrot, http://dbpedia.org/resource/Olson's_petrel, http://dbpedia.org/resource/Oʻahu_nukupuʻu, http://dbpedia.org/resource/Oʻahu_ʻōʻō, http://dbpedia.org/resource/Pagan_reed_warbler, http://dbpedia.org/resource/Palaeoloxodon_falconeri, http://dbpedia.org/resource/Paradise_parrot, http://dbpedia.org/resource/Pied_raven, http://dbpedia.org/resource/Puerto_Rican_barn_owl, http://dbpedia.org/resource/Red-moustached_fruit_dove, http://dbpedia.org/resource/Rodrigues_bulbul, http://dbpedia.org/resource/Rodrigues_night_heron, http://dbpedia.org/resource/Rodrigues_owl, http://dbpedia.org/resource/Rodrigues_parrot, http://dbpedia.org/resource/Rodrigues_pigeon, http://dbpedia.org/resource/Rodrigues_rail, http://dbpedia.org/resource/Rodrigues_starling, http://dbpedia.org/resource/Réunion_fody, http://dbpedia.org/resource/Réunion_kestrel, http://dbpedia.org/resource/Réunion_night_heron, http://dbpedia.org/resource/Réunion_owl, http://dbpedia.org/resource/Réunion_parakeet, http://dbpedia.org/resource/Réunion_pink_pigeon, http://dbpedia.org/resource/Réunion_rail, http://dbpedia.org/resource/Réunion_swamphen, http://dbpedia.org/resource/Saint_Helena_crake, http://dbpedia.org/resource/Saint_Helena_cuckoo, http://dbpedia.org/resource/Saint_Helena_dove, http://dbpedia.org/resource/Saint_Helena_hoopoe, http://dbpedia.org/resource/Saint_Helena_petrel, http://dbpedia.org/resource/Saint_Helena_swamphen, http://dbpedia.org/resource/San_Benedicto_rock_wren, http://dbpedia.org/resource/Seychelles_parakeet, http://dbpedia.org/resource/Slender-billed_grackle, http://dbpedia.org/resource/Society_parakeet, http://dbpedia.org/resource/South_Island_kōkako, http://dbpedia.org/resource/South_Island_piopio, http://dbpedia.org/resource/South_Island_snipe, http://dbpedia.org/resource/Spectacled_cormorant, http://dbpedia.org/resource/Steller's_sea_cow, http://dbpedia.org/resource/Straight-tusked_elephant, http://dbpedia.org/resource/Syrian_elephant, http://dbpedia.org/resource/Tahiti_sandpiper, http://dbpedia.org/resource/Tanna_ground_dove, http://dbpedia.org/resource/Tasman_starling, http://dbpedia.org/resource/Tasmanian_emu, http://dbpedia.org/resource/Thick-billed_ground_dove, http://dbpedia.org/resource/Tongatapu_rail, http://dbpedia.org/resource/Tristan_moorhen, http://dbpedia.org/resource/Upland_moa, http://dbpedia.org/resource/Wake_Island_rail, http://dbpedia.org/resource/Western_Lewin's_rail, http://dbpedia.org/resource/Western_rufous_bristlebird, http://dbpedia.org/resource/ʻUla-ʻai-hawane, http://dbpedia.org/resource/ʻĀmaui, http://dbpedia.org/resource/Tropidophis_bucculentus, http://dbpedia.org/resource/Rocky_Mountain_locust, http://dbpedia.org/resource/Waitaha_penguin, http://dbpedia.org/resource/Coregonus_bezola, http://dbpedia.org/resource/Coregonus_fera, http://dbpedia.org/resource/Coregonus_gutturosus, http://dbpedia.org/resource/European_ass, http://dbpedia.org/resource/Piopio_(bird), http://dbpedia.org/resource/Chatham_kaka, http://dbpedia.org/resource/Crescent_nail-tail_wallaby, http://dbpedia.org/resource/Eastern_hare-wallaby, http://dbpedia.org/resource/Lake_Mackay_hare-wallaby, http://dbpedia.org/resource/Toolache_wallaby, http://dbpedia.org/resource/Chadwick_Beach_cotton_mouse, http://dbpedia.org/resource/San_Marcos_gambusia, http://dbpedia.org/resource/Big-eared_hopping_mouse, http://dbpedia.org/resource/Darling_Downs_hopping_mouse, http://dbpedia.org/resource/Great_hopping_mouse, http://dbpedia.org/resource/Lesser_ʻakialoa, http://dbpedia.org/resource/Short-tailed_hopping_mouse, http://dbpedia.org/resource/Réunion_giant_tortoise, http://dbpedia.org/resource/Coues'_gadwall, http://dbpedia.org/resource/Northland_skink, http://dbpedia.org/resource/Gravenche, http://dbpedia.org/resource/Mangaia_swiftlet, http://dbpedia.org/resource/Bennu_heron, http://dbpedia.org/resource/Bucorvus_brailloni, http://dbpedia.org/resource/Lottia_edmitchelli, http://dbpedia.org/resource/Twisted-toothed_mouse, http://dbpedia.org/resource/Confused_moth, http://dbpedia.org/resource/Genophantis_leahi, http://dbpedia.org/resource/Giant_ghost-faced_bat, http://dbpedia.org/resource/Hilo_noctuid_moth, http://dbpedia.org/resource/Kaholuamano_noctuid_moth, http://dbpedia.org/resource/Kerr's_noctuid_moth, http://dbpedia.org/resource/Lovegrass_noctuid_moth, http://dbpedia.org/resource/Midway_noctuid_moth, http://dbpedia.org/resource/Minute_noctuid_moth, http://dbpedia.org/resource/Poko_noctuid_moth, http://dbpedia.org/resource/Scotorythra_nesiotes, http://dbpedia.org/resource/Tritocleis_microphylla, http://dbpedia.org/resource/Parras_pupfish, http://dbpedia.org/resource/Channeled_pebblesnail, http://dbpedia.org/resource/Corded_purg, http://dbpedia.org/resource/Ouachita_pebblesnail, http://dbpedia.org/resource/Posticobia_norfolkensis, http://dbpedia.org/resource/Quintalia_flosculus, http://dbpedia.org/resource/Quintalia_stoddartii, http://dbpedia.org/resource/Reverse_pebblesnail, http://dbpedia.org/resource/Samoana_inflata, http://dbpedia.org/resource/Sinployea_canalis, http://dbpedia.org/resource/Sinployea_decorticata, http://dbpedia.org/resource/Sinployea_harveyensis, http://dbpedia.org/resource/Sinployea_otareae, http://dbpedia.org/resource/Sinployea_planospira, http://dbpedia.org/resource/Sinployea_proxima, http://dbpedia.org/resource/Sinployea_rudis, http://dbpedia.org/resource/Sinployea_tenuicostata, http://dbpedia.org/resource/Sinployea_youngi, http://dbpedia.org/resource/Taipidon_anceyana, http://dbpedia.org/resource/Taipidon_marquesana, http://dbpedia.org/resource/Taipidon_octolamellata, http://dbpedia.org/resource/Thick-lipped_pebblesnail, http://dbpedia.org/resource/Sitalcicus_gardineri, http://dbpedia.org/resource/Daphnobela, http://dbpedia.org/resource/Pentremites, http://dbpedia.org/resource/Partula_faba, http://dbpedia.org/resource/Partula_turgida, http://dbpedia.org/resource/Namibcypris, http://dbpedia.org/resource/Keraea, http://dbpedia.org/resource/Pseudophilautus_adspersus, http://dbpedia.org/resource/Pseudophilautus_dimbullae, http://dbpedia.org/resource/Pseudophilautus_eximius, http://dbpedia.org/resource/Pseudophilautus_extirpo, http://dbpedia.org/resource/Pseudophilautus_halyi, http://dbpedia.org/resource/Pseudophilautus_leucorhinus, http://dbpedia.org/resource/Pseudophilautus_malcolmsmithi, http://dbpedia.org/resource/Pseudophilautus_nanus, http://dbpedia.org/resource/Pseudophilautus_nasutus, http://dbpedia.org/resource/Pseudophilautus_oxyrhynchus, http://dbpedia.org/resource/Pseudophilautus_rugatus, http://dbpedia.org/resource/Pseudophilautus_stellatus, http://dbpedia.org/resource/Pseudophilautus_temporalis, http://dbpedia.org/resource/Pseudophilautus_variabilis, http://dbpedia.org/resource/Pseudophilautus_zal, http://dbpedia.org/resource/Atlas_bear, http://dbpedia.org/resource/Gastric-brooding_frog, http://dbpedia.org/resource/Hagerman_horse, http://dbpedia.org/resource/Pig-footed_bandicoot, http://dbpedia.org/resource/Formosan_clouded_leopard, http://dbpedia.org/resource/Robust_white-eye, http://dbpedia.org/resource/Bush_moa, http://dbpedia.org/resource/Dinornis, http://dbpedia.org/resource/Dominican_green-and-yellow_macaw, http://dbpedia.org/resource/Eastern_moa, http://dbpedia.org/resource/Jamaican_red_macaw, http://dbpedia.org/resource/North_Island_giant_moa, http://dbpedia.org/resource/Pachyornis_australis, http://dbpedia.org/resource/Saccocoma, http://dbpedia.org/resource/South_Island_giant_moa, http://dbpedia.org/resource/Bramble_Cay_melomys, http://dbpedia.org/resource/Hirasea_acutissima, http://dbpedia.org/resource/Interrupted_rocksnail, http://dbpedia.org/resource/Bishop's_ʻōʻō, http://dbpedia.org/resource/Oʻahu_ʻakepa, http://dbpedia.org/resource/Belgrandiella_intermedia, http://dbpedia.org/resource/Discula_lyelliana, http://dbpedia.org/resource/Discula_tetrica, http://dbpedia.org/resource/Discus_engonatus, http://dbpedia.org/resource/Discus_retextus, http://dbpedia.org/resource/Discus_textilis, http://dbpedia.org/resource/Geomitra_delphinuloides, http://dbpedia.org/resource/Geomitra_grabhami, http://dbpedia.org/resource/Graecoanatolica_macedonica, http://dbpedia.org/resource/Gyralina_hausdorfi, http://dbpedia.org/resource/Janulus_pompylius, http://dbpedia.org/resource/Keraea_garachicoensis, http://dbpedia.org/resource/Leiostyla_abbreviata, http://dbpedia.org/resource/Leiostyla_gibba, http://dbpedia.org/resource/Leiostyla_lamellosa, http://dbpedia.org/resource/Ohridohauffenia_drimica, http://dbpedia.org/resource/Parmacella_gervaisii, http://dbpedia.org/resource/Pseudocampylaea_loweii, http://dbpedia.org/resource/Tobias'_caddisfly, http://dbpedia.org/resource/Zonites_santoriniensis, http://dbpedia.org/resource/Zonites_siphnicus, http://dbpedia.org/resource/Palaeoloxodon_mnaidriensis, http://dbpedia.org/resource/Tubuai_rail, http://dbpedia.org/resource/Broad-billed_moa, http://dbpedia.org/resource/Wide-toothed_hutia, http://dbpedia.org/resource/North_African_elephant, http://dbpedia.org/resource/Canary_Islands_oystercatcher, http://dbpedia.org/resource/Eastern_Canary_Islands_chiffchaff, http://dbpedia.org/resource/Roque_Chico_de_Salmor_giant_lizard, http://dbpedia.org/resource/Broad-billed_parrot, http://dbpedia.org/resource/Domed_Mauritius_giant_tortoise, http://dbpedia.org/resource/Mascarene_coot, http://dbpedia.org/resource/Mascarene_grey_parakeet, http://dbpedia.org/resource/Mauritian_giant_skink, http://dbpedia.org/resource/Mauritius_blue_pigeon, http://dbpedia.org/resource/Mauritius_night_heron, http://dbpedia.org/resource/Mauritius_owl, http://dbpedia.org/resource/Red_rail, http://dbpedia.org/resource/Round_Island_burrowing_boa, http://dbpedia.org/resource/Réunion_ibis, http://dbpedia.org/resource/Saddle-backed_Mauritius_giant_tortoise, http://dbpedia.org/resource/Small_Mauritian_flying_fox, http://dbpedia.org/resource/Spotted_green_pigeon, http://dbpedia.org/resource/Mamo, http://dbpedia.org/resource/Bonin_wood_pigeon, http://dbpedia.org/resource/Sterna_milne-edwardsii, http://dbpedia.org/resource/Tahiti_rail, http://dbpedia.org/resource/Tristramella_intermedia, http://dbpedia.org/resource/Desert_rat-kangaroo, http://dbpedia.org/resource/Nullarbor_dwarf_bettong, http://dbpedia.org/resource/Mascarene_teal, http://dbpedia.org/resource/Blue-gray_mouse, http://dbpedia.org/resource/Buhler's_coryphomys, http://dbpedia.org/resource/Candango_mouse, http://dbpedia.org/resource/Lagostomus_crassus, http://dbpedia.org/resource/Lesser_stick-nest_rat, http://dbpedia.org/resource/Megalomys_desmarestii, http://dbpedia.org/resource/Noronhomys, http://dbpedia.org/resource/St._Michel_nesophontes, http://dbpedia.org/resource/Eodesmatodon, http://dbpedia.org/resource/Dysoneuridae, http://dbpedia.org/resource/Juliomys_anoblepas, http://dbpedia.org/resource/Whiteline_topminnow, http://dbpedia.org/resource/Agrotis_cremata, http://dbpedia.org/resource/Agrotis_melanoneura, http://dbpedia.org/resource/Agrotis_microreas, http://dbpedia.org/resource/Agrotis_panoplias, http://dbpedia.org/resource/Agrotis_photophila, http://dbpedia.org/resource/Agrotis_tephrias, http://dbpedia.org/resource/Kākāwahie, http://dbpedia.org/resource/Lāna'i_hookbill, http://dbpedia.org/resource/Pentarthrum_blackburni, http://dbpedia.org/resource/Rhyacophila_amabilis, http://dbpedia.org/resource/Rhyncogonus_bryani, http://dbpedia.org/resource/Triaenodes_phalacris, http://dbpedia.org/resource/Trigonoscuta_rossi, http://dbpedia.org/resource/Trigonoscuta_yorbalindae, http://dbpedia.org/resource/Partula_arguta, http://dbpedia.org/resource/Partula_atilis, http://dbpedia.org/resource/Partula_auriculata, http://dbpedia.org/resource/Partula_bilineata, http://dbpedia.org/resource/Partula_crassilabris, http://dbpedia.org/resource/Partula_cuneata, http://dbpedia.org/resource/Partula_cytherea, http://dbpedia.org/resource/Partula_dolichostoma, http://dbpedia.org/resource/Partula_dolorosa, http://dbpedia.org/resource/Partula_eremita, http://dbpedia.org/resource/Partula_filosa, http://dbpedia.org/resource/Partula_formosa, http://dbpedia.org/resource/Partula_garretti, http://dbpedia.org/resource/Partula_labrusca, http://dbpedia.org/resource/Partula_leptochila, http://dbpedia.org/resource/Partula_levistriata, http://dbpedia.org/resource/Partula_lutea, http://dbpedia.org/resource/Partula_planilabrum, http://dbpedia.org/resource/Partula_producta, http://dbpedia.org/resource/Partula_protracta, http://dbpedia.org/resource/Partula_remota, http://dbpedia.org/resource/Partula_rustica, http://dbpedia.org/resource/Partula_sagitta, http://dbpedia.org/resource/Partula_thalia, http://dbpedia.org/resource/Partula_umbilicata, http://dbpedia.org/resource/Xestospiza, http://dbpedia.org/resource/Techirghiol_stickleback, http://dbpedia.org/resource/Bulldog_rat, http://dbpedia.org/resource/Maclear's_rat, http://dbpedia.org/resource/Ptychodus_mortoni, http://dbpedia.org/resource/Caribbean_monk_seal, http://dbpedia.org/resource/Falkland_Islands_wolf, http://dbpedia.org/resource/Haast's_eagle, http://dbpedia.org/resource/Western_black_rhinoceros, http://dbpedia.org/resource/Kawekaweau, http://dbpedia.org/resource/Sea_mink, http://dbpedia.org/resource/Broad-faced_potoroo, http://dbpedia.org/resource/Conquered_lorikeet, http://dbpedia.org/resource/Sinoto's_lorikeet, http://dbpedia.org/resource/Macroscincus, http://dbpedia.org/resource/Rallicola_extinctus, http://dbpedia.org/resource/High-billed_crow, http://dbpedia.org/resource/Lake_Pedder_earthworm, http://dbpedia.org/resource/New_Zealand_grayling, http://dbpedia.org/resource/Palaeoloxodon_chaniensis, http://dbpedia.org/resource/Henderson_archaic_pigeon, http://dbpedia.org/resource/Henderson_imperial_pigeon, http://dbpedia.org/resource/Balearic_shrew, http://dbpedia.org/resource/Sardinian_shrew, http://dbpedia.org/resource/Perrin's_cave_beetle, http://dbpedia.org/resource/Dupontia_proletaria, http://dbpedia.org/resource/Harmogenanina_linophora, http://dbpedia.org/resource/Harmogenanina_subdetecta, http://dbpedia.org/resource/Unio_cariei, http://dbpedia.org/resource/Biotocus_turbinatus, http://dbpedia.org/resource/Digerus_gibberulus, http://dbpedia.org/resource/Littoridina_gaudichaudii, http://dbpedia.org/resource/Megalobulimus_cardosoi, http://dbpedia.org/resource/Rhinodrilus_fafner, http://dbpedia.org/resource/Gravicalymene, http://dbpedia.org/resource/Greater_akialoa, http://dbpedia.org/resource/Primitive_koa_finch, http://dbpedia.org/resource/Scissor-billed_koa_finch, http://dbpedia.org/resource/Thaumatodon_multilamellata, http://dbpedia.org/resource/Partula_exigua, http://dbpedia.org/resource/Hawaii_mamo, http://dbpedia.org/resource/Aepyornis, http://dbpedia.org/resource/Atlas_wild_ass, http://dbpedia.org/resource/Cuban_coney, http://dbpedia.org/resource/Eastern_cougar, http://dbpedia.org/resource/Japanese_sea_lion, http://dbpedia.org/resource/Majorcan_giant_dormouse, http://dbpedia.org/resource/Malagasy_crowned_eagle, http://dbpedia.org/resource/Malagasy_hippopotamus, http://dbpedia.org/resource/Mammuthus_creticus, http://dbpedia.org/resource/Moho_(genus), http://dbpedia.org/resource/Sardinian_pika, http://dbpedia.org/resource/Tarpan, http://dbpedia.org/resource/Tyto_pollens, http://dbpedia.org/resource/Southern_black_rhinoceros, http://dbpedia.org/resource/Syrian_wild_ass, http://dbpedia.org/resource/Silver_trout, http://dbpedia.org/resource/Chatham_duck, http://dbpedia.org/resource/Forbes'_snipe, http://dbpedia.org/resource/New_Zealand_bittern, http://dbpedia.org/resource/Snipe-rail, http://dbpedia.org/resource/Lord_Howe_fantail, http://dbpedia.org/resource/Giant_solenodon, http://dbpedia.org/resource/Dusicyon_avus, http://dbpedia.org/resource/Maiden_rocksnail, http://dbpedia.org/resource/Lana'i_'alauahio, http://dbpedia.org/resource/Maui_Nui_'akialoa, http://dbpedia.org/resource/Oahu_'akialoa, http://dbpedia.org/resource/Aptenodytes_ridgeni, http://dbpedia.org/resource/Pemberton's_deer_mouse, http://dbpedia.org/resource/Coelodonta_tologoijensis, http://dbpedia.org/resource/Rhynchopsitta_phillipsi, http://dbpedia.org/resource/Catostomus_catostomus_cristatus, http://dbpedia.org/resource/Heteropsomys, http://dbpedia.org/resource/Anabarilius_macrolepis, http://dbpedia.org/resource/Jamaican_wood_rail, http://dbpedia.org/resource/Mauritian_turtle_dove, http://dbpedia.org/resource/Puerto_Rican_parakeet, http://dbpedia.org/resource/Red-throated_wood_rail, http://dbpedia.org/resource/Tahiti_crake, http://dbpedia.org/resource/Antillean_cave_rat, http://dbpedia.org/resource/Dysoneura, http://dbpedia.org/resource/Dysoneura_zherikhini, http://dbpedia.org/resource/Stephanorhinus, http://dbpedia.org/resource/Sri_Lanka_lion, http://dbpedia.org/resource/St_Kilda_house_mouse, http://dbpedia.org/resource/Advena_campbelli, http://dbpedia.org/resource/Angrobia_dulvertonensis, http://dbpedia.org/resource/Long-tailed_hopping_mouse, http://dbpedia.org/resource/Nancibella_quintalia, http://dbpedia.org/resource/Tornelasmias_capricorni, http://dbpedia.org/resource/White-footed_rabbit-rat, http://dbpedia.org/resource/Acanthometropus_pecatonica, http://dbpedia.org/resource/Alloperla_roberti, http://dbpedia.org/resource/Ameiva_cineracea, http://dbpedia.org/resource/American_chestnut_moth, http://dbpedia.org/resource/Amistad_gambusia, http://dbpedia.org/resource/Amphicyclotulus_guadeloupensis, http://dbpedia.org/resource/Anthony's_woodrat, http://dbpedia.org/resource/Ash_Meadows_killifish, http://dbpedia.org/resource/Atalaye_nesophontes, http://dbpedia.org/resource/Banff_longnose_dace, http://dbpedia.org/resource/Barbados_raccoon, http://dbpedia.org/resource/Bermuda_flicker, http://dbpedia.org/resource/Bermuda_hawk, http://dbpedia.org/resource/Bermuda_saw-whet_owl, http://dbpedia.org/resource/Blackfin_cisco, http://dbpedia.org/resource/Bunker's_woodrat, http://dbpedia.org/resource/Cachorrito_de_la_Trinidad, http://dbpedia.org/resource/Chestnut_clearwing_moth, http://dbpedia.org/resource/Chestnut_ermine_moth, http://dbpedia.org/resource/Chirostoma_bartoni, http://dbpedia.org/resource/Clappia_umbilicata, http://dbpedia.org/resource/Coleophora_leucochrysella, http://dbpedia.org/resource/Conozoa_hyalina, http://dbpedia.org/resource/Craugastor_escoces, http://dbpedia.org/resource/Cuban_ivory-billed_woodpecker, http://dbpedia.org/resource/Cuban_nesophontes, http://dbpedia.org/resource/Cyprinodon_ceciliae, http://dbpedia.org/resource/Deepwater_cisco, http://dbpedia.org/resource/Elimia_gibbera, http://dbpedia.org/resource/Elimia_macglameriana, http://dbpedia.org/resource/Goff's_pocket_gopher, http://dbpedia.org/resource/Graceful_priapella, http://dbpedia.org/resource/Greater_Cuban_nesophontes, http://dbpedia.org/resource/Gull_Island_vole, http://dbpedia.org/resource/Haitian_nesophontes, http://dbpedia.org/resource/Highnut, http://dbpedia.org/resource/Hispaniolan_edible_rat, http://dbpedia.org/resource/Imposter_hutia, http://dbpedia.org/resource/Incerticyclus_cinereus, http://dbpedia.org/resource/Incerticyclus_martinicensis, http://dbpedia.org/resource/Independence_Valley_tui_chub, http://dbpedia.org/resource/Insular_cave_rat, http://dbpedia.org/resource/Jamaica_giant_galliwasp, http://dbpedia.org/resource/Jamaican_monkey, http://dbpedia.org/resource/Leptoxis_torrefacta, http://dbpedia.org/resource/Lesser_Cuban_nesophontes, http://dbpedia.org/resource/Lesser_bilby, http://dbpedia.org/resource/Little_Swan_Island_hutia, http://dbpedia.org/resource/Longjaw_cisco, http://dbpedia.org/resource/Longnut, http://dbpedia.org/resource/Marcano's_solenodon, http://dbpedia.org/resource/Marstonia_olivacea, http://dbpedia.org/resource/Martinique_curly-tailed_lizard, http://dbpedia.org/resource/Martinique_giant_ameiva, http://dbpedia.org/resource/Maryland_darter, http://dbpedia.org/resource/Megalomys_luciae, http://dbpedia.org/resource/Montane_hutia, http://dbpedia.org/resource/Navassa_curly-tailed_lizard, http://dbpedia.org/resource/Neoplanorbis_carinatus, http://dbpedia.org/resource/Neoplanorbis_smithi, http://dbpedia.org/resource/Neoplanorbis_tantillus, http://dbpedia.org/resource/Neoplanorbis_umbilicatus, http://dbpedia.org/resource/Nesophontes, http://dbpedia.org/resource/Oleacina_guadeloupensis, http://dbpedia.org/resource/Oligoryzomys_victus, http://dbpedia.org/resource/Oriente_cave_rat, http://dbpedia.org/resource/Oryzomys_antillarum, http://dbpedia.org/resource/Oryzomys_nelsoni, http://dbpedia.org/resource/Pacifastacus_nigrescens, http://dbpedia.org/resource/Pallid_beach_mouse, http://dbpedia.org/resource/Parras_characodon, http://dbpedia.org/resource/Pennatomys, http://dbpedia.org/resource/Pentagenia_robusta, http://dbpedia.org/resource/Phleophagan_chestnut_moth, http://dbpedia.org/resource/Pleurodonte_desidens, http://dbpedia.org/resource/Procambarus_angustatus, http://dbpedia.org/resource/Puerto_Rican_hutia, http://dbpedia.org/resource/Puerto_Rican_nesophontes, http://dbpedia.org/resource/Saint_Croix_racer, http://dbpedia.org/resource/Samana_hutia, http://dbpedia.org/resource/San_Martín_Island_woodrat, http://dbpedia.org/resource/Slender_Cuban_nesophontes, http://dbpedia.org/resource/Smith_Island_cottontail, http://dbpedia.org/resource/Stonemyia_volutina, http://dbpedia.org/resource/Stygobromus_lucifugus, http://dbpedia.org/resource/Syncaris_pasadenae, http://dbpedia.org/resource/Tacoma_pocket_gopher, http://dbpedia.org/resource/Tecopa_pupfish, http://dbpedia.org/resource/Torre's_cave_rat, http://dbpedia.org/resource/Triaenodes_tridonata, http://dbpedia.org/resource/Tule_shrew, http://dbpedia.org/resource/Western_Cuban_nesophontes, http://dbpedia.org/resource/Xerces_blue, http://dbpedia.org/resource/O'ahu_petrel, http://dbpedia.org/resource/New_Zealand_musk_duck, http://dbpedia.org/resource/Hypnomys, http://dbpedia.org/resource/Leithia, http://dbpedia.org/resource/Mauritian_wood_pigeon, http://dbpedia.org/resource/Ryukyu_wood_pigeon, http://dbpedia.org/resource/Phyllococcus_oahuensis, http://dbpedia.org/resource/Cebu_hanging_parrot, http://dbpedia.org/resource/Vegas_Valley_leopard_frog, http://dbpedia.org/resource/‘Eua_rail, http://dbpedia.org/resource/Conilurus_capricornensis, http://dbpedia.org/resource/Aldabra_banded_snail, http://dbpedia.org/resource/Gastrocopta_chichijimana, http://dbpedia.org/resource/Gastrocopta_ogasawarana, http://dbpedia.org/resource/Hirasea_planulata, http://dbpedia.org/resource/Lamellidea_monodonta, http://dbpedia.org/resource/Lamellidea_nakadai, http://dbpedia.org/resource/Littoraria_flammea, http://dbpedia.org/resource/Ryukyu_kingfisher, http://dbpedia.org/resource/Tristramella_magdelainae, http://dbpedia.org/resource/Trochoidea_picardi, http://dbpedia.org/resource/Vitrinula_chaunax, http://dbpedia.org/resource/Vitrinula_chichijimana, http://dbpedia.org/resource/Vitrinula_hahajimana, http://dbpedia.org/resource/Red-headed_macaw, http://dbpedia.org/resource/Mauke_starling, http://dbpedia.org/resource/Samoana_jackieburchi, http://dbpedia.org/resource/Bigmouth_rocksnail, http://dbpedia.org/resource/Clydonitaceae, http://dbpedia.org/resource/Hesperocolletes_douglasi, http://dbpedia.org/resource/King_Kong_grosbeak, http://dbpedia.org/resource/Wahi_grosbeak, http://dbpedia.org/resource/Xestospiza_conica, http://dbpedia.org/resource/Xestospiza_fastigialis, http://dbpedia.org/resource/Eelgrass_limpet, http://dbpedia.org/resource/Tribonyx_hodgenorum, http://dbpedia.org/resource/Gölçük_toothcarp, http://dbpedia.org/resource/Ungava_brown_bear, http://dbpedia.org/resource/Norfolk_pigeon, http://dbpedia.org/resource/Norfolk_starling, http://dbpedia.org/resource/Norfolk_thrush, http://dbpedia.org/resource/Norfolk_triller, http://dbpedia.org/resource/Cryptoprocta_spelea, http://dbpedia.org/resource/Ptilodus, http://dbpedia.org/resource/Tristramella_sacra, http://dbpedia.org/resource/Urania_sloanus, http://dbpedia.org/resource/Domed_Rodrigues_giant_tortoise, http://dbpedia.org/resource/Saddle-backed_Rodrigues_giant_tortoise, http://dbpedia.org/resource/Auckland_Islands_shore_plover, http://dbpedia.org/resource/Bluff_Downs_giant_python, http://dbpedia.org/resource/Utah_Lake_sculpin, http://dbpedia.org/resource/Apteribis, http://dbpedia.org/resource/Jamaican_ibis, http://dbpedia.org/resource/Nesotrochis, http://dbpedia.org/resource/Moorea_sandpiper, http://dbpedia.org/resource/Saint_Helena_earwig, http://dbpedia.org/resource/Gould's_mouse, http://dbpedia.org/resource/Japanese_river_otter, http://dbpedia.org/resource/Puerto_Rican_flower_bat, http://dbpedia.org/resource/Laysan_dropseed_noctuid_moth, http://dbpedia.org/resource/Laysan_noctuid_moth, http://dbpedia.org/resource/Procellaris_grotis_noctuid_moth, http://dbpedia.org/resource/Mangaia_crake, http://dbpedia.org/resource/Mangaia_rail, http://dbpedia.org/resource/Megaoryzomys, http://dbpedia.org/resource/Nesoryzomys_indefessus, http://dbpedia.org/resource/Mecodema_punctellum, http://dbpedia.org/resource/Atelopus_longirostris, http://dbpedia.org/resource/Atelopus_vogli, http://dbpedia.org/resource/Gerandibis, http://dbpedia.org/resource/Kona_giant_looper_moth, http://dbpedia.org/resource/Great_ground_dove, http://dbpedia.org/resource/Rodrigues_blue_pigeon, http://dbpedia.org/resource/Cyprinodon_arcuatus, http://dbpedia.org/resource/Placostylus_cuniculinsulae, http://dbpedia.org/resource/Lymnaea_tomentosa_hamiltoni, http://dbpedia.org/resource/Roper_River_scrub_robin, http://dbpedia.org/resource/Haitian_edible_rat, http://dbpedia.org/resource/Battus_polydamas_antiquus, http://dbpedia.org/resource/Partula_approximata, http://dbpedia.org/resource/Partula_callifera, http://dbpedia.org/resource/Partula_candida, http://dbpedia.org/resource/Partula_castanea, http://dbpedia.org/resource/Partula_cedista, http://dbpedia.org/resource/Partula_citrina, http://dbpedia.org/resource/Partula_compacta, http://dbpedia.org/resource/Partula_imperforata, http://dbpedia.org/resource/Partula_levilineata, http://dbpedia.org/resource/Partula_microstoma, http://dbpedia.org/resource/Partula_protea, http://dbpedia.org/resource/Partula_radiata, http://dbpedia.org/resource/Partula_raiatensis, http://dbpedia.org/resource/Partula_robusta, http://dbpedia.org/resource/Partula_salifera, http://dbpedia.org/resource/Partula_variabilis, http://dbpedia.org/resource/Partula_vittata, http://dbpedia.org/resource/Perritos_de_sandia, http://dbpedia.org/resource/Pseudobactricia_ridleyi, http://dbpedia.org/resource/Spiny-knee_leaf_frog, http://dbpedia.org/resource/Dysoneura_trifurcata, http://dbpedia.org/resource/Acanthobrama_hulensis, http://dbpedia.org/resource/Alburnus_akili, http://dbpedia.org/resource/Ameca_shiner, http://dbpedia.org/resource/Barbus_microbarbis, http://dbpedia.org/resource/Chondrostoma_scodrense, http://dbpedia.org/resource/Clear_Lake_splittail, http://dbpedia.org/resource/Cyprinus_yilongensis, http://dbpedia.org/resource/Durango_shiner, http://dbpedia.org/resource/Evarra_bustamantei, http://dbpedia.org/resource/Evarra_eigenmanni, http://dbpedia.org/resource/Evarra_tlahuacensis, http://dbpedia.org/resource/Harelip_sucker, http://dbpedia.org/resource/Las_Vegas_dace, http://dbpedia.org/resource/Pahranagat_spinedace, http://dbpedia.org/resource/Phantom_shiner, http://dbpedia.org/resource/Red-bellied_gracile_opossum, http://dbpedia.org/resource/Salado_shiner, http://dbpedia.org/resource/Snake_River_sucker, http://dbpedia.org/resource/Stumptooth_minnow, http://dbpedia.org/resource/Telestes_ukliva, http://dbpedia.org/resource/Thicktail_chub, http://dbpedia.org/resource/Dusky_flying_fox, http://dbpedia.org/resource/Guam_flying_fox, http://dbpedia.org/resource/Large_Palau_flying_fox, http://dbpedia.org/resource/Wood_harrier, http://dbpedia.org/resource/Japanese_wolf, http://dbpedia.org/resource/Rodrigues_day_gecko, http://dbpedia.org/resource/Rodrigues_giant_day_gecko, http://dbpedia.org/resource/Salvelinus_neocomensis, http://dbpedia.org/resource/Ratas_Island_lizard, http://dbpedia.org/resource/Santo_Stefano_lizard, http://dbpedia.org/resource/Mauritius_sheldgoose, http://dbpedia.org/resource/Réunion_sheldgoose, http://dbpedia.org/resource/Malagasy_lapwing, http://dbpedia.org/resource/Daphnobela_juncea, http://dbpedia.org/resource/Flores_cave_rat, http://dbpedia.org/resource/Great_Maui_crake, http://dbpedia.org/resource/Great_O‘ahu_crake, http://dbpedia.org/resource/Highland_finch, http://dbpedia.org/resource/Hoopoe-billed_ʻakialoa, http://dbpedia.org/resource/Huahine_cuckoo-dove, http://dbpedia.org/resource/Huahine_gull, http://dbpedia.org/resource/Huahine_rail, http://dbpedia.org/resource/Huahine_swamphen, http://dbpedia.org/resource/Malagasy_sheldgoose, http://dbpedia.org/resource/Niue_night_heron, http://dbpedia.org/resource/Niue_rail, http://dbpedia.org/resource/Nuku_Hiva_rail, http://dbpedia.org/resource/Stout-legged_finch, http://dbpedia.org/resource/Tahuata_rail, http://dbpedia.org/resource/Ua_Huka_rail, http://dbpedia.org/resource/Achatinella_abbreviata, http://dbpedia.org/resource/Achatinella_buddii, http://dbpedia.org/resource/Achatinella_caesia, http://dbpedia.org/resource/Achatinella_casta, http://dbpedia.org/resource/Achatinella_decora, http://dbpedia.org/resource/Achatinella_dimorpha, http://dbpedia.org/resource/Achatinella_elegans, http://dbpedia.org/resource/Achatinella_juddii, http://dbpedia.org/resource/Achatinella_juncea, http://dbpedia.org/resource/Achatinella_lehuiensis, http://dbpedia.org/resource/Achatinella_livida, http://dbpedia.org/resource/Achatinella_papyracea, http://dbpedia.org/resource/Achatinella_spaldingi, http://dbpedia.org/resource/Achatinella_thaanumi, http://dbpedia.org/resource/Achatinella_valida, http://dbpedia.org/resource/Nesoryzomys_darwini, http://dbpedia.org/resource/Hoan_Kiem_turtle, http://dbpedia.org/resource/Megalagrion_jugorum, http://dbpedia.org/resource/Afrocyclops_pauliani, http://dbpedia.org/resource/Aplocheilichthys_sp._nov._'Naivasha', http://dbpedia.org/resource/Caldwellia_philyrina, http://dbpedia.org/resource/Campolaemus_perexilis, http://dbpedia.org/resource/Chilonopsis_blofeldi, http://dbpedia.org/resource/Chilonopsis_exulatus, http://dbpedia.org/resource/Chilonopsis_helena, http://dbpedia.org/resource/Chilonopsis_melanoides, http://dbpedia.org/resource/Chilonopsis_nonpareil, http://dbpedia.org/resource/Chilonopsis_subplicatus, http://dbpedia.org/resource/Chilonopsis_subtruncatus, http://dbpedia.org/resource/Chilonopsis_turtoni, http://dbpedia.org/resource/Colparion_madgei, http://dbpedia.org/resource/Ctenoglypta_newtoni, http://dbpedia.org/resource/Cyclophorus_horridulum, http://dbpedia.org/resource/Cyclosurus_mariei, http://dbpedia.org/resource/Deloneura_immaculata, http://dbpedia.org/resource/Eastwood's_long-tailed_seps, http://dbpedia.org/resource/Erepta_nevilli, http://dbpedia.org/resource/Gibbus_lyonetianus, http://dbpedia.org/resource/Gonidomus_newtoni, http://dbpedia.org/resource/Gonospira_nevilli, http://dbpedia.org/resource/Gulella_mayottensis, http://dbpedia.org/resource/Helenoconcha_leptalea, http://dbpedia.org/resource/Helenoconcha_minutissima, http://dbpedia.org/resource/Helenoconcha_polyodon, http://dbpedia.org/resource/Helenoconcha_pseustes, http://dbpedia.org/resource/Helenoconcha_sexdentata, http://dbpedia.org/resource/Helenodiscus_bilamellata, http://dbpedia.org/resource/Helenodiscus_vernoni, http://dbpedia.org/resource/Lepidochrysops_hypopolia, http://dbpedia.org/resource/Libythea_cinyras, http://dbpedia.org/resource/Pachnodus_velutinus, http://dbpedia.org/resource/Pachystyla_rufozonata, http://dbpedia.org/resource/Pantanodon_madagascariensis, http://dbpedia.org/resource/Pseudohelenoconcha_spurca, http://dbpedia.org/resource/Ptychochromis_onilahy, http://dbpedia.org/resource/Pupilla_obliquicosta, http://dbpedia.org/resource/Rhachis_comorensis, http://dbpedia.org/resource/Rhachis_sanguineus, http://dbpedia.org/resource/Tropidophora_desmazuresi, http://dbpedia.org/resource/Tropidophora_semilineata, http://dbpedia.org/resource/Typhlops_cariei, http://dbpedia.org/resource/Bay_thrush, http://dbpedia.org/resource/Plate-toothed_giant_hutia, http://dbpedia.org/resource/Mount_Glorious_day_frog, http://dbpedia.org/resource/Acorn_pearly_mussel, http://dbpedia.org/resource/Acorn_ramshorn, http://dbpedia.org/resource/Agate_rocksnail, http://dbpedia.org/resource/Alabama_clubshell, http://dbpedia.org/resource/Alabama_pigtoe, http://dbpedia.org/resource/Amastra_albolabris, http://dbpedia.org/resource/Amastra_cornea, http://dbpedia.org/resource/Amastra_crassilabrum, http://dbpedia.org/resource/Amastra_elongata, http://dbpedia.org/resource/Amastra_forbesi, http://dbpedia.org/resource/Amastra_pellucida, http://dbpedia.org/resource/Amastra_porcus, http://dbpedia.org/resource/Amastra_reticulata, http://dbpedia.org/resource/Amastra_subrostrata, http://dbpedia.org/resource/Amastra_subsoror, http://dbpedia.org/resource/Amastra_tenuispira, http://dbpedia.org/resource/Amastra_umbilicata, http://dbpedia.org/resource/Angled_riffleshell, http://dbpedia.org/resource/Arc-form_pearly_mussel, http://dbpedia.org/resource/Arcuate_pearly_mussel, http://dbpedia.org/resource/Auriculella_expansa, http://dbpedia.org/resource/Auriculella_uniplicata, http://dbpedia.org/resource/Boulder_snail, http://dbpedia.org/resource/Brown_pigtoe, http://dbpedia.org/resource/Bythinella_intermedia, http://dbpedia.org/resource/Carolina_elktoe, http://dbpedia.org/resource/Closed_elimia, http://dbpedia.org/resource/Cobble_elimia, http://dbpedia.org/resource/Constricted_elimia, http://dbpedia.org/resource/Coosa_elktoe, http://dbpedia.org/resource/Coosa_pigtoe, http://dbpedia.org/resource/Coosa_rocksnail, http://dbpedia.org/resource/Cumberland_leafshell, http://dbpedia.org/resource/Elimia_lachryma, http://dbpedia.org/resource/Excised_slitshell, http://dbpedia.org/resource/Fine-rayed_pearly_mussel, http://dbpedia.org/resource/Fish_Lake_physa, http://dbpedia.org/resource/Forkshell, http://dbpedia.org/resource/Fusiform_elimia, http://dbpedia.org/resource/Gyrotoma, http://dbpedia.org/resource/Hazel_pigtoe, http://dbpedia.org/resource/Hearty_elimia, http://dbpedia.org/resource/Heavy_pigtoe, http://dbpedia.org/resource/High-spired_elimia, http://dbpedia.org/resource/Leucocharis_loyaltiensis, http://dbpedia.org/resource/Leucocharis_porphyrocheila, http://dbpedia.org/resource/Lined_pocketbook, http://dbpedia.org/resource/Lyrate_rocksnail, http://dbpedia.org/resource/Lyropupa_perlonga, http://dbpedia.org/resource/Mautodontha_acuticosta, http://dbpedia.org/resource/Mautodontha_consimilis, http://dbpedia.org/resource/Mautodontha_consobrina, http://dbpedia.org/resource/Mautodontha_maupiensis, http://dbpedia.org/resource/Mautodontha_parvidens, http://dbpedia.org/resource/Mautodontha_punctiperforata, http://dbpedia.org/resource/Mautodontha_saintjohni, http://dbpedia.org/resource/Mautodontha_subtilis, http://dbpedia.org/resource/Mautodontha_unilamellata, http://dbpedia.org/resource/Mautodontha_zebrina, http://dbpedia.org/resource/Mount_Alifana_partula, http://dbpedia.org/resource/Mount_Matafao_different_snail, http://dbpedia.org/resource/Narrow_catspaw, http://dbpedia.org/resource/Nearby_pearly_mussel, http://dbpedia.org/resource/Newcombia_philippiana, http://dbpedia.org/resource/Ochlockonee_arcmussel, http://dbpedia.org/resource/Ovate_clubshell, http://dbpedia.org/resource/Pachystyla_waynepagei, http://dbpedia.org/resource/Pagoda_slitshell, http://dbpedia.org/resource/Panulena_perrugosa, http://dbpedia.org/resource/Partula_aurantia, http://dbpedia.org/resource/Partula_fusca, http://dbpedia.org/resource/Partulina_crassa, http://dbpedia.org/resource/Partulina_montagui, http://dbpedia.org/resource/Perdicella_fulgurans, http://dbpedia.org/resource/Perdicella_maniensis, http://dbpedia.org/resource/Perdicella_zebra, http://dbpedia.org/resource/Perdicella_zebrina, http://dbpedia.org/resource/Pupa_elimia, http://dbpedia.org/resource/Puzzle_elimia, http://dbpedia.org/resource/Pygmy_elimia, http://dbpedia.org/resource/Pyramid_slitshell, http://dbpedia.org/resource/Recovery_pearly_mussel, http://dbpedia.org/resource/Ribbed_elimia, http://dbpedia.org/resource/Ribbed_slitshell, http://dbpedia.org/resource/Rotund_rocksnail, http://dbpedia.org/resource/Rough-lined_elimia, http://dbpedia.org/resource/Round_slitshell, http://dbpedia.org/resource/Sampson's_naiad, http://dbpedia.org/resource/Scioto_pigtoe, http://dbpedia.org/resource/Shoal_sprite, http://dbpedia.org/resource/Short-spired_elimia, http://dbpedia.org/resource/Striate_slitshell, http://dbpedia.org/resource/Striped_rocksnail, http://dbpedia.org/resource/True_pigtoe, http://dbpedia.org/resource/Turgid_riffle_shell, http://dbpedia.org/resource/Yellow_pigtoe, http://dbpedia.org/resource/Dutch_Alcon_Blue, http://dbpedia.org/resource/Libera_subcavernula, http://dbpedia.org/resource/Libera_tumuloides, http://dbpedia.org/resource/Oncovena, http://dbpedia.org/resource/Plectostoma_sciaphilum, http://dbpedia.org/resource/Spirobolellus_praslinus, http://dbpedia.org/resource/Houting, http://dbpedia.org/resource/Amsterdam_wigeon, http://dbpedia.org/resource/Asian_ostrich, http://dbpedia.org/resource/Henderson_ground_dove, http://dbpedia.org/resource/Turtle-jawed_moa-nalo, http://dbpedia.org/resource/Salmo_pallaryi, http://dbpedia.org/resource/Ptychochromoides_itasy, http://dbpedia.org/resource/Eurygnathohippus_woldegabrieli, http://dbpedia.org/resource/İznik_shemaya, http://dbpedia.org/resource/Sardinian_lynx, http://dbpedia.org/resource/Maui_ʻakepa, http://dbpedia.org/resource/Ciridops, http://dbpedia.org/resource/Schizoglossa_major, http://dbpedia.org/resource/Heliaster_solaris, http://dbpedia.org/resource/Kenai_Peninsula_wolf, http://dbpedia.org/resource/Levuana_moth, http://dbpedia.org/resource/Scioto_madtom, http://dbpedia.org/resource/Tonga_ground_skink, http://dbpedia.org/resource/Chloridops, http://dbpedia.org/resource/Eğirdir_minnow, http://dbpedia.org/resource/Clavicoccus_erinaceus, http://dbpedia.org/resource/Dryophthorus_distinguendus, http://dbpedia.org/resource/Hygrotus_artus, http://dbpedia.org/resource/Liocypris_grandis, http://dbpedia.org/resource/Neduba_extincta, http://dbpedia.org/resource/Emperoptera_mirabilis, http://dbpedia.org/resource/Marquesas_cuckoo-dove, http://dbpedia.org/resource/Carelia_anceophila, http://dbpedia.org/resource/Carelia_bicolor, http://dbpedia.org/resource/Carelia_cochlea, http://dbpedia.org/resource/Carelia_cumingiana, http://dbpedia.org/resource/Carelia_dolei, http://dbpedia.org/resource/Carelia_evelynae, http://dbpedia.org/resource/Carelia_glossema, http://dbpedia.org/resource/Carelia_hyattiana, http://dbpedia.org/resource/Carelia_kalalauensis, http://dbpedia.org/resource/Carelia_knudseni, http://dbpedia.org/resource/Carelia_lirata, http://dbpedia.org/resource/Carelia_lymani, http://dbpedia.org/resource/Carelia_mirabilis, http://dbpedia.org/resource/Carelia_necra, http://dbpedia.org/resource/Carelia_olivacea, http://dbpedia.org/resource/Carelia_paradoxa, http://dbpedia.org/resource/Carelia_periscelis, http://dbpedia.org/resource/Carelia_pilsbryi, http://dbpedia.org/resource/Carelia_sinclairi, http://dbpedia.org/resource/Carelia_tenebrosa, http://dbpedia.org/resource/Carelia_turricula, http://dbpedia.org/resource/Drosophila_lanaiensis, http://dbpedia.org/resource/Oodemas_laysanensis, http://dbpedia.org/resource/Daphnobela_minutissima, http://dbpedia.org/resource/Daphnobela_miocaenica] [INFO] Predicted Answer: [] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 53, P: 0.367, R: 0.403, macro F1: 0.364, macro F1*: 0.384 -[INFO] QALD Cumulative metrics, sample: 53, P: 0.518, R: 0.403, macro F1: 0.364, QALD macro F1: 0.453 -[INFO] Current time: 409 ms, average: 3694.85 ms/ques, entity detection: 800.47 ms/ques (linking tool: 712.89 ms/ques), relation detection: 1178.09 ms/ques (similarity: 650.96 ms/ques), EDG generation: 95.96 ms/ques +[INFO] Cumulative metrics, sample: 53, P: 0.367, R: 0.403, macro F1: 0.384 +[INFO] QALD Cumulative metrics, sample: 53, P: 0.518, R: 0.403, average F1: 0.364, QALD macro F1: 0.453 +[INFO] Current question consumed time: 391 ms, average: 3470.70 ms/ques, entity detection: 731.60 ms/ques (linking tool: 651.00 ms/ques), relation detection: 1133.51 ms/ques (similarity: 655.57 ms/ques), EDG generation: 96.57 ms/ques [INFO] Entity Linking P: 0.18867924528301888, R: 0.18867924528301888, micro F1: 0.18867924528301888, macro F1: 0.18867924528301885 [INFO] Relation Linking P: 0.20754716981132076, R: 0.20754716981132076, micro F1: 0.20754716981132076, macro F1: 0.20754716981132076 [INFO] Type Linking P: 0.9433962264150944, R: 0.9433962264150944, micro F1: 0.9433962264150944, macro F1: 0.9433962264150944 @@ -802,9 +802,9 @@ Question 53: Who was the wife of President Lincoln? [INFO] Golden Answer: [http://dbpedia.org/resource/Mary_Todd_Lincoln] [INFO] Predicted Answer: [] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 54, P: 0.361, R: 0.396, macro F1: 0.357, macro F1*: 0.377 -[INFO] QALD Cumulative metrics, sample: 54, P: 0.527, R: 0.396, macro F1: 0.357, QALD macro F1: 0.452 -[INFO] Current time: 1111 ms, average: 3647.00 ms/ques, entity detection: 791.19 ms/ques (linking tool: 703.56 ms/ques), relation detection: 1160.70 ms/ques (similarity: 641.59 ms/ques), EDG generation: 94.37 ms/ques +[INFO] Cumulative metrics, sample: 54, P: 0.361, R: 0.396, macro F1: 0.377 +[INFO] QALD Cumulative metrics, sample: 54, P: 0.527, R: 0.396, average F1: 0.357, QALD macro F1: 0.452 +[INFO] Current question consumed time: 1032 ms, average: 3425.54 ms/ques, entity detection: 723.09 ms/ques (linking tool: 642.31 ms/ques), relation detection: 1116.83 ms/ques (similarity: 646.15 ms/ques), EDG generation: 94.96 ms/ques [INFO] Entity Linking P: 0.2037037037037037, R: 0.2037037037037037, micro F1: 0.2037037037037037, macro F1: 0.2037037037037037 [INFO] Relation Linking P: 0.2222222222222222, R: 0.2222222222222222, micro F1: 0.2222222222222222, macro F1: 0.2222222222222222 [INFO] Type Linking P: 0.9444444444444444, R: 0.9444444444444444, micro F1: 0.9444444444444444, macro F1: 0.9444444444444444 @@ -817,9 +817,9 @@ Question 54: How many awards has Bertrand Russell? [INFO] Golden Answer: [5] [INFO] Predicted Answer: [5] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 55, P: 0.372, R: 0.407, macro F1: 0.369, macro F1*: 0.389 -[INFO] QALD Cumulative metrics, sample: 55, P: 0.536, R: 0.407, macro F1: 0.369, QALD macro F1: 0.462 -[INFO] Current time: 1846 ms, average: 3614.25 ms/ques, entity detection: 781.20 ms/ques (linking tool: 693.80 ms/ques), relation detection: 1150.45 ms/ques (similarity: 640.87 ms/ques), EDG generation: 92.82 ms/ques +[INFO] Cumulative metrics, sample: 55, P: 0.372, R: 0.407, macro F1: 0.389 +[INFO] QALD Cumulative metrics, sample: 55, P: 0.536, R: 0.407, average F1: 0.369, QALD macro F1: 0.462 +[INFO] Current question consumed time: 1636 ms, average: 3393.00 ms/ques, entity detection: 713.38 ms/ques (linking tool: 633.00 ms/ques), relation detection: 1106.47 ms/ques (similarity: 645.38 ms/ques), EDG generation: 93.40 ms/ques [INFO] Entity Linking P: 0.2, R: 0.2, micro F1: 0.2, macro F1: 0.20000000000000004 [INFO] Relation Linking P: 0.21818181818181817, R: 0.21818181818181817, micro F1: 0.21818181818181817, macro F1: 0.21818181818181817 [INFO] Type Linking P: 0.9454545454545454, R: 0.9454545454545454, micro F1: 0.9454545454545454, macro F1: 0.9454545454545454 @@ -832,9 +832,9 @@ Question 55: Is Pamela Anderson a vegan? [INFO] Golden Answer: [true] [INFO] Predicted Answer: [true] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 56, P: 0.383, R: 0.417, macro F1: 0.380, macro F1*: 0.400 -[INFO] QALD Cumulative metrics, sample: 56, P: 0.544, R: 0.417, macro F1: 0.380, QALD macro F1: 0.472 -[INFO] Current time: 1361 ms, average: 3574.02 ms/ques, entity detection: 772.73 ms/ques (linking tool: 684.93 ms/ques), relation detection: 1138.88 ms/ques (similarity: 634.23 ms/ques), EDG generation: 91.29 ms/ques +[INFO] Cumulative metrics, sample: 56, P: 0.383, R: 0.417, macro F1: 0.400 +[INFO] QALD Cumulative metrics, sample: 56, P: 0.544, R: 0.417, average F1: 0.380, QALD macro F1: 0.472 +[INFO] Current question consumed time: 1257 ms, average: 3354.88 ms/ques, entity detection: 705.59 ms/ques (linking tool: 624.88 ms/ques), relation detection: 1095.18 ms/ques (similarity: 638.70 ms/ques), EDG generation: 91.88 ms/ques [INFO] Entity Linking P: 0.19642857142857142, R: 0.19642857142857142, micro F1: 0.19642857142857142, macro F1: 0.19642857142857142 [INFO] Relation Linking P: 0.23214285714285715, R: 0.23214285714285715, micro F1: 0.23214285714285715, macro F1: 0.23214285714285715 [INFO] Type Linking P: 0.9464285714285714, R: 0.9464285714285714, micro F1: 0.9464285714285714, macro F1: 0.9464285714285714 @@ -847,9 +847,9 @@ Question 56: Which space probes were sent into orbit around the sun? [INFO] Golden Answer: [http://dbpedia.org/resource/Ulysses_(spacecraft), http://dbpedia.org/resource/Interface_Region_Imaging_Spectrograph, http://dbpedia.org/resource/Reuven_Ramaty_High_Energy_Solar_Spectroscopic_Imager, http://dbpedia.org/resource/Helios_(spacecraft), http://dbpedia.org/resource/STEREO, http://dbpedia.org/resource/Solar_and_Heliospheric_Observatory, http://dbpedia.org/resource/Advanced_Composition_Explorer, http://dbpedia.org/resource/Hinode, http://dbpedia.org/resource/Solar_Dynamics_Observatory, http://dbpedia.org/resource/Solar_Probe_Plus, http://dbpedia.org/resource/TRACE, http://dbpedia.org/resource/Solar_Orbiter, http://dbpedia.org/resource/Genesis_(spacecraft), http://dbpedia.org/resource/Picard_(satellite), http://dbpedia.org/resource/Koronas-Foton, http://dbpedia.org/resource/Pioneer_6,_7,_8,_and_9] [INFO] Predicted Answer: [http://dbpedia.org/resource/Pushan, http://dbpedia.org/resource/Ansa_(Hinduism), http://dbpedia.org/resource/Tvastar, http://dbpedia.org/resource/Bhaga, http://dbpedia.org/resource/Aryaman, http://dbpedia.org/resource/Dhatri] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 57, P: 0.377, R: 0.410, macro F1: 0.374, macro F1*: 0.393 -[INFO] QALD Cumulative metrics, sample: 57, P: 0.535, R: 0.410, macro F1: 0.374, QALD macro F1: 0.464 -[INFO] Current time: 1901 ms, average: 3544.67 ms/ques, entity detection: 763.88 ms/ques (linking tool: 675.77 ms/ques), relation detection: 1127.86 ms/ques (similarity: 627.54 ms/ques), EDG generation: 89.98 ms/ques +[INFO] Cumulative metrics, sample: 57, P: 0.377, R: 0.410, macro F1: 0.393 +[INFO] QALD Cumulative metrics, sample: 57, P: 0.535, R: 0.410, average F1: 0.374, QALD macro F1: 0.464 +[INFO] Current question consumed time: 1787 ms, average: 3327.37 ms/ques, entity detection: 697.49 ms/ques (linking tool: 616.53 ms/ques), relation detection: 1085.28 ms/ques (similarity: 631.91 ms/ques), EDG generation: 90.58 ms/ques [INFO] Entity Linking P: 0.19298245614035087, R: 0.19298245614035087, micro F1: 0.19298245614035087, macro F1: 0.19298245614035087 [INFO] Relation Linking P: 0.22807017543859648, R: 0.22807017543859648, micro F1: 0.22807017543859648, macro F1: 0.22807017543859648 [INFO] Type Linking P: 0.9473684210526315, R: 0.9473684210526315, micro F1: 0.9473684210526315, macro F1: 0.9473684210526315 @@ -862,9 +862,9 @@ Question 57: Which German cities have more than 250000 inhabitants? [INFO] Golden Answer: [http://dbpedia.org/resource/Bielefeld, http://dbpedia.org/resource/Berlin, http://dbpedia.org/resource/Cologne, http://dbpedia.org/resource/Düsseldorf, http://dbpedia.org/resource/Essen, http://dbpedia.org/resource/Frankfurt, http://dbpedia.org/resource/Hamburg, http://dbpedia.org/resource/Hanover, http://dbpedia.org/resource/Mannheim, http://dbpedia.org/resource/Munich, http://dbpedia.org/resource/Nuremberg, http://dbpedia.org/resource/Bochum, http://dbpedia.org/resource/Bonn, http://dbpedia.org/resource/Dortmund, http://dbpedia.org/resource/Duisburg, http://dbpedia.org/resource/Mönchengladbach, http://dbpedia.org/resource/Münster, http://dbpedia.org/resource/Wuppertal, http://dbpedia.org/resource/Braunschweig, http://dbpedia.org/resource/Gelsenkirchen, http://dbpedia.org/resource/Karlsruhe, http://dbpedia.org/resource/Stuttgart, http://dbpedia.org/resource/Augsburg, http://dbpedia.org/resource/Bremen, http://dbpedia.org/resource/Wiesbaden] [INFO] Predicted Answer: [] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 58, P: 0.370, R: 0.403, macro F1: 0.367, macro F1*: 0.386 -[INFO] QALD Cumulative metrics, sample: 58, P: 0.543, R: 0.403, macro F1: 0.367, QALD macro F1: 0.462 -[INFO] Current time: 1744 ms, average: 3513.62 ms/ques, entity detection: 753.60 ms/ques (linking tool: 665.52 ms/ques), relation detection: 1115.38 ms/ques (similarity: 620.59 ms/ques), EDG generation: 88.74 ms/ques +[INFO] Cumulative metrics, sample: 58, P: 0.370, R: 0.403, macro F1: 0.386 +[INFO] QALD Cumulative metrics, sample: 58, P: 0.543, R: 0.403, average F1: 0.367, QALD macro F1: 0.462 +[INFO] Current question consumed time: 1590 ms, average: 3297.41 ms/ques, entity detection: 687.95 ms/ques (linking tool: 607.09 ms/ques), relation detection: 1072.93 ms/ques (similarity: 624.78 ms/ques), EDG generation: 89.24 ms/ques [INFO] Entity Linking P: 0.20689655172413793, R: 0.20689655172413793, micro F1: 0.20689655172413793, macro F1: 0.20689655172413793 [INFO] Relation Linking P: 0.2413793103448276, R: 0.2413793103448276, micro F1: 0.2413793103448276, macro F1: 0.2413793103448276 [INFO] Type Linking P: 0.9482758620689655, R: 0.9482758620689655, micro F1: 0.9482758620689655, macro F1: 0.9482758620689655 @@ -877,9 +877,9 @@ Question 58: Who is the daughter of Robert Kennedy married to? [INFO] Golden Answer: [http://dbpedia.org/resource/Andrew_Cuomo] [INFO] Predicted Answer: [http://dbpedia.org/resource/Tosca_Reno, ;] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 59, P: 0.364, R: 0.396, macro F1: 0.361, macro F1*: 0.379 -[INFO] QALD Cumulative metrics, sample: 59, P: 0.533, R: 0.396, macro F1: 0.361, QALD macro F1: 0.454 -[INFO] Current time: 1407 ms, average: 3477.92 ms/ques, entity detection: 746.03 ms/ques (linking tool: 657.53 ms/ques), relation detection: 1104.44 ms/ques (similarity: 615.15 ms/ques), EDG generation: 87.59 ms/ques +[INFO] Cumulative metrics, sample: 59, P: 0.364, R: 0.396, macro F1: 0.379 +[INFO] QALD Cumulative metrics, sample: 59, P: 0.533, R: 0.396, average F1: 0.361, QALD macro F1: 0.454 +[INFO] Current question consumed time: 1293 ms, average: 3263.44 ms/ques, entity detection: 681.44 ms/ques (linking tool: 600.00 ms/ques), relation detection: 1062.19 ms/ques (similarity: 618.93 ms/ques), EDG generation: 88.00 ms/ques [INFO] Entity Linking P: 0.2033898305084746, R: 0.2033898305084746, micro F1: 0.2033898305084746, macro F1: 0.20338983050847462 [INFO] Relation Linking P: 0.23728813559322035, R: 0.23728813559322035, micro F1: 0.23728813559322035, macro F1: 0.23728813559322035 [INFO] Type Linking P: 0.9491525423728814, R: 0.9491525423728814, micro F1: 0.9491525423728814, macro F1: 0.9491525423728814 @@ -892,9 +892,9 @@ Question 59: How many students does the Free University of Amsterdam have? [INFO] Golden Answer: [23656] [INFO] Predicted Answer: [31186] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 60, P: 0.358, R: 0.389, macro F1: 0.355, macro F1*: 0.373 -[INFO] QALD Cumulative metrics, sample: 60, P: 0.524, R: 0.389, macro F1: 0.355, QALD macro F1: 0.447 -[INFO] Current time: 2845 ms, average: 3467.37 ms/ques, entity detection: 742.50 ms/ques (linking tool: 653.57 ms/ques), relation detection: 1099.83 ms/ques (similarity: 619.08 ms/ques), EDG generation: 86.38 ms/ques +[INFO] Cumulative metrics, sample: 60, P: 0.358, R: 0.389, macro F1: 0.373 +[INFO] QALD Cumulative metrics, sample: 60, P: 0.524, R: 0.389, average F1: 0.355, QALD macro F1: 0.447 +[INFO] Current question consumed time: 2536 ms, average: 3251.32 ms/ques, entity detection: 676.93 ms/ques (linking tool: 596.30 ms/ques), relation detection: 1057.40 ms/ques (similarity: 622.85 ms/ques), EDG generation: 86.70 ms/ques [INFO] Entity Linking P: 0.2, R: 0.2, micro F1: 0.2, macro F1: 0.20000000000000004 [INFO] Relation Linking P: 0.23333333333333334, R: 0.23333333333333334, micro F1: 0.23333333333333334, macro F1: 0.23333333333333334 [INFO] Type Linking P: 0.95, R: 0.95, micro F1: 0.95, macro F1: 0.9500000000000001 @@ -907,9 +907,9 @@ Question 60: What is the revenue of IBM? [INFO] Golden Answer: [8.1741E10] [INFO] Predicted Answer: [8.1741E10] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 61, P: 0.368, R: 0.399, macro F1: 0.366, macro F1*: 0.383 -[INFO] QALD Cumulative metrics, sample: 61, P: 0.532, R: 0.399, macro F1: 0.366, QALD macro F1: 0.456 -[INFO] Current time: 2139 ms, average: 3445.59 ms/ques, entity detection: 736.74 ms/ques (linking tool: 647.28 ms/ques), relation detection: 1096.72 ms/ques (similarity: 616.51 ms/ques), EDG generation: 85.11 ms/ques +[INFO] Cumulative metrics, sample: 61, P: 0.368, R: 0.399, macro F1: 0.383 +[INFO] QALD Cumulative metrics, sample: 61, P: 0.532, R: 0.399, average F1: 0.366, QALD macro F1: 0.456 +[INFO] Current question consumed time: 1948 ms, average: 3229.95 ms/ques, entity detection: 671.64 ms/ques (linking tool: 590.44 ms/ques), relation detection: 1054.05 ms/ques (similarity: 620.16 ms/ques), EDG generation: 85.43 ms/ques [INFO] Entity Linking P: 0.19672131147540983, R: 0.19672131147540983, micro F1: 0.19672131147540983, macro F1: 0.1967213114754098 [INFO] Relation Linking P: 0.22950819672131148, R: 0.22950819672131148, micro F1: 0.22950819672131148, macro F1: 0.22950819672131148 [INFO] Type Linking P: 0.9508196721311475, R: 0.9508196721311475, micro F1: 0.9508196721311475, macro F1: 0.9508196721311475 @@ -922,9 +922,9 @@ Question 61: How many James Bond movies do exist? [INFO] Golden Answer: [34] [INFO] Predicted Answer: [] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 62, P: 0.362, R: 0.393, macro F1: 0.360, macro F1*: 0.377 -[INFO] QALD Cumulative metrics, sample: 62, P: 0.540, R: 0.393, macro F1: 0.360, QALD macro F1: 0.455 -[INFO] Current time: 876 ms, average: 3404.15 ms/ques, entity detection: 729.39 ms/ques (linking tool: 640.11 ms/ques), relation detection: 1082.15 ms/ques (similarity: 608.50 ms/ques), EDG generation: 83.87 ms/ques +[INFO] Cumulative metrics, sample: 62, P: 0.362, R: 0.393, macro F1: 0.377 +[INFO] QALD Cumulative metrics, sample: 62, P: 0.540, R: 0.393, average F1: 0.360, QALD macro F1: 0.455 +[INFO] Current question consumed time: 748 ms, average: 3189.92 ms/ques, entity detection: 664.69 ms/ques (linking tool: 583.76 ms/ques), relation detection: 1039.89 ms/ques (similarity: 612.06 ms/ques), EDG generation: 84.18 ms/ques [INFO] Entity Linking P: 0.20967741935483872, R: 0.20967741935483872, micro F1: 0.20967741935483872, macro F1: 0.20967741935483872 [INFO] Relation Linking P: 0.24193548387096775, R: 0.24193548387096775, micro F1: 0.24193548387096775, macro F1: 0.24193548387096775 [INFO] Type Linking P: 0.9516129032258065, R: 0.9516129032258065, micro F1: 0.9516129032258065, macro F1: 0.9516129032258065 @@ -937,9 +937,9 @@ Question 62: Who does the voice of Bart Simpson? [INFO] Golden Answer: [http://dbpedia.org/resource/Nancy_Cartwright] [INFO] Predicted Answer: [] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 63, P: 0.357, R: 0.387, macro F1: 0.354, macro F1*: 0.371 -[INFO] QALD Cumulative metrics, sample: 63, P: 0.547, R: 0.387, macro F1: 0.354, QALD macro F1: 0.453 -[INFO] Current time: 1058 ms, average: 3366.90 ms/ques, entity detection: 720.60 ms/ques (linking tool: 631.17 ms/ques), relation detection: 1069.25 ms/ques (similarity: 601.25 ms/ques), EDG generation: 83.87 ms/ques +[INFO] Cumulative metrics, sample: 63, P: 0.357, R: 0.387, macro F1: 0.371 +[INFO] QALD Cumulative metrics, sample: 63, P: 0.547, R: 0.387, average F1: 0.354, QALD macro F1: 0.453 +[INFO] Current question consumed time: 1050 ms, average: 3155.95 ms/ques, entity detection: 657.14 ms/ques (linking tool: 575.89 ms/ques), relation detection: 1027.54 ms/ques (similarity: 604.75 ms/ques), EDG generation: 84.21 ms/ques [INFO] Entity Linking P: 0.2222222222222222, R: 0.2222222222222222, micro F1: 0.2222222222222222, macro F1: 0.2222222222222222 [INFO] Relation Linking P: 0.25396825396825395, R: 0.25396825396825395, micro F1: 0.25396825396825395, macro F1: 0.25396825396825395 [INFO] Type Linking P: 0.9523809523809523, R: 0.9523809523809523, micro F1: 0.9523809523809523, macro F1: 0.9523809523809523 @@ -952,9 +952,9 @@ Question 63: Who was Tom Hanks married to? [INFO] Golden Answer: [http://dbpedia.org/resource/Rita_Wilson] [INFO] Predicted Answer: [] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 64, P: 0.351, R: 0.381, macro F1: 0.348, macro F1*: 0.365 -[INFO] QALD Cumulative metrics, sample: 64, P: 0.554, R: 0.381, macro F1: 0.348, QALD macro F1: 0.451 -[INFO] Current time: 714 ms, average: 3325.45 ms/ques, entity detection: 713.47 ms/ques (linking tool: 623.92 ms/ques), relation detection: 1054.22 ms/ques (similarity: 592.94 ms/ques), EDG generation: 82.70 ms/ques +[INFO] Cumulative metrics, sample: 64, P: 0.351, R: 0.381, macro F1: 0.365 +[INFO] QALD Cumulative metrics, sample: 64, P: 0.554, R: 0.381, average F1: 0.348, QALD macro F1: 0.451 +[INFO] Current question consumed time: 657 ms, average: 3116.91 ms/ques, entity detection: 650.69 ms/ques (linking tool: 569.30 ms/ques), relation detection: 1013.22 ms/ques (similarity: 596.45 ms/ques), EDG generation: 83.03 ms/ques [INFO] Entity Linking P: 0.234375, R: 0.234375, micro F1: 0.234375, macro F1: 0.234375 [INFO] Relation Linking P: 0.265625, R: 0.265625, micro F1: 0.265625, macro F1: 0.265625 [INFO] Type Linking P: 0.953125, R: 0.953125, micro F1: 0.953125, macro F1: 0.953125 @@ -967,9 +967,9 @@ Question 64: Through which countries does the Yenisei river flow? [INFO] Golden Answer: [http://dbpedia.org/resource/Mongolia, http://dbpedia.org/resource/Russia] [INFO] Predicted Answer: [http://dbpedia.org/resource/Mongolia, http://dbpedia.org/resource/Russia] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 65, P: 0.361, R: 0.390, macro F1: 0.358, macro F1*: 0.375 -[INFO] QALD Cumulative metrics, sample: 65, P: 0.561, R: 0.390, macro F1: 0.358, QALD macro F1: 0.460 -[INFO] Current time: 1556 ms, average: 3298.23 ms/ques, entity detection: 704.62 ms/ques (linking tool: 616.09 ms/ques), relation detection: 1046.69 ms/ques (similarity: 592.78 ms/ques), EDG generation: 81.62 ms/ques +[INFO] Cumulative metrics, sample: 65, P: 0.361, R: 0.390, macro F1: 0.375 +[INFO] QALD Cumulative metrics, sample: 65, P: 0.561, R: 0.390, average F1: 0.358, QALD macro F1: 0.460 +[INFO] Current question consumed time: 1453 ms, average: 3091.31 ms/ques, entity detection: 642.46 ms/ques (linking tool: 562.05 ms/ques), relation detection: 1005.86 ms/ques (similarity: 596.25 ms/ques), EDG generation: 81.92 ms/ques [INFO] Entity Linking P: 0.23076923076923078, R: 0.23076923076923078, micro F1: 0.23076923076923078, macro F1: 0.23076923076923078 [INFO] Relation Linking P: 0.26153846153846155, R: 0.26153846153846155, micro F1: 0.26153846153846155, macro F1: 0.26153846153846155 [INFO] Type Linking P: 0.9384615384615385, R: 0.9384615384615385, micro F1: 0.9384615384615385, macro F1: 0.9384615384615385 @@ -982,9 +982,9 @@ Question 65: Give me all cars that are produced in Germany. [INFO] Golden Answer: [http://dbpedia.org/resource/BMW_7_Series_(F01), http://dbpedia.org/resource/Porsche_550, http://dbpedia.org/resource/Porsche_928, http://dbpedia.org/resource/Porsche_944, http://dbpedia.org/resource/Porsche_991, http://dbpedia.org/resource/Porsche_993, http://dbpedia.org/resource/Porsche_996, http://dbpedia.org/resource/Porsche_Boxster/Cayman, http://dbpedia.org/resource/Porsche_Carrera_GT, http://dbpedia.org/resource/Porsche_Cayenne, http://dbpedia.org/resource/Porsche_Macan, http://dbpedia.org/resource/Porsche_Mission_E, http://dbpedia.org/resource/Mercedes-Benz_GLC-Class, http://dbpedia.org/resource/Volkswagen_Amarok, http://dbpedia.org/resource/Volkswagen_Golf_Mk5, http://dbpedia.org/resource/Mercedes-Benz_W128, http://dbpedia.org/resource/Mercedes-Benz_O405, http://dbpedia.org/resource/Mercedes-Benz_770, http://dbpedia.org/resource/Volkswagen_Passat_(B8), http://dbpedia.org/resource/Benz_10/30_PS, http://dbpedia.org/resource/Mercedes_15/70/100_PS, http://dbpedia.org/resource/Mercedes_24/100/140_PS, http://dbpedia.org/resource/Mercedes-Benz_W11, http://dbpedia.org/resource/Audi_A2, http://dbpedia.org/resource/Audi_Front, http://dbpedia.org/resource/Audi_Quattro, http://dbpedia.org/resource/Audi_S4, http://dbpedia.org/resource/Audi_S5, http://dbpedia.org/resource/Audi_S6, http://dbpedia.org/resource/Audi_Type_A, http://dbpedia.org/resource/Audi_Type_B, http://dbpedia.org/resource/Audi_Type_P, http://dbpedia.org/resource/Volkswagen_Golf, http://dbpedia.org/resource/Volkswagen_Golf_Mk4, http://dbpedia.org/resource/Mercedes-Benz_S-Class, http://dbpedia.org/resource/Ford_Scorpio, http://dbpedia.org/resource/BMW_6_Series_(E63), http://dbpedia.org/resource/Mercedes-Benz_SLS_AMG, http://dbpedia.org/resource/Volkswagen_Type_14A_(Hebmüller_Cabriolet), http://dbpedia.org/resource/Mercedes-Benz_W110, http://dbpedia.org/resource/Mercedes-Benz_W113, http://dbpedia.org/resource/BMW_3_Series_(E90), http://dbpedia.org/resource/BMW_5_Series_(E60), http://dbpedia.org/resource/Maybach_Zeppelin, http://dbpedia.org/resource/BMW_3/20, http://dbpedia.org/resource/Opel_Kadett_B, http://dbpedia.org/resource/MAN_NG272(2), http://dbpedia.org/resource/Neoplan_N407, http://dbpedia.org/resource/Ford_Model_T, http://dbpedia.org/resource/Volkswagen_Beetle, http://dbpedia.org/resource/BMW_1_Series_(F20), http://dbpedia.org/resource/BMW_2_Series_(F22), http://dbpedia.org/resource/BMW_328, http://dbpedia.org/resource/BMW_4_Series_(F32), http://dbpedia.org/resource/BMW_5_Series_(F10), http://dbpedia.org/resource/BMW_7_Series_(G11), http://dbpedia.org/resource/BMW_M6, http://dbpedia.org/resource/BMW_i3, http://dbpedia.org/resource/BMW_i8, http://dbpedia.org/resource/Ford_Model_A_(1927–31), http://dbpedia.org/resource/Volkswagen_Crafter, http://dbpedia.org/resource/Fiat_508, http://dbpedia.org/resource/BMW_New_Class, http://dbpedia.org/resource/Volkswagen_Golf_Mk1, http://dbpedia.org/resource/Mercedes-Benz_CL-Class_(C215), http://dbpedia.org/resource/Volkswagen_Passat_(B1), http://dbpedia.org/resource/Mercedes-Benz_W15, http://dbpedia.org/resource/Wartburg_1.3, http://dbpedia.org/resource/Volkswagen_EA489_Basistransporter, http://dbpedia.org/resource/Mercedes-Benz_W03, http://dbpedia.org/resource/Mercedes-Benz_W18, http://dbpedia.org/resource/Audi_Type_K, http://dbpedia.org/resource/Bentley_Continental_Flying_Spur_(2005), http://dbpedia.org/resource/Volkswagen_Phaeton, http://dbpedia.org/resource/Mercedes-AMG_GT, http://dbpedia.org/resource/Cadillac_Catera, http://dbpedia.org/resource/Citroën_Traction_Avant, http://dbpedia.org/resource/Opel_Signum, http://dbpedia.org/resource/BMW_8_Series, http://dbpedia.org/resource/Maybach_57_and_62, http://dbpedia.org/resource/Meillerwagen, http://dbpedia.org/resource/Mercedes-Benz_CL-Class, http://dbpedia.org/resource/Mercedes-Benz_CLK_GTR, http://dbpedia.org/resource/Volkswagen_CC, http://dbpedia.org/resource/Volkswagen_Golf_Mk3, http://dbpedia.org/resource/Volkswagen_Golf_Mk6, http://dbpedia.org/resource/Blitzen_Benz, http://dbpedia.org/resource/Volkswagen_Golf_Mk2, http://dbpedia.org/resource/Mercedes-Benz_W111, http://dbpedia.org/resource/Mercedes-Benz_W180, http://dbpedia.org/resource/Neoplan_Centroliner, http://dbpedia.org/resource/Neoplan_Megaliner, http://dbpedia.org/resource/Neoplan_Skyliner, http://dbpedia.org/resource/BMW_3_Series_(E36), http://dbpedia.org/resource/Mercedes-Benz_S-Class_(W220), http://dbpedia.org/resource/BMW_1_Series_(E87), http://dbpedia.org/resource/Mercedes-Benz_320A, http://dbpedia.org/resource/Mercedes-Benz_380_(1933), http://dbpedia.org/resource/Mercedes-Benz_W08, http://dbpedia.org/resource/Mercedes-Benz_W136, http://dbpedia.org/resource/Mercedes-Benz_W191, http://dbpedia.org/resource/MAN_NL202, http://dbpedia.org/resource/Audi_920, http://dbpedia.org/resource/Mercedes-Benz_SSK, http://dbpedia.org/resource/Messerschmitt_KR175, http://dbpedia.org/resource/Neoplan_Jumbocruiser, http://dbpedia.org/resource/Merkur_Scorpio, http://dbpedia.org/resource/Mercedes-Benz_CLK-Class, http://dbpedia.org/resource/Porsche_911_GT2, http://dbpedia.org/resource/Volkswagen_Polo_G40, http://dbpedia.org/resource/Volkswagen_Polo_Mk4, http://dbpedia.org/resource/BMW_7_Series_(E38), http://dbpedia.org/resource/Mercedes-Benz_E-Class_(W210), http://dbpedia.org/resource/Mercedes-Benz_W142, http://dbpedia.org/resource/Audi_Coupé_(B2), http://dbpedia.org/resource/BMW_303, http://dbpedia.org/resource/Mercedes-Benz_SL-Class_(R231), http://dbpedia.org/resource/Audi_Type_C, http://dbpedia.org/resource/Mercedes-Benz_Atego, http://dbpedia.org/resource/Mercedes-Benz_B-Class, http://dbpedia.org/resource/Mercedes-Benz_CLS-Class, http://dbpedia.org/resource/Mercedes-Benz_GLA-Class, http://dbpedia.org/resource/Mercedes-Benz_GLK-Class, http://dbpedia.org/resource/Mercedes-Benz_S-Class_(W221), http://dbpedia.org/resource/BMW_7_Series_(E65), http://dbpedia.org/resource/Audi_RS_4, http://dbpedia.org/resource/Opel_Patent_Motor_Car, http://dbpedia.org/resource/Audi_Type_M, http://dbpedia.org/resource/Audi_Type_D, http://dbpedia.org/resource/Audi_Type_E, http://dbpedia.org/resource/Audi_Type_G, http://dbpedia.org/resource/Artega_GT, http://dbpedia.org/resource/Mercedes-Benz_500E, http://dbpedia.org/resource/Opel_Kadett_C, http://dbpedia.org/resource/Ford_Focus_Electric, http://dbpedia.org/resource/Mercedes-Benz_W108, http://dbpedia.org/resource/Mercedes-Benz_W112, http://dbpedia.org/resource/Audi_Type_R, http://dbpedia.org/resource/Audi_Type_SS, http://dbpedia.org/resource/Audi_Type_T, http://dbpedia.org/resource/Volkswagen_Passat_(B6), http://dbpedia.org/resource/Volkswagen_Polo_Mk2, http://dbpedia.org/resource/Audi_Le_Mans_quattro, http://dbpedia.org/resource/BMW_2_Series_Active_Tourer, http://dbpedia.org/resource/Mercedes-Benz_W153, http://dbpedia.org/resource/BMW_320, http://dbpedia.org/resource/Mercedes-Benz_W21, http://dbpedia.org/resource/Trabant_601, http://dbpedia.org/resource/Ford_Cougar, http://dbpedia.org/resource/BMW_327, http://dbpedia.org/resource/Mercedes-Benz_W143, http://dbpedia.org/resource/Veritas_(automobile), http://dbpedia.org/resource/CityEl, http://dbpedia.org/resource/BMW_7_Series_(E32), http://dbpedia.org/resource/Mercedes-Benz_E-Class_(W211), http://dbpedia.org/resource/Mercedes-Benz_SL-Class_(R230), http://dbpedia.org/resource/Mercedes-Benz_500K, http://dbpedia.org/resource/E-Wolf_e2, http://dbpedia.org/resource/Volkswagen_Taro, http://dbpedia.org/resource/MAN_Lion's_Chassis, http://dbpedia.org/resource/MAN_NG272, http://dbpedia.org/resource/Mercedes-Benz_W02, http://dbpedia.org/resource/TMG_EV_P001, http://dbpedia.org/resource/Volkswagen_Passat_(B4), http://dbpedia.org/resource/Audi_RS_2_Avant, http://dbpedia.org/resource/Alpina_B10_Bi-Turbo, http://dbpedia.org/resource/Audi_Quattro__Audi_Sport_Quattro_S1_E2__1, http://dbpedia.org/resource/Audi_R8__V10_–_Frontansicht,_13._März_2011,_Wuppert__1, http://dbpedia.org/resource/BMW_M3__E30__1, http://dbpedia.org/resource/BMW_M3__E36__1, http://dbpedia.org/resource/BMW_M3__E46_M3__1, http://dbpedia.org/resource/BMW_M3__E90/92/93__1, http://dbpedia.org/resource/BMW_M3__F80__1, http://dbpedia.org/resource/BMW_M5__BMW_E28_M5__1, http://dbpedia.org/resource/BMW_M5__BMW_E34_M5__1, http://dbpedia.org/resource/BMW_M5__BMW_E39_M5__1, http://dbpedia.org/resource/BMW_M5__BMW_E60_M5__1, http://dbpedia.org/resource/BMW_M5__BMW_F10_M5__1, http://dbpedia.org/resource/BMW_M6__Second_generation_E63/E64__1, http://dbpedia.org/resource/BMW_M6__Third_generation_F12/F13__1, http://dbpedia.org/resource/Ford_C-Max__First_generation__1, http://dbpedia.org/resource/Mercedes-Benz_A-Class__W176__1, http://dbpedia.org/resource/Opel_Senator__A__1, http://dbpedia.org/resource/Opel_Senator__B__1, http://dbpedia.org/resource/BMW_5_Series_(F10)__BMW_F10_M5__1, http://dbpedia.org/resource/Opel_Insignia__Insignia__1, http://dbpedia.org/resource/Audi_A5__8T__1, http://dbpedia.org/resource/Ford_Consul__Granada_Mark_I_based__1, http://dbpedia.org/resource/Mercedes-Benz_CLS-Class__W218__1, http://dbpedia.org/resource/Opel_Ascona__A__1, http://dbpedia.org/resource/Opel_Ascona__B__1, http://dbpedia.org/resource/Opel_Ascona__C__1, http://dbpedia.org/resource/Opel_Omega__A__1, http://dbpedia.org/resource/Opel_Omega__B1__1, http://dbpedia.org/resource/Opel_Omega__B2__1, http://dbpedia.org/resource/Ford_Escort_(Europe)__Fifth_generation__1, http://dbpedia.org/resource/Ford_Escort_(Europe)__First_generation__1, http://dbpedia.org/resource/Ford_Escort_(Europe)__Fourth_generation__1, http://dbpedia.org/resource/Ford_Escort_(Europe)__Second_generation__1, http://dbpedia.org/resource/Ford_Escort_(Europe)__Sixth_generation__1, http://dbpedia.org/resource/Ford_Escort_(Europe)__Third_generation__1, http://dbpedia.org/resource/Callaway_Cars__Callaway_C16__1, http://dbpedia.org/resource/Ford_Scorpio__First_generation__1, http://dbpedia.org/resource/Opel_Kadett__D__1, http://dbpedia.org/resource/Opel_Kadett__E__1, http://dbpedia.org/resource/Opel_Kadett__I__1, http://dbpedia.org/resource/Bentley_Continental_Flying_Spur_(2005)__Flying_Spur__1, http://dbpedia.org/resource/Opel_GT__GT__1, http://dbpedia.org/resource/Porsche_Panamera__First_generation_970__1, http://dbpedia.org/resource/Porsche_Panamera__Second_generation_971__1, http://dbpedia.org/resource/BMW_303__BMW_309__1, http://dbpedia.org/resource/BMW_303__BMW_315/1,_BMW_319/1__1, http://dbpedia.org/resource/Porsche_Panamera, http://dbpedia.org/resource/Audi_A5, http://dbpedia.org/resource/Audi_R8, http://dbpedia.org/resource/Mercedes-Benz_A-Class, http://dbpedia.org/resource/BMW_M3, http://dbpedia.org/resource/BMW_M5, http://dbpedia.org/resource/BMW_Z4_(E89), http://dbpedia.org/resource/Opel_Ascona, http://dbpedia.org/resource/Opel_GT, http://dbpedia.org/resource/Opel_Omega, http://dbpedia.org/resource/Opel_Insignia, http://dbpedia.org/resource/Opel_Kadett, http://dbpedia.org/resource/Opel_Senator, http://dbpedia.org/resource/Ford_Consul, http://dbpedia.org/resource/Ford_Escort_(Europe), http://dbpedia.org/resource/Mercedes-Benz_Bionic, http://dbpedia.org/resource/Mercedes-Benz_SK, http://dbpedia.org/resource/Mercedes-Benz_W10, http://dbpedia.org/resource/Mercedes-Benz_W105, http://dbpedia.org/resource/Mercedes-Benz_GL-Class, http://dbpedia.org/resource/Mercedes-Benz_M-Class, http://dbpedia.org/resource/Mercedes-Benz_W140, http://dbpedia.org/resource/Mercedes-Benz_W123, http://dbpedia.org/resource/Mercedes-Benz_W114, http://dbpedia.org/resource/Mercedes-Maybach_6, http://dbpedia.org/resource/Mercedes-Benz_190_SL, http://dbpedia.org/resource/Mercedes-Benz_Vario, http://dbpedia.org/resource/Mercedes-Benz_540K, http://dbpedia.org/resource/Mercedes-Benz_C-Class_(W203), http://dbpedia.org/resource/Mercedes-Benz_W188, http://dbpedia.org/resource/Mercedes-Benz_G-Class, http://dbpedia.org/resource/Mercedes-Benz_260_D, http://dbpedia.org/resource/Mercedes-Benz_NG, http://dbpedia.org/resource/Mercedes-Benz_W116, http://dbpedia.org/resource/Mercedes-Benz_W189, http://dbpedia.org/resource/Mercedes-Benz_Vito, http://dbpedia.org/resource/Benz_Patent-Motorwagen, http://dbpedia.org/resource/Mercedes-Benz_W126, http://dbpedia.org/resource/Mercedes-Benz_E-Class, http://dbpedia.org/resource/Mercedes-Benz_SL-Class, http://dbpedia.org/resource/Mercedes-Benz_S-Class_(W222), http://dbpedia.org/resource/Mercedes-Benz_300_SL, http://dbpedia.org/resource/Mercedes-Benz_E-Class_(C207), http://dbpedia.org/resource/Mercedes-Benz_CL-Class_(C216), http://dbpedia.org/resource/Mercedes-Benz_W187, http://dbpedia.org/resource/Unimog_435, http://dbpedia.org/resource/Mercedes-Benz_BlueZERO, http://dbpedia.org/resource/Mercedes-Benz_SLK-Class_(R170), http://dbpedia.org/resource/Mercedes-Benz_SL-Class_(R129), http://dbpedia.org/resource/Mercedes-Benz_Actros, http://dbpedia.org/resource/Mercedes-Benz_CLA-Class, http://dbpedia.org/resource/Mercedes-Benz_Econic, http://dbpedia.org/resource/Mercedes-Benz_R-Class, http://dbpedia.org/resource/Mercedes-Benz_R107_and_C107, http://dbpedia.org/resource/Mercedes-Benz_M110_engine, http://dbpedia.org/resource/Mercedes-Benz_TN, http://dbpedia.org/resource/Mercedes-Benz_600, http://dbpedia.org/resource/Mercedes-Benz_W31, http://dbpedia.org/resource/Mercedes-Benz_Citan, http://dbpedia.org/resource/Mercedes-Benz_MB100, http://dbpedia.org/resource/Mercedes-Benz_W124, http://dbpedia.org/resource/Mercedes-Benz_Ponton, http://dbpedia.org/resource/Mercedes-Benz_S-Class_(C217), http://dbpedia.org/resource/Benz_Velo, http://dbpedia.org/resource/Mercedes-Benz_F800, http://dbpedia.org/resource/Mercedes-Benz_Silver_Lightning, http://dbpedia.org/resource/Mercedes-Benz_W201, http://dbpedia.org/resource/Mercedes-Benz_W186, http://dbpedia.org/resource/Mercedes-Benz_F700, http://dbpedia.org/resource/Mercedes-Benz_F125, http://dbpedia.org/resource/Opel_Blitz, http://dbpedia.org/resource/Opel_Diplomat, http://dbpedia.org/resource/Opel_Sintra, http://dbpedia.org/resource/Opel_Trixx, http://dbpedia.org/resource/GM_HydroGen3, http://dbpedia.org/resource/Opel_1.2_litre, http://dbpedia.org/resource/Opel_1.3_litre, http://dbpedia.org/resource/Opel_GTC, http://dbpedia.org/resource/Opel_Vectra, http://dbpedia.org/resource/Opel_(Olympia)_Rekord_P1, http://dbpedia.org/resource/Opel_Rekord_Series_A, http://dbpedia.org/resource/Opel_Commodore, http://dbpedia.org/resource/Opel_4/8_PS, http://dbpedia.org/resource/Opel_Adam, http://dbpedia.org/resource/Opel_Admiral, http://dbpedia.org/resource/Opel_Antara, http://dbpedia.org/resource/Opel_Calibra, http://dbpedia.org/resource/Opel_Combo, http://dbpedia.org/resource/Opel_Kapitän, http://dbpedia.org/resource/Opel_Karl, http://dbpedia.org/resource/Opel_Laubfrosch, http://dbpedia.org/resource/Opel_Manta, http://dbpedia.org/resource/Opel_Meriva, http://dbpedia.org/resource/Opel_Mokka, http://dbpedia.org/resource/Opel_Olympia, http://dbpedia.org/resource/Opel_Olympia_Rekord, http://dbpedia.org/resource/Opel_Rekord_Series_E, http://dbpedia.org/resource/Opel_Zafira, http://dbpedia.org/resource/Opel_Rekord_P2, http://dbpedia.org/resource/Opel_2.0_litre, http://dbpedia.org/resource/Holden_Astra, http://dbpedia.org/resource/Holden_Barina, http://dbpedia.org/resource/Opel_Cascada, http://dbpedia.org/resource/Opel_Rekord, http://dbpedia.org/resource/Opel_Regent, http://dbpedia.org/resource/Opel_Astra, http://dbpedia.org/resource/Opel_Corsa, http://dbpedia.org/resource/Opel_Flextreme, http://dbpedia.org/resource/Opel_Slalom, http://dbpedia.org/resource/Opel_Flextreme_GT/E, http://dbpedia.org/resource/Opel_Rekord_Series_B, http://dbpedia.org/resource/Opel_5/12_PS, http://dbpedia.org/resource/Opel_Agila, http://dbpedia.org/resource/Opel_Kadett_A, http://dbpedia.org/resource/Opel_Monza, http://dbpedia.org/resource/Opel_Tigra, http://dbpedia.org/resource/Opel_1.8_litre, http://dbpedia.org/resource/Opel_10/30_(10/35)_PS, http://dbpedia.org/resource/Opel_K-180, http://dbpedia.org/resource/Opel_Rekord_Series_D, http://dbpedia.org/resource/Opel_Junior, http://dbpedia.org/resource/Opel_Frogster, http://dbpedia.org/resource/Opel_RAK_e, http://dbpedia.org/resource/Opel_Super_6, http://dbpedia.org/resource/Opel_10/40_PS, http://dbpedia.org/resource/Opel_8/40_PS, http://dbpedia.org/resource/Opel_Rekord_Series_C, http://dbpedia.org/resource/Opel_Monza__2013_Monza_Concept__1, http://dbpedia.org/resource/Opel_Rekord__Series_E__1, http://dbpedia.org/resource/Opel_Speedster__Opel_ECO_Speedster__1, http://dbpedia.org/resource/Opel_Insignia__2003_Insignia_Concept__1, http://dbpedia.org/resource/Opel_Rekord_Series_E__1__1, http://dbpedia.org/resource/Opel_Rekord_Series_E__2__1, http://dbpedia.org/resource/Opel_Omega__V8_prototypes__1, http://dbpedia.org/resource/Saturn_Vue__Second_generation__1, http://dbpedia.org/resource/Volkswagen_Santana, http://dbpedia.org/resource/Volkswagen_Sharan, http://dbpedia.org/resource/Volkswagen_LT, http://dbpedia.org/resource/Volkswagen_Scirocco, http://dbpedia.org/resource/Volkswagen_Iltis, http://dbpedia.org/resource/Volkswagen_Passat, http://dbpedia.org/resource/Volkswagen_Polo, http://dbpedia.org/resource/Volkswagen_CrossBlue, http://dbpedia.org/resource/Volkswagen_Up, http://dbpedia.org/resource/Volkswagen_Kommandeurswagen, http://dbpedia.org/resource/Volkswagen_SP2, http://dbpedia.org/resource/Volkswagen_W12, http://dbpedia.org/resource/Volkswagen_Type_2, http://dbpedia.org/resource/Volkswagen_Type_3, http://dbpedia.org/resource/Volkswagen_Plattenwagen, http://dbpedia.org/resource/Volkswagen_Concept_BlueSport, http://dbpedia.org/resource/Volkswagen_1-litre_car, http://dbpedia.org/resource/Volkswagen_181, http://dbpedia.org/resource/Volkswagen_Beetle_(A5), http://dbpedia.org/resource/Volkswagen_Citi_Golf, http://dbpedia.org/resource/Volkswagen_Corrado, http://dbpedia.org/resource/Volkswagen_Derby, http://dbpedia.org/resource/Volkswagen_Eos, http://dbpedia.org/resource/Volkswagen_Fox, http://dbpedia.org/resource/Volkswagen_Gol, http://dbpedia.org/resource/Volkswagen_Jetta, http://dbpedia.org/resource/Volkswagen_K70, http://dbpedia.org/resource/Volkswagen_Karmann_Ghia, http://dbpedia.org/resource/Volkswagen_Kübelwagen, http://dbpedia.org/resource/Volkswagen_New_Beetle, http://dbpedia.org/resource/Volkswagen_Passat_NMS, http://dbpedia.org/resource/Volkswagen_Schwimmwagen, http://dbpedia.org/resource/Volkswagen_Tiguan, http://dbpedia.org/resource/Volkswagen_Touran, http://dbpedia.org/resource/Volkswagen_Type_4, http://dbpedia.org/resource/Volkswagen_Polo_Mk5, http://dbpedia.org/resource/Volkswagen_Type_2_(T3), http://dbpedia.org/resource/Volkswagen_Polo_Mk3, http://dbpedia.org/resource/Volkswagen_Lavida, http://dbpedia.org/resource/Volkswagen_New_Compact_Coupé, http://dbpedia.org/resource/Volkswagen_Transporter_(T4), http://dbpedia.org/resource/Volkswagen_Golf_Mk7, http://dbpedia.org/resource/Volkswagen_Type_147_Kleinlieferwagen, http://dbpedia.org/resource/Volkswagen_Passat_Lingyu, http://dbpedia.org/resource/Volkswagen_Taigun, http://dbpedia.org/resource/Volkswagen_ARVW, http://dbpedia.org/resource/Volkswagen_Passat_(B5), http://dbpedia.org/resource/Volkswagen_Brasília, http://dbpedia.org/resource/Volkswagen_Concept_D, http://dbpedia.org/resource/Volkswagen_Cross_Coupe_GTE_Concept, http://dbpedia.org/resource/Volkswagen_Phideon, http://dbpedia.org/resource/Volkswagen_Concept_A, http://dbpedia.org/resource/Volkswagen_T-Roc, http://dbpedia.org/resource/Volkswagen_Lupo, http://dbpedia.org/resource/Volkswagen_GX3, http://dbpedia.org/resource/Volkswagen_Passat_Lingyu__Volkswagen_Passat_Lingyu__1, http://dbpedia.org/resource/Volkswagen_LT__1st_generation__1, http://dbpedia.org/resource/Volkswagen_Type_2__T1__1, http://dbpedia.org/resource/Ford_P7, http://dbpedia.org/resource/Ford_Taunus_G93A, http://dbpedia.org/resource/Ford_Taunus_P2, http://dbpedia.org/resource/Ford_Taunus_P3, http://dbpedia.org/resource/Ford_Taunus_P4, http://dbpedia.org/resource/Ford_Taunus_P5, http://dbpedia.org/resource/Ford_Taunus_P6, http://dbpedia.org/resource/Ford_Taunus_TC, http://dbpedia.org/resource/Ford_Taunus, http://dbpedia.org/resource/Ford_Eifel, http://dbpedia.org/resource/Ford_Köln, http://dbpedia.org/resource/Ford_Model_Y, http://dbpedia.org/resource/Ford_Taunus_P1, http://dbpedia.org/resource/Heinkel_Kabine, http://dbpedia.org/resource/Isdera_Imperator_108i, http://dbpedia.org/resource/Audi_RS_6, http://dbpedia.org/resource/Jeep_Wrangler, http://dbpedia.org/resource/Chrysler_Cirrus, http://dbpedia.org/resource/Smart_Crosstown, http://dbpedia.org/resource/Mercedes-Benz_C-Class_(W205), http://dbpedia.org/resource/Mercedes-Benz_M273_engine, http://dbpedia.org/resource/Mercedes-Benz_M278_engine, http://dbpedia.org/resource/Mercedes-Benz_Sprinter, http://dbpedia.org/resource/Mercedes-Benz_7G-Tronic_transmission, http://dbpedia.org/resource/Mercedes-Benz_SLK-Class_(R172), http://dbpedia.org/resource/Mercedes-Benz_9G-Tronic_transmission, http://dbpedia.org/resource/Mercedes-Benz_F-Cell, http://dbpedia.org/resource/Dodge_Viper, http://dbpedia.org/resource/Mercedes-Benz_C-Class_(W204), http://dbpedia.org/resource/Mercedes-Benz_E-Class_(W212), http://dbpedia.org/resource/Plymouth_Voyager, http://dbpedia.org/resource/Dodge_Stratus, http://dbpedia.org/resource/Mercedes-Benz_C-Class, http://dbpedia.org/resource/Smart_Fortwo, http://dbpedia.org/resource/Smart_electric_drive, http://dbpedia.org/resource/Mercedes-Benz_E-Class_(W213), http://dbpedia.org/resource/Mercedes-Benz_4G-Tronic_transmission, http://dbpedia.org/resource/Mercedes-Benz_5G-Tronic_transmission, http://dbpedia.org/resource/Chrysler_300__First_generation__1, http://dbpedia.org/resource/Mercedes-Benz_SLK-Class__Mercedes-Benz_R172__1, http://dbpedia.org/resource/Junkers_J_1, http://dbpedia.org/resource/MAN_Lion's_City, http://dbpedia.org/resource/MAN_24.3x0, http://dbpedia.org/resource/Borgward_BX7, http://dbpedia.org/resource/Borgward_B_611, http://dbpedia.org/resource/Borgward_Hansa_1500, http://dbpedia.org/resource/Borgward_Hansa_2400, http://dbpedia.org/resource/Borgward_Isabella, http://dbpedia.org/resource/Borgward_P100, http://dbpedia.org/resource/Goliath_1100, http://dbpedia.org/resource/Goliath_Pioneer, http://dbpedia.org/resource/Lloyd_Arabella, http://dbpedia.org/resource/Lloyd_300, http://dbpedia.org/resource/Lloyd_400, http://dbpedia.org/resource/Lloyd_600, http://dbpedia.org/resource/MAN_SL200, http://dbpedia.org/resource/9ff_GT9, http://dbpedia.org/resource/Goliath_GP700, http://dbpedia.org/resource/MAN_NDxx3F, http://dbpedia.org/resource/MAN_NLxx3F, http://dbpedia.org/resource/MAN_SL202, http://dbpedia.org/resource/Ruf_CTR, http://dbpedia.org/resource/ERuf_Model_A, http://dbpedia.org/resource/Ruf_BTR, http://dbpedia.org/resource/Ruf_CTR2, http://dbpedia.org/resource/Ruf_Rt_12, http://dbpedia.org/resource/Ruf_Turbo_R, http://dbpedia.org/resource/Ruf_RGT, http://dbpedia.org/resource/Ruf_RK_Coupe, http://dbpedia.org/resource/Porsche_968, http://dbpedia.org/resource/Chrysler_Crossfire, http://dbpedia.org/resource/Merkur_XR4Ti, http://dbpedia.org/resource/Volkswagen_Transporter_(T5), http://dbpedia.org/resource/Volkswagen_Caddy, http://dbpedia.org/resource/Volkswagen_LT__2nd_generation__1, http://dbpedia.org/resource/Neoplan_N4009, http://dbpedia.org/resource/Neoplan_Transliner, http://dbpedia.org/resource/Neoplan_N4016, http://dbpedia.org/resource/Brabus_Bullit, http://dbpedia.org/resource/Brabus_C_V8, http://dbpedia.org/resource/Brabus_E_V12, http://dbpedia.org/resource/Brabus_Rocket, http://dbpedia.org/resource/Trabant, http://dbpedia.org/resource/Sachsenring_P240, http://dbpedia.org/resource/TechArt_Magnum, http://dbpedia.org/resource/Chrysler_300, http://dbpedia.org/resource/Mercedes-Benz_SLK-Class, http://dbpedia.org/resource/Volvo_FE] [INFO] Predicted Answer: [http://dbpedia.org/resource/R4M] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 66, P: 0.356, R: 0.384, macro F1: 0.353, macro F1*: 0.369 -[INFO] QALD Cumulative metrics, sample: 66, P: 0.553, R: 0.384, macro F1: 0.353, QALD macro F1: 0.453 -[INFO] Current time: 19622 ms, average: 3545.56 ms/ques, entity detection: 771.00 ms/ques (linking tool: 681.95 ms/ques), relation detection: 1118.05 ms/ques (similarity: 644.55 ms/ques), EDG generation: 80.64 ms/ques +[INFO] Cumulative metrics, sample: 66, P: 0.356, R: 0.384, macro F1: 0.369 +[INFO] QALD Cumulative metrics, sample: 66, P: 0.553, R: 0.384, average F1: 0.353, QALD macro F1: 0.453 +[INFO] Current question consumed time: 18328 ms, average: 3322.17 ms/ques, entity detection: 706.11 ms/ques (linking tool: 625.26 ms/ques), relation detection: 1071.35 ms/ques (similarity: 647.86 ms/ques), EDG generation: 80.92 ms/ques [INFO] Entity Linking P: 0.22727272727272727, R: 0.22727272727272727, micro F1: 0.22727272727272727, macro F1: 0.22727272727272727 [INFO] Relation Linking P: 0.25757575757575757, R: 0.25757575757575757, micro F1: 0.25757575757575757, macro F1: 0.25757575757575757 [INFO] Type Linking P: 0.9393939393939394, R: 0.9393939393939394, micro F1: 0.9393939393939394, macro F1: 0.9393939393939394 @@ -997,9 +997,9 @@ Question 66: When did Michael Jackson die? [INFO] Golden Answer: [2009-6-25] [INFO] Predicted Answer: [2009-6-25, 2009-06-25] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 67, P: 0.365, R: 0.393, macro F1: 0.363, macro F1*: 0.379 -[INFO] QALD Cumulative metrics, sample: 67, P: 0.559, R: 0.393, macro F1: 0.363, QALD macro F1: 0.462 -[INFO] Current time: 2418 ms, average: 3528.73 ms/ques, entity detection: 767.49 ms/ques (linking tool: 678.01 ms/ques), relation detection: 1115.07 ms/ques (similarity: 641.97 ms/ques), EDG generation: 79.52 ms/ques +[INFO] Cumulative metrics, sample: 67, P: 0.365, R: 0.393, macro F1: 0.379 +[INFO] QALD Cumulative metrics, sample: 67, P: 0.559, R: 0.393, average F1: 0.363, QALD macro F1: 0.462 +[INFO] Current question consumed time: 2195 ms, average: 3305.34 ms/ques, entity detection: 702.58 ms/ques (linking tool: 621.60 ms/ques), relation detection: 1068.42 ms/ques (similarity: 645.28 ms/ques), EDG generation: 79.81 ms/ques [INFO] Entity Linking P: 0.22388059701492538, R: 0.22388059701492538, micro F1: 0.22388059701492538, macro F1: 0.22388059701492538 [INFO] Relation Linking P: 0.2537313432835821, R: 0.2537313432835821, micro F1: 0.2537313432835821, macro F1: 0.2537313432835821 [INFO] Type Linking P: 0.9402985074626866, R: 0.9402985074626866, micro F1: 0.9402985074626866, macro F1: 0.9402985074626865 @@ -1012,9 +1012,9 @@ Question 67: What is the highest volcano in Africa? [INFO] Golden Answer: [http://dbpedia.org/resource/Mount_Kilimanjaro] [INFO] Predicted Answer: [http://dbpedia.org/resource/Okavango_Delta, http://dbpedia.org/resource/Kirkwood_Formation, http://dbpedia.org/resource/Kenya_national_rugby_league_team, http://dbpedia.org/resource/Ghost, http://dbpedia.org/resource/Vampire, http://dbpedia.org/resource/Chenini_Formation, http://dbpedia.org/resource/Tendaguru_Formation, http://dbpedia.org/resource/Mare_aux_Songes, http://dbpedia.org/resource/APOPO, http://dbpedia.org/resource/Dhampir, http://dbpedia.org/resource/Trunko, http://dbpedia.org/resource/Dinosaur_Beds, http://dbpedia.org/resource/Koum_Formation, http://dbpedia.org/resource/Farak_Formation, http://dbpedia.org/resource/Werewolf, http://dbpedia.org/resource/Tiouraren_Formation, http://dbpedia.org/resource/Vengeful_ghost, http://dbpedia.org/resource/African_Monetary_Fund, http://dbpedia.org/resource/Enon_Formation, http://dbpedia.org/resource/Football_at_the_Afro-Asian_Games, http://dbpedia.org/resource/Sundays_River_Formation] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 68, P: 0.360, R: 0.388, macro F1: 0.357, macro F1*: 0.373 -[INFO] QALD Cumulative metrics, sample: 68, P: 0.551, R: 0.388, macro F1: 0.357, QALD macro F1: 0.455 -[INFO] Current time: 2798 ms, average: 3517.99 ms/ques, entity detection: 765.79 ms/ques (linking tool: 676.09 ms/ques), relation detection: 1112.88 ms/ques (similarity: 638.82 ms/ques), EDG generation: 78.50 ms/ques +[INFO] Cumulative metrics, sample: 68, P: 0.360, R: 0.388, macro F1: 0.373 +[INFO] QALD Cumulative metrics, sample: 68, P: 0.551, R: 0.388, average F1: 0.357, QALD macro F1: 0.455 +[INFO] Current question consumed time: 2449 ms, average: 3292.75 ms/ques, entity detection: 700.90 ms/ques (linking tool: 619.68 ms/ques), relation detection: 1065.93 ms/ques (similarity: 641.97 ms/ques), EDG generation: 78.79 ms/ques [INFO] Entity Linking P: 0.22058823529411764, R: 0.22058823529411764, micro F1: 0.22058823529411764, macro F1: 0.22058823529411764 [INFO] Relation Linking P: 0.25, R: 0.25, micro F1: 0.25, macro F1: 0.25 [INFO] Type Linking P: 0.9411764705882353, R: 0.9411764705882353, micro F1: 0.9411764705882353, macro F1: 0.9411764705882353 @@ -1027,9 +1027,9 @@ Question 68: Which poet wrote the most books? [INFO] Golden Answer: [http://dbpedia.org/resource/Jules_Verne] [INFO] Predicted Answer: [http://dbpedia.org/resource/Henry_Adamson, http://dbpedia.org/resource/Alan_Halsey, http://dbpedia.org/resource/Andrew_Duncan_(poet), http://dbpedia.org/resource/Arthur_Kelton, http://dbpedia.org/resource/Barry_MacSweeney, http://dbpedia.org/resource/Charles_Best_(poet), http://dbpedia.org/resource/Francis_W._Tancred, http://dbpedia.org/resource/Gordon_Wharton, http://dbpedia.org/resource/Jenny_Joseph, http://dbpedia.org/resource/Kevin_Bailey_(poet), http://dbpedia.org/resource/Maggie_O'Sullivan, http://dbpedia.org/resource/Paul_Summers_(poet), http://dbpedia.org/resource/Peter_Didsbury, http://dbpedia.org/resource/William_Thom_(poet), http://dbpedia.org/resource/Margaret_Holford, http://dbpedia.org/resource/Auden_Group, http://dbpedia.org/resource/Ernest_Rhys, http://dbpedia.org/resource/Arofan, http://dbpedia.org/resource/Bedo_Brwynllys, http://dbpedia.org/resource/Casnodyn, http://dbpedia.org/resource/Cneppyn_Gwerthrynion, http://dbpedia.org/resource/Dafydd_Bach_ap_Madog_Wladaidd, http://dbpedia.org/resource/Dafydd_Benfras, http://dbpedia.org/resource/Dafydd_Gorlech, http://dbpedia.org/resource/Dafydd_Llwyd_ap_Llywelyn_ap_Gruffudd_(Dafydd_Llwyd_o_Fathafarn), http://dbpedia.org/resource/Dafydd_Nanmor, http://dbpedia.org/resource/Deio_ab_Ieuan_Du, http://dbpedia.org/resource/Einion_Offeiriad, http://dbpedia.org/resource/Einion_Wan, http://dbpedia.org/resource/Einion_ap_Gwalchmai, http://dbpedia.org/resource/Elidir_Sais, http://dbpedia.org/resource/Elijah_Fenton, http://dbpedia.org/resource/Evan_Rees_(Dyfed), http://dbpedia.org/resource/Gruffudd_Gryg, http://dbpedia.org/resource/Gruffudd_Hiraethog, http://dbpedia.org/resource/Gruffudd_Llwyd, http://dbpedia.org/resource/Gruffudd_ab_Adda, http://dbpedia.org/resource/Gutun_Owain, http://dbpedia.org/resource/Gwalchmai_ap_Meilyr, http://dbpedia.org/resource/Gwilym_Rhyfel, http://dbpedia.org/resource/Gwilym_Tew, http://dbpedia.org/resource/Gwilym_ab_Ieuan_Hen, http://dbpedia.org/resource/Gwynfardd_Brycheiniog, http://dbpedia.org/resource/Hywel_Foel_ap_Griffri_ap_Pwyll_Wyddel, http://dbpedia.org/resource/Hywel_Swrdwal, http://dbpedia.org/resource/Ieuan_Brydydd_Hir, http://dbpedia.org/resource/Ieuan_Gethin, http://dbpedia.org/resource/Ieuan_ap_Tudur_Penllyn, http://dbpedia.org/resource/Iorwerth_Beli, http://dbpedia.org/resource/Lewys_Morgannwg, http://dbpedia.org/resource/Lewys_Môn, http://dbpedia.org/resource/Llawdden, http://dbpedia.org/resource/Llywarch_Llaety, http://dbpedia.org/resource/Llywarch_ap_Llywelyn, http://dbpedia.org/resource/Llywelyn_Goch_ap_Meurig_Hen, http://dbpedia.org/resource/Llywelyn_Siôn, http://dbpedia.org/resource/Madog_ap_Gwallter, http://dbpedia.org/resource/Maredudd_ap_Rhys, http://dbpedia.org/resource/Meilyr_Brydydd, http://dbpedia.org/resource/Meilyr_ap_Gwalchmai, http://dbpedia.org/resource/Owain_ap_Llywelyn_ab_y_Moel, http://dbpedia.org/resource/Peryf_ap_Cedifor, http://dbpedia.org/resource/Rhisiart_ap_Rhys, http://dbpedia.org/resource/Rhys_Brydydd, http://dbpedia.org/resource/Rhys_Cain, http://dbpedia.org/resource/Rhys_Nanmor, http://dbpedia.org/resource/Rhys_Prichard, http://dbpedia.org/resource/Sefnyn, http://dbpedia.org/resource/Simwnt_Fychan, http://dbpedia.org/resource/Siôn_Ceri, http://dbpedia.org/resource/Siôn_Tudur, http://dbpedia.org/resource/Siôn_ap_Hywel, http://dbpedia.org/resource/Tomos_Prys, http://dbpedia.org/resource/Tudur_ap_Gwyn_Hagr, http://dbpedia.org/resource/Y_Prydydd_Bychan, http://dbpedia.org/resource/William_Smith_(poet), http://dbpedia.org/resource/Murray_Lachlan_Young, http://dbpedia.org/resource/Jane_Williams_(Ysgafell), http://dbpedia.org/resource/Christopher_Pitt, http://dbpedia.org/resource/Mary_Carey,_Lady_Carey, http://dbpedia.org/resource/Mary_Chandler, http://dbpedia.org/resource/William_Bosworth, http://dbpedia.org/resource/Mererid_Hopwood, http://dbpedia.org/resource/Hew_Dalrymple_(advocate), http://dbpedia.org/resource/Joseph_Leftwich, http://dbpedia.org/resource/Angharad_James, http://dbpedia.org/resource/Jon_Stone_(poet), http://dbpedia.org/resource/Sarah_Corbett_(poet), http://dbpedia.org/resource/Debjani_Chatterjee, http://dbpedia.org/resource/Elizabeth_Bath, http://dbpedia.org/resource/Mary_Oxlie, http://dbpedia.org/resource/Richard_Gall, http://dbpedia.org/resource/Tom_Lovatt-Williams, http://dbpedia.org/resource/Morus_Dwyfech, http://dbpedia.org/resource/Hollie_Poetry__UK-poet-stub__1, http://dbpedia.org/resource/Spike_Hawkins, http://dbpedia.org/resource/Frederick_Tennyson, http://dbpedia.org/resource/Owen_Wynne_Jones, http://dbpedia.org/resource/Francis_Meynell, http://dbpedia.org/resource/Arthur_fforde, http://dbpedia.org/resource/Fabian_S._Woodley, http://dbpedia.org/resource/S._E._Cottam, http://dbpedia.org/resource/Duncan_Forbes_(poet), http://dbpedia.org/resource/John_Moat, http://dbpedia.org/resource/John_Pook, http://dbpedia.org/resource/Thomas_Rowland_Hughes, http://dbpedia.org/resource/Ken_Edwards, http://dbpedia.org/resource/Theodore_Maynard, http://dbpedia.org/resource/Gwynn_ap_Gwilym, http://dbpedia.org/resource/Matthew_Stevenson, http://dbpedia.org/resource/John_Leslie_Barford, http://dbpedia.org/resource/Freda_Downie, http://dbpedia.org/resource/T._Glynne_Davies, http://dbpedia.org/resource/Thomas_Francis_Dillon_Croker, http://dbpedia.org/resource/Nathaniel_Jones_(poet), http://dbpedia.org/resource/Francis_Harvey_(poet), http://dbpedia.org/resource/Robert_Davidson_(poet), http://dbpedia.org/resource/Lewys_Dwnn, http://dbpedia.org/resource/John_Fairfax_(poet), http://dbpedia.org/resource/Luke_Kennard, http://dbpedia.org/resource/Andy_Croft, http://dbpedia.org/resource/Robert_Ellis_(Cynddelw), http://dbpedia.org/resource/Hannah_Lowe, http://dbpedia.org/resource/J._Ingo, http://dbpedia.org/resource/Alexander_Hume, http://dbpedia.org/resource/Aneirin_Talfan_Davies, http://dbpedia.org/resource/William_Thomas_(Gwilym_Marles), http://dbpedia.org/resource/Viola_Garvin, http://dbpedia.org/resource/Maureen_Boyle, http://dbpedia.org/resource/Gulielmus_Peregrinus, http://dbpedia.org/resource/Christopher_Meredith, http://dbpedia.org/resource/Ieuan_ap_Hywel_Swrdwal, http://dbpedia.org/resource/Oliver_Reynolds, http://dbpedia.org/resource/Tony_Conran, http://dbpedia.org/resource/Meta_Orred, http://dbpedia.org/resource/Owain_Owain, http://dbpedia.org/resource/Twm_Morys, http://dbpedia.org/resource/Robin_Fulton, http://dbpedia.org/resource/John_Castillo_(poet), http://dbpedia.org/resource/Philip_Owens, http://dbpedia.org/resource/David_Richards_(Dafydd_Ionawr), http://dbpedia.org/resource/Karen_Gershon, http://dbpedia.org/resource/Sarah_Williams, http://dbpedia.org/resource/Sidney_Royse_Lysaght, http://dbpedia.org/resource/Mona_Arshi, http://dbpedia.org/resource/Edward_Hughes_(poet), http://dbpedia.org/resource/Sarah_Piers, http://dbpedia.org/resource/Siôn_Aled_Owen, http://dbpedia.org/resource/Sharon_Morris, http://dbpedia.org/resource/John_Row_(poet), http://dbpedia.org/resource/Phyllis_Hartnoll, http://dbpedia.org/resource/Giles_Andreae, http://dbpedia.org/resource/Iolo_Goch, http://dbpedia.org/resource/Victor_Plarr, http://dbpedia.org/resource/Christopher_Middleton_(poet), http://dbpedia.org/resource/Aonghas_MacNeacail, http://dbpedia.org/resource/Edmund_John, http://dbpedia.org/resource/John_M._Caie, http://dbpedia.org/resource/National_Poet_of_Wales, http://dbpedia.org/resource/Nigel_Heseltine, http://dbpedia.org/resource/Nigel_de_Longchamps, http://dbpedia.org/resource/Denys_Corbet, http://dbpedia.org/resource/Joyce_Mansour, http://dbpedia.org/resource/A._Y._Campbell, http://dbpedia.org/resource/Tony_Connor, http://dbpedia.org/resource/William_Julius_Mickle, http://dbpedia.org/resource/John_Dyfnallt_Owen, http://dbpedia.org/resource/Matthew_Le_Geyt, http://dbpedia.org/resource/Paul_Robinson_(poet), http://dbpedia.org/resource/Dollie_Radford, http://dbpedia.org/resource/J._Griffyth_Fairfax, http://dbpedia.org/resource/Siôn_Bradford, http://dbpedia.org/resource/Donald_Evans_(Welsh_poet), http://dbpedia.org/resource/Eluned_Phillips, http://dbpedia.org/resource/John_Blackwell_(Alun), http://dbpedia.org/resource/John_Thomas_Job, http://dbpedia.org/resource/Jonathan_Hughes_(poet), http://dbpedia.org/resource/Rhys_Cadwaladr, http://dbpedia.org/resource/William_Ambrose_(Emrys), http://dbpedia.org/resource/Dafydd_Ddu_o_Hiraddug, http://dbpedia.org/resource/Roger_Stevens_(poet), http://dbpedia.org/resource/Sam_Riviere, http://dbpedia.org/resource/Maria_Abdy, http://dbpedia.org/resource/Hugh_McMillan_(poet), http://dbpedia.org/resource/Thomas_Noel_(poet), http://dbpedia.org/resource/Thomas_Beach_(poet), http://dbpedia.org/resource/Henry_Scott_Riddell, http://dbpedia.org/resource/Gwerful_Mechain, http://dbpedia.org/resource/David_Davis_(Castellhywel), http://dbpedia.org/resource/Robert_Hetrick, http://dbpedia.org/resource/Violet_Fane, http://dbpedia.org/resource/John_Walters_(priest_and_poet), http://dbpedia.org/resource/Mary_Alcock, http://dbpedia.org/resource/Eric_Thirkell_Cooper, http://dbpedia.org/resource/Barry_Cole, http://dbpedia.org/resource/Thomas_Osbert_Mordaunt, http://dbpedia.org/resource/William_Miller_(poet), http://dbpedia.org/resource/Iorwerth_Peate, http://dbpedia.org/resource/Dorothy_Margaret_Stuart, http://dbpedia.org/resource/Eiluned_Lewis, http://dbpedia.org/resource/Henry_Petowe, http://dbpedia.org/resource/Haldreyn, http://dbpedia.org/resource/Silyn_Roberts, http://dbpedia.org/resource/Morgan_Llwyd, http://dbpedia.org/resource/David_Lewis_(poet), http://dbpedia.org/resource/R._N._Taber, http://dbpedia.org/resource/Isaac_Daniel_Hooson, http://dbpedia.org/resource/William_Hamilton_(Jacobite_poet), http://dbpedia.org/resource/Hilary_Corke, http://dbpedia.org/resource/Thomas_Blackburn_(poet), http://dbpedia.org/resource/Alfred_Williams_(poet), http://dbpedia.org/resource/Antony_Dunn, http://dbpedia.org/resource/Charles_Boyle_(poet), http://dbpedia.org/resource/Dennis_Wilson_(poet), http://dbpedia.org/resource/Dilys_Cadwaladr, http://dbpedia.org/resource/Eleanor_Rees, http://dbpedia.org/resource/Esther_Morgan, http://dbpedia.org/resource/John_Mole_(poet), http://dbpedia.org/resource/John_Tripp_(poet), http://dbpedia.org/resource/Olivia_Cole_(poet), http://dbpedia.org/resource/Paul_Batchelor, http://dbpedia.org/resource/Paul_Henry_(poet), http://dbpedia.org/resource/Sally_Read, http://dbpedia.org/resource/Terence_Tiller, http://dbpedia.org/resource/Thomas_Warner_(poet), http://dbpedia.org/resource/Emma_Tatham, http://dbpedia.org/resource/Thomas_Kibble_Hervey, http://dbpedia.org/resource/Christine_Evans_(poet), http://dbpedia.org/resource/Martha_Llwyd, http://dbpedia.org/resource/Sarah_Fuller_Flower_Adams, http://dbpedia.org/resource/Wilfred_Rowland_Childe, http://dbpedia.org/resource/David_Thomas_(Dewi_Hefin), http://dbpedia.org/resource/John_Hartley_(poet), http://dbpedia.org/resource/James_Logie_Robertson, http://dbpedia.org/resource/Phyllis_King, http://dbpedia.org/resource/Mairtín_Crawford, http://dbpedia.org/resource/Roy_MacGregor-Hastie, http://dbpedia.org/resource/Henry_Sewell_Stokes, http://dbpedia.org/resource/Colin_Mackay_(writer), http://dbpedia.org/resource/Mercer_Simpson, http://dbpedia.org/resource/Paul_Moon_James, http://dbpedia.org/resource/William_Wynn_(poet), http://dbpedia.org/resource/Mordaunt_Currie, http://dbpedia.org/resource/Kenneth_H._Ashley, http://dbpedia.org/resource/Geoffrey_Dearmer, http://dbpedia.org/resource/John_Abbot_(poet), http://dbpedia.org/resource/Harry_Smart, http://dbpedia.org/resource/Margaret_Tait, http://dbpedia.org/resource/Roland_Mathias, http://dbpedia.org/resource/William_Brighty_Rands, http://dbpedia.org/resource/William_Williams_(Crwys), http://dbpedia.org/resource/John_Richard_Williams_(poet), http://dbpedia.org/resource/Muriel_Stuart, http://dbpedia.org/resource/Walter_Lyon_(poet), http://dbpedia.org/resource/Alan_Rook, http://dbpedia.org/resource/Dorian_Cooke, http://dbpedia.org/resource/Charlie_Smith_(Romani_poet), http://dbpedia.org/resource/Gwyn_Morgan_(writer), http://dbpedia.org/resource/Robert_Nicoll, http://dbpedia.org/resource/Christabella_Rogers, http://dbpedia.org/resource/Kate_Rhodes, http://dbpedia.org/resource/M._L._Emmett, http://dbpedia.org/resource/Iris_Tree, http://dbpedia.org/resource/Gerard_Woodward, http://dbpedia.org/resource/Roderick_Watson, http://dbpedia.org/resource/Caroline_Bergvall, http://dbpedia.org/resource/Francis_Berry, http://dbpedia.org/resource/Maggie_Hannan, http://dbpedia.org/resource/Mark_Ford_(poet), http://dbpedia.org/resource/Walter_Harte, http://dbpedia.org/resource/John_Anstey_(poet), http://dbpedia.org/resource/T._Harri_Jones, http://dbpedia.org/resource/John_Cadvan_Davies, http://dbpedia.org/resource/Chris_Torrance, http://dbpedia.org/resource/Jim_Burns_(poet), http://dbpedia.org/resource/Keith_Jebb, http://dbpedia.org/resource/Sophie_Robinson, http://dbpedia.org/resource/Harriet_King_(poet), http://dbpedia.org/resource/Edgar_Phillips, http://dbpedia.org/resource/Geraint_Bowen_(poet), http://dbpedia.org/resource/James_Morrison_(Geordie_songwriter), http://dbpedia.org/resource/W._J._Gruffydd_(Elerydd), http://dbpedia.org/resource/I._D._Ffraid, http://dbpedia.org/resource/Edward_Davies_(Celtic), http://dbpedia.org/resource/John_Penrose_Angold, http://dbpedia.org/resource/Dic_Goodman, http://dbpedia.org/resource/Howard_Sergeant, http://dbpedia.org/resource/Thomas_Shimmin, http://dbpedia.org/resource/Thomas_de_Barry, http://dbpedia.org/resource/Gwerful_Fychan, http://dbpedia.org/resource/Les_Merton, http://dbpedia.org/resource/James_Bandinel_(priest), http://dbpedia.org/resource/Edmund_Elviden, http://dbpedia.org/resource/Eliseus_Williams, http://dbpedia.org/resource/Gwilym_R._Jones, http://dbpedia.org/resource/Thomas_Moss_(minister), http://dbpedia.org/resource/Hywel_ap_Syr_Mathew, http://dbpedia.org/resource/Frances_Leviston, http://dbpedia.org/resource/Peter_Whigham, http://dbpedia.org/resource/Rick_Holland, http://dbpedia.org/resource/Richard_Roderick, http://dbpedia.org/resource/William_Fennor, http://dbpedia.org/resource/Isabella_Fyvie_Mayo, http://dbpedia.org/resource/Matthias_Barr, http://dbpedia.org/resource/David_Ellis_(priest), http://dbpedia.org/resource/Bettina_Jonic__UK-poet-stub__1, http://dbpedia.org/resource/Allen_Fisher__UK-poet-stub__1] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 69, P: 0.355, R: 0.382, macro F1: 0.352, macro F1*: 0.368 -[INFO] QALD Cumulative metrics, sample: 69, P: 0.543, R: 0.382, macro F1: 0.352, QALD macro F1: 0.448 -[INFO] Current time: 394 ms, average: 3472.71 ms/ques, entity detection: 754.71 ms/ques (linking tool: 666.30 ms/ques), relation detection: 1101.32 ms/ques (similarity: 632.33 ms/ques), EDG generation: 77.49 ms/ques +[INFO] Cumulative metrics, sample: 69, P: 0.355, R: 0.382, macro F1: 0.368 +[INFO] QALD Cumulative metrics, sample: 69, P: 0.543, R: 0.382, average F1: 0.352, QALD macro F1: 0.448 +[INFO] Current question consumed time: 391 ms, average: 3250.70 ms/ques, entity detection: 690.74 ms/ques (linking tool: 610.70 ms/ques), relation detection: 1054.80 ms/ques (similarity: 635.46 ms/ques), EDG generation: 78.07 ms/ques [INFO] Entity Linking P: 0.2318840579710145, R: 0.2318840579710145, micro F1: 0.2318840579710145, macro F1: 0.2318840579710145 [INFO] Relation Linking P: 0.2608695652173913, R: 0.2608695652173913, micro F1: 0.2608695652173913, macro F1: 0.2608695652173913 [INFO] Type Linking P: 0.927536231884058, R: 0.927536231884058, micro F1: 0.927536231884058, macro F1: 0.927536231884058 @@ -1042,9 +1042,9 @@ Question 69: Give me all gangsters from the prohibition era. [INFO] Golden Answer: [http://dbpedia.org/resource/Joe_Masseria, http://dbpedia.org/resource/Louis_Buchalter, http://dbpedia.org/resource/Fred_Burke, http://dbpedia.org/resource/Gus_Winkler, http://dbpedia.org/resource/Louis_Campagna, http://dbpedia.org/resource/Al_Capone, http://dbpedia.org/resource/Bugs_Moran, http://dbpedia.org/resource/Dean_O'Banion, http://dbpedia.org/resource/Salvatore_Maranzano, http://dbpedia.org/resource/Ignazio_Lupo, http://dbpedia.org/resource/Mike_Genna, http://dbpedia.org/resource/Bugsy_Siegel, http://dbpedia.org/resource/Lucky_Luciano, http://dbpedia.org/resource/Roger_Touhy, http://dbpedia.org/resource/Ciro_Terranova, http://dbpedia.org/resource/Leo_Vincent_Brothers, http://dbpedia.org/resource/Owney_Madden, http://dbpedia.org/resource/Nicolo_Schiro, http://dbpedia.org/resource/Salvatore_D'Aquila, http://dbpedia.org/resource/Vito_Bonventre, http://dbpedia.org/resource/John_%22Pudgy%22_Dunn, http://dbpedia.org/resource/William_Egan_(gangster)] [INFO] Predicted Answer: [] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 70, P: 0.350, R: 0.377, macro F1: 0.347, macro F1*: 0.363 -[INFO] QALD Cumulative metrics, sample: 70, P: 0.550, R: 0.377, macro F1: 0.347, QALD macro F1: 0.447 -[INFO] Current time: 858 ms, average: 3435.36 ms/ques, entity detection: 745.50 ms/ques (linking tool: 657.54 ms/ques), relation detection: 1087.67 ms/ques (similarity: 624.66 ms/ques), EDG generation: 76.59 ms/ques +[INFO] Cumulative metrics, sample: 70, P: 0.350, R: 0.377, macro F1: 0.363 +[INFO] QALD Cumulative metrics, sample: 70, P: 0.550, R: 0.377, average F1: 0.347, QALD macro F1: 0.447 +[INFO] Current question consumed time: 810 ms, average: 3215.83 ms/ques, entity detection: 682.39 ms/ques (linking tool: 602.69 ms/ques), relation detection: 1041.79 ms/ques (similarity: 627.73 ms/ques), EDG generation: 77.14 ms/ques [INFO] Entity Linking P: 0.24285714285714285, R: 0.24285714285714285, micro F1: 0.24285714285714285, macro F1: 0.24285714285714285 [INFO] Relation Linking P: 0.2714285714285714, R: 0.2714285714285714, micro F1: 0.2714285714285714, macro F1: 0.2714285714285714 [INFO] Type Linking P: 0.9285714285714286, R: 0.9285714285714286, micro F1: 0.9285714285714286, macro F1: 0.9285714285714286 @@ -1057,9 +1057,9 @@ Question 70: Give me all spacecrafts that flew to Mars. [INFO] Golden Answer: [http://dbpedia.org/resource/Spirit_(rover), http://dbpedia.org/resource/Mars_Science_Laboratory, http://dbpedia.org/resource/Sojourner_(rover), http://dbpedia.org/resource/MELOS, http://dbpedia.org/resource/Curiosity_(rover), http://dbpedia.org/resource/Mars_2020, http://dbpedia.org/resource/Opportunity_(rover), http://dbpedia.org/resource/Mars_Pathfinder, http://dbpedia.org/resource/2020_Chinese_Mars_Mission, http://dbpedia.org/resource/ExoMars_(rover), http://dbpedia.org/resource/Mars_3, http://dbpedia.org/resource/Astrobiology_Field_Laboratory, http://dbpedia.org/resource/Mars_2, http://dbpedia.org/resource/Mars_Astrobiology_Explorer-Cacher] [INFO] Predicted Answer: [http://dbpedia.org/resource/White_Martian, http://dbpedia.org/resource/Martian_Manhunter, http://dbpedia.org/resource/Zook_(comics), http://dbpedia.org/resource/Ma'alefa'ak] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 71, P: 0.345, R: 0.371, macro F1: 0.342, macro F1*: 0.357 -[INFO] QALD Cumulative metrics, sample: 71, P: 0.542, R: 0.371, macro F1: 0.342, QALD macro F1: 0.441 -[INFO] Current time: 2743 ms, average: 3425.61 ms/ques, entity detection: 740.99 ms/ques (linking tool: 651.63 ms/ques), relation detection: 1083.55 ms/ques (similarity: 627.24 ms/ques), EDG generation: 75.73 ms/ques +[INFO] Cumulative metrics, sample: 71, P: 0.345, R: 0.371, macro F1: 0.357 +[INFO] QALD Cumulative metrics, sample: 71, P: 0.542, R: 0.371, average F1: 0.342, QALD macro F1: 0.441 +[INFO] Current question consumed time: 2558 ms, average: 3206.56 ms/ques, entity detection: 677.48 ms/ques (linking tool: 597.14 ms/ques), relation detection: 1037.93 ms/ques (similarity: 630.35 ms/ques), EDG generation: 76.27 ms/ques [INFO] Entity Linking P: 0.23943661971830985, R: 0.23943661971830985, micro F1: 0.23943661971830985, macro F1: 0.23943661971830985 [INFO] Relation Linking P: 0.2676056338028169, R: 0.2676056338028169, micro F1: 0.2676056338028169, macro F1: 0.2676056338028169 [INFO] Type Linking P: 0.9295774647887324, R: 0.9295774647887324, micro F1: 0.9295774647887324, macro F1: 0.9295774647887324 @@ -1072,9 +1072,9 @@ Question 71: Show me all Czech movies. [INFO] Golden Answer: [http://dbpedia.org/resource/Rebelové, http://dbpedia.org/resource/The_Illusionist_(2006_film), http://dbpedia.org/resource/The_Last_Train_(2006_film), http://dbpedia.org/resource/Hard_to_Be_a_God_(2013_film), http://dbpedia.org/resource/Witnesses_(film_series), http://dbpedia.org/resource/The_Red_Baron_(2008_film), http://dbpedia.org/resource/The_Magical_Duvet, http://dbpedia.org/resource/Pusinky, http://dbpedia.org/resource/Solomon_Kane_(film), http://dbpedia.org/resource/The_Ninth_Day, http://dbpedia.org/resource/The_Visitors:_Bastille_Day, http://dbpedia.org/resource/All_My_Loved_Ones, http://dbpedia.org/resource/The_Snake_Brothers, http://dbpedia.org/resource/Divided_We_Fall_(film), http://dbpedia.org/resource/Želary, http://dbpedia.org/resource/Body_Without_Soul, http://dbpedia.org/resource/Janosik._Prawdziwa_historia, http://dbpedia.org/resource/Shoes_(2012_film), http://dbpedia.org/resource/Ro(c)k_podvraťáků, http://dbpedia.org/resource/ROMing, http://dbpedia.org/resource/Babicka_(2003_film), http://dbpedia.org/resource/The_Brothers_Grimm_(film), http://dbpedia.org/resource/Life_and_Extraordinary_Adventures_of_Private_Ivan_Chonkin, http://dbpedia.org/resource/The_Cycle_(2009_film), http://dbpedia.org/resource/Goat_Story, http://dbpedia.org/resource/Goat_Story_2, http://dbpedia.org/resource/Choking_Hazard, http://dbpedia.org/resource/Something_Like_Happiness, http://dbpedia.org/resource/Skřítek, http://dbpedia.org/resource/Oliver_Twist_(2005_film), http://dbpedia.org/resource/Bad_Company_(2002_film), http://dbpedia.org/resource/Normal_(2009_film), http://dbpedia.org/resource/The_Shadow_over_Prague, http://dbpedia.org/resource/Year_of_the_Devil, http://dbpedia.org/resource/Kruté_radosti, http://dbpedia.org/resource/In_the_Mirror_of_Maya_Deren, http://dbpedia.org/resource/One_Night_in_One_City, http://dbpedia.org/resource/Tobruk_(2008_film), http://dbpedia.org/resource/Cosy_Dens, http://dbpedia.org/resource/Little_Otik, http://dbpedia.org/resource/Václav_(film), http://dbpedia.org/resource/Bach's_Fight_for_Freedom, http://dbpedia.org/resource/Lea_(film), http://dbpedia.org/resource/Kooky, http://dbpedia.org/resource/Stolen_Kosovo, http://dbpedia.org/resource/The_Pagan_Queen, http://dbpedia.org/resource/Not_Angels_But_Angels, http://dbpedia.org/resource/Golet_v_údolí] [INFO] Predicted Answer: [] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 72, P: 0.340, R: 0.366, macro F1: 0.338, macro F1*: 0.352 -[INFO] QALD Cumulative metrics, sample: 72, P: 0.548, R: 0.366, macro F1: 0.338, QALD macro F1: 0.439 -[INFO] Current time: 497 ms, average: 3384.93 ms/ques, entity detection: 731.71 ms/ques (linking tool: 643.08 ms/ques), relation detection: 1069.24 ms/ques (similarity: 619.00 ms/ques), EDG generation: 74.78 ms/ques +[INFO] Cumulative metrics, sample: 72, P: 0.340, R: 0.366, macro F1: 0.352 +[INFO] QALD Cumulative metrics, sample: 72, P: 0.548, R: 0.366, average F1: 0.338, QALD macro F1: 0.439 +[INFO] Current question consumed time: 473 ms, average: 3168.61 ms/ques, entity detection: 669.00 ms/ques (linking tool: 589.31 ms/ques), relation detection: 1024.31 ms/ques (similarity: 622.10 ms/ques), EDG generation: 75.31 ms/ques [INFO] Entity Linking P: 0.25, R: 0.25, micro F1: 0.25, macro F1: 0.25 [INFO] Relation Linking P: 0.2777777777777778, R: 0.2777777777777778, micro F1: 0.2777777777777778, macro F1: 0.2777777777777778 [INFO] Type Linking P: 0.9305555555555556, R: 0.9305555555555556, micro F1: 0.9305555555555556, macro F1: 0.9305555555555556 @@ -1087,9 +1087,9 @@ Question 72: Give me all taikonauts. [INFO] Golden Answer: [http://dbpedia.org/resource/Zhai_Zhigang, http://dbpedia.org/resource/Fei_Junlong, http://dbpedia.org/resource/Jing_Haipeng, http://dbpedia.org/resource/Zhang_Xiaoguang, http://dbpedia.org/resource/Nie_Haisheng, http://dbpedia.org/resource/Chen_Dong_(astronaut), http://dbpedia.org/resource/Tao_Jiali] [INFO] Predicted Answer: [] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 73, P: 0.335, R: 0.361, macro F1: 0.333, macro F1*: 0.348 -[INFO] QALD Cumulative metrics, sample: 73, P: 0.554, R: 0.361, macro F1: 0.333, QALD macro F1: 0.437 -[INFO] Current time: 633 ms, average: 3347.23 ms/ques, entity detection: 722.32 ms/ques (linking tool: 634.79 ms/ques), relation detection: 1056.85 ms/ques (similarity: 611.63 ms/ques), EDG generation: 73.85 ms/ques +[INFO] Cumulative metrics, sample: 73, P: 0.335, R: 0.361, macro F1: 0.348 +[INFO] QALD Cumulative metrics, sample: 73, P: 0.554, R: 0.361, average F1: 0.333, QALD macro F1: 0.437 +[INFO] Current question consumed time: 594 ms, average: 3133.34 ms/ques, entity detection: 660.41 ms/ques (linking tool: 581.71 ms/ques), relation detection: 1012.44 ms/ques (similarity: 614.71 ms/ques), EDG generation: 74.36 ms/ques [INFO] Entity Linking P: 0.2602739726027397, R: 0.2602739726027397, micro F1: 0.2602739726027397, macro F1: 0.2602739726027397 [INFO] Relation Linking P: 0.2876712328767123, R: 0.2876712328767123, micro F1: 0.2876712328767123, macro F1: 0.2876712328767123 [INFO] Type Linking P: 0.9315068493150684, R: 0.9315068493150684, micro F1: 0.9315068493150684, macro F1: 0.9315068493150684 @@ -1102,9 +1102,9 @@ Question 73: Which countries have more than ten volcanoes? [INFO] Golden Answer: [http://dbpedia.org/resource/Guatemala, http://dbpedia.org/resource/Indonesia, http://dbpedia.org/resource/Sumatra, http://dbpedia.org/resource/El_Salvador, http://dbpedia.org/resource/United_States, http://dbpedia.org/resource/Ecuador, http://dbpedia.org/resource/Hokkaido, http://dbpedia.org/resource/Ethiopia, http://dbpedia.org/resource/Papua_New_Guinea, http://dbpedia.org/resource/Russia, http://dbpedia.org/resource/Kamchatka_Peninsula, http://dbpedia.org/resource/British_Columbia, http://dbpedia.org/resource/Colombia, http://dbpedia.org/resource/Argentina, http://dbpedia.org/resource/Chile, http://dbpedia.org/resource/Java, http://dbpedia.org/resource/Bolivia, http://dbpedia.org/resource/Japan] [INFO] Predicted Answer: [http://dbpedia.org/resource/Abadan, http://dbpedia.org/resource/American_literature, http://dbpedia.org/resource/American_people, http://dbpedia.org/resource/Ancient_Egyptian, http://dbpedia.org/resource/Arab_states_of_the_Persian_Gulf, http://dbpedia.org/resource/Arkhangelsk_Oblast, http://dbpedia.org/resource/Arzawa, http://dbpedia.org/resource/Australia_cricket_team, http://dbpedia.org/resource/Australia_women's_national_basketball_team, http://dbpedia.org/resource/Australian_cricket_team, http://dbpedia.org/resource/BDSM, http://dbpedia.org/resource/Bahá'í_Faith_in_Iran, http://dbpedia.org/resource/Baku_Khanate, http://dbpedia.org/resource/Balinese_Hinduism, http://dbpedia.org/resource/Baltimore-Washington_Metropolitan_Area, http://dbpedia.org/resource/Barbados_cricket_team, http://dbpedia.org/resource/Batavian_Republic, http://dbpedia.org/resource/Beylik_of_Teke, http://dbpedia.org/resource/Bishopric_of_Lebus, http://dbpedia.org/resource/Breton_people, http://dbpedia.org/resource/British_Boxing_Board_of_Control, http://dbpedia.org/resource/British_Citizen, http://dbpedia.org/resource/British_Malaya, http://dbpedia.org/resource/British_Pakistani, http://dbpedia.org/resource/British_nationality, http://dbpedia.org/resource/Britishness, http://dbpedia.org/resource/Bronze_Age_India, http://dbpedia.org/resource/Brugge, http://dbpedia.org/resource/Canton_Ticino, http://dbpedia.org/resource/Caroline_Islands, http://dbpedia.org/resource/Carpathian_Ruthenia, http://dbpedia.org/resource/Celtic_Neopaganism, http://dbpedia.org/resource/Cherson_(theme), http://dbpedia.org/resource/Cinema_of_China, http://dbpedia.org/resource/Cinema_of_India, http://dbpedia.org/resource/Coalition_Provisional_Authority, http://dbpedia.org/resource/Colony_of_Natal, http://dbpedia.org/resource/County_of_Artois, http://dbpedia.org/resource/County_of_Barcelona, http://dbpedia.org/resource/County_of_Burgundy, http://dbpedia.org/resource/Crown_of_Bohemia, http://dbpedia.org/resource/Czech_Socialist_Republic, http://dbpedia.org/resource/Dakota_Territory, http://dbpedia.org/resource/Dardani, http://dbpedia.org/resource/Demographics_of_India, http://dbpedia.org/resource/Denmark-Norway, http://dbpedia.org/resource/Dissolution_of_Czechoslovakia, http://dbpedia.org/resource/Duchy_of_Aquitaine, http://dbpedia.org/resource/Duchy_of_Burgundy, http://dbpedia.org/resource/Duchy_of_Gascony, http://dbpedia.org/resource/Duchy_of_Nassau, http://dbpedia.org/resource/Duchy_of_Savoy, http://dbpedia.org/resource/Dutch_cuisine, http://dbpedia.org/resource/EC_postcode_area, http://dbpedia.org/resource/Early_modern_France, http://dbpedia.org/resource/East_German, http://dbpedia.org/resource/East_Roman_Empire, http://dbpedia.org/resource/East_Texas, http://dbpedia.org/resource/Egyptian_people, http://dbpedia.org/resource/Emirate_of_Afghanistan, http://dbpedia.org/resource/English_cricket_team, http://dbpedia.org/resource/European_Russia, http://dbpedia.org/resource/Extraterritoriality, http://dbpedia.org/resource/FIBA, http://dbpedia.org/resource/Federal_Territory_(Malaysia), http://dbpedia.org/resource/First_Babylonian_Dynasty, http://dbpedia.org/resource/First_Intermediate_Period_of_Egypt, http://dbpedia.org/resource/Free_City_of_Lübeck, http://dbpedia.org/resource/Free_State_Province, http://dbpedia.org/resource/French-Canadian, http://dbpedia.org/resource/French_Fifth_Republic, http://dbpedia.org/resource/French_immigration_to_Mexico, http://dbpedia.org/resource/French_nationality_law, http://dbpedia.org/resource/Fukuoka, http://dbpedia.org/resource/Fédération_Internationale_de_Volleyball, http://dbpedia.org/resource/Geats, http://dbpedia.org/resource/General_Roman_Calendar, http://dbpedia.org/resource/Gloucestershire, http://dbpedia.org/resource/Gojoseon, http://dbpedia.org/resource/Governorate_of_Livonia, http://dbpedia.org/resource/Heligoland_Bight, http://dbpedia.org/resource/Hesse-Homburg, http://dbpedia.org/resource/History_of_Germany, http://dbpedia.org/resource/International_Wheelchair_Basketball_Federation, http://dbpedia.org/resource/Italy_national_basketball_team, http://dbpedia.org/resource/Jains, http://dbpedia.org/resource/Jund_Filastin, http://dbpedia.org/resource/Kalimantan, http://dbpedia.org/resource/Karamanids, http://dbpedia.org/resource/Kerma_Culture, http://dbpedia.org/resource/Khanate_of_Kalat, http://dbpedia.org/resource/Khazars, http://dbpedia.org/resource/Kingdom_of_Pamplona, http://dbpedia.org/resource/Kingdom_of_the_Burgundians, http://dbpedia.org/resource/Kirghiz_Soviet_Socialist_Republic, http://dbpedia.org/resource/Korean_Peninsula, http://dbpedia.org/resource/Korean_cuisine, http://dbpedia.org/resource/Korean_people, http://dbpedia.org/resource/La_Belle_Alliance, http://dbpedia.org/resource/Liang_dynasty, http://dbpedia.org/resource/List_of_Anglican_Church_calendars, http://dbpedia.org/resource/List_of_serial_killers_by_country, http://dbpedia.org/resource/Lithuania_men's_national_basketball_team, http://dbpedia.org/resource/Lithuanian_Jews, http://dbpedia.org/resource/Liu_Song_dynasty, http://dbpedia.org/resource/Madrid,_Spain, http://dbpedia.org/resource/Mamluk_dynasty_of_Iraq, http://dbpedia.org/resource/Manx_people, http://dbpedia.org/resource/Marlborough_House, http://dbpedia.org/resource/Marsica, http://dbpedia.org/resource/Member_states_of_the_Commonwealth_of_Nations, http://dbpedia.org/resource/Mexican_cuisine, http://dbpedia.org/resource/Moche_culture, http://dbpedia.org/resource/Moghulistan, http://dbpedia.org/resource/Moscow_Kremlin, http://dbpedia.org/resource/Mount_Emei, http://dbpedia.org/resource/Multiple_citizenship, http://dbpedia.org/resource/Nanyue, http://dbpedia.org/resource/Negev, http://dbpedia.org/resource/Netherlands_(terminology), http://dbpedia.org/resource/Netherlands_East_Indies, http://dbpedia.org/resource/New_Britain, http://dbpedia.org/resource/North_West_Province_(South_Africa), http://dbpedia.org/resource/Northamptonshire, http://dbpedia.org/resource/Northeast_China, http://dbpedia.org/resource/Northern_Basque_Country, http://dbpedia.org/resource/Northern_Greece, http://dbpedia.org/resource/Northern_Wei, http://dbpedia.org/resource/Northwestern_United_States, http://dbpedia.org/resource/Nueva_Vizcaya,_New_Spain, http://dbpedia.org/resource/Nuevo_Santander, http://dbpedia.org/resource/Orange_Free_State, http://dbpedia.org/resource/Pakistani_cricket_team, http://dbpedia.org/resource/Palestinian_Christians, http://dbpedia.org/resource/Peerage_of_the_United_Kingdom, http://dbpedia.org/resource/Prince-Bishopric_of_Augsburg, http://dbpedia.org/resource/Principality_of_Achaea, http://dbpedia.org/resource/Province_of_Massachusetts_Bay, http://dbpedia.org/resource/Province_of_Pennsylvania, http://dbpedia.org/resource/Province_of_Quebec_(1763–91), http://dbpedia.org/resource/Pyrénées_Atlantiques, http://dbpedia.org/resource/Republic_of_Pirates, http://dbpedia.org/resource/Roman_Judea, http://dbpedia.org/resource/Russian-American, http://dbpedia.org/resource/Russian_Turkestan, http://dbpedia.org/resource/Ruthenians, http://dbpedia.org/resource/Santa_Maria_del_Mar,_Barcelona, http://dbpedia.org/resource/Scordisci, http://dbpedia.org/resource/Second_Czechoslovak_Republic, http://dbpedia.org/resource/Semirechye_Oblast, http://dbpedia.org/resource/Sened_(town), http://dbpedia.org/resource/Shi'a_Islam, http://dbpedia.org/resource/South_African, http://dbpedia.org/resource/South_African_cricket_team, http://dbpedia.org/resource/South_African_cuisine, http://dbpedia.org/resource/Southern_Qi, http://dbpedia.org/resource/Southern_Song, http://dbpedia.org/resource/Spaniard, http://dbpedia.org/resource/Spanish_State, http://dbpedia.org/resource/Sri_Lankan_cricket_team, http://dbpedia.org/resource/State_of_New_York, http://dbpedia.org/resource/Sudetenland, http://dbpedia.org/resource/Sultanate_of_Sambas, http://dbpedia.org/resource/Taiwanese_cuisine, http://dbpedia.org/resource/Thanjavur_Maratha_kingdom, http://dbpedia.org/resource/Tribe, http://dbpedia.org/resource/Trinidadian, http://dbpedia.org/resource/Upper_Hungary, http://dbpedia.org/resource/Upper_Lusatia, http://dbpedia.org/resource/Upper_Mustang, http://dbpedia.org/resource/Upstate_New_York, http://dbpedia.org/resource/Vallespir, http://dbpedia.org/resource/Varangian, http://dbpedia.org/resource/Victory_in_Europe_Day, http://dbpedia.org/resource/Vilna, http://dbpedia.org/resource/Wageningen, http://dbpedia.org/resource/Warsaw,_Poland, http://dbpedia.org/resource/Washington_state, http://dbpedia.org/resource/Weimar_Germany, http://dbpedia.org/resource/West_Bank_Areas_in_the_Oslo_II_Accord, http://dbpedia.org/resource/West_Kalimantan, http://dbpedia.org/resource/Yakutia, http://dbpedia.org/resource/Yamhad, http://dbpedia.org/resource/Yotvingians, http://dbpedia.org/resource/Adam's_Grave, http://dbpedia.org/resource/Aden_Protectorate, http://dbpedia.org/resource/America, http://dbpedia.org/resource/Anglo-Indian_cuisine, http://dbpedia.org/resource/Anglosphere, http://dbpedia.org/resource/Annam_(French_protectorate), http://dbpedia.org/resource/Annobón, http://dbpedia.org/resource/Archbishopric_of_Riga, http://dbpedia.org/resource/Aztec_Empire, http://dbpedia.org/resource/Baghdad_Vilayet, http://dbpedia.org/resource/Baltic_Germans, http://dbpedia.org/resource/Baltic_provinces, http://dbpedia.org/resource/Barghawata, http://dbpedia.org/resource/Basra_Vilayet, http://dbpedia.org/resource/Beer_in_Canada, http://dbpedia.org/resource/Brandenburger_Gold_Coast, http://dbpedia.org/resource/British_Hong_Kong, http://dbpedia.org/resource/British_Solomon_Islands, http://dbpedia.org/resource/British_Somaliland, http://dbpedia.org/resource/British_Western_Pacific_Territories, http://dbpedia.org/resource/British_Windward_Islands, http://dbpedia.org/resource/Buryatia, http://dbpedia.org/resource/Buyeo_Kingdom, http://dbpedia.org/resource/Canadian_cricket_team, http://dbpedia.org/resource/Canadian_cuisine, http://dbpedia.org/resource/Cap_Bon, http://dbpedia.org/resource/Cape_Bon, http://dbpedia.org/resource/Catalan_Republic, http://dbpedia.org/resource/Chaldia, http://dbpedia.org/resource/Chicago_Metropolitan_Area, http://dbpedia.org/resource/Chuvashia, http://dbpedia.org/resource/Cimmeria_(Conan), http://dbpedia.org/resource/Citizenship_of_the_United_States, http://dbpedia.org/resource/Colombian_cuisine, http://dbpedia.org/resource/Colony_of_New_Zealand, http://dbpedia.org/resource/Confederate_government_of_Missouri, http://dbpedia.org/resource/Constitution_of_the_United_States_of_America, http://dbpedia.org/resource/Cook_Islanders, http://dbpedia.org/resource/Crimean_Regional_Government, http://dbpedia.org/resource/Crow_Nation, http://dbpedia.org/resource/Cuisine_of_Jamaica, http://dbpedia.org/resource/Cuisine_of_Trinidad_and_Tobago, http://dbpedia.org/resource/Curacao, http://dbpedia.org/resource/Danish_Gold_Coast, http://dbpedia.org/resource/Danish_India, http://dbpedia.org/resource/Decline_of_the_Ottoman_Empire, http://dbpedia.org/resource/Delta_state, http://dbpedia.org/resource/Democratic_Republic_of_Georgia, http://dbpedia.org/resource/Demographics_of_Morocco, http://dbpedia.org/resource/Dervish_state, http://dbpedia.org/resource/Diocese_of_Macedonia, http://dbpedia.org/resource/Division_of_Northern_Territory, http://dbpedia.org/resource/Don_Republic, http://dbpedia.org/resource/Drimnagh, http://dbpedia.org/resource/Duchy_of_Courland_and_Semigallia, http://dbpedia.org/resource/Duchy_of_Courland_and_Semigallia_(1918), http://dbpedia.org/resource/Duchy_of_Greater_Poland, http://dbpedia.org/resource/Duchy_of_Masovia, http://dbpedia.org/resource/Duchy_of_Neopatras, http://dbpedia.org/resource/Duchy_of_Pomerania, http://dbpedia.org/resource/Dutch_(ethnic_group), http://dbpedia.org/resource/Dutch_Gold_Coast, http://dbpedia.org/resource/Dutch_Language_Union, http://dbpedia.org/resource/Dutch_New_Guinea, http://dbpedia.org/resource/E_postcode_area, http://dbpedia.org/resource/East_African_cricket_team, http://dbpedia.org/resource/Eastern_Hungarian_Kingdom, http://dbpedia.org/resource/Eastern_Iranian_languages, http://dbpedia.org/resource/Emirate_of_Jabal_Shammar, http://dbpedia.org/resource/England_national_cricket_team, http://dbpedia.org/resource/English_Council_of_State, http://dbpedia.org/resource/Espiritu_Santo, http://dbpedia.org/resource/Falkland_Islands_Dependencies, http://dbpedia.org/resource/Fengjie, http://dbpedia.org/resource/Florida_in_the_American_Civil_War, http://dbpedia.org/resource/Free_Imperial_City_of_Besançon, http://dbpedia.org/resource/French_Cochinchina, http://dbpedia.org/resource/French_Morocco, http://dbpedia.org/resource/French_Protectorate_of_Laos, http://dbpedia.org/resource/French_protectorate_of_Morocco, http://dbpedia.org/resource/French_protectorate_of_Tunisia, http://dbpedia.org/resource/Gaya_confederacy, http://dbpedia.org/resource/German_American, http://dbpedia.org/resource/German_East_Africa, http://dbpedia.org/resource/German_South-West_Africa, http://dbpedia.org/resource/Gilbert_and_Ellice_Islands, http://dbpedia.org/resource/Giudicato_of_Cagliari, http://dbpedia.org/resource/Global_citizenship, http://dbpedia.org/resource/Grand_Duchy_of_Finland, http://dbpedia.org/resource/Great_Lakes_region_(North_America), http://dbpedia.org/resource/Guangzhouwan, http://dbpedia.org/resource/Hejaz_Vilayet, http://dbpedia.org/resource/History_of_British_nationality_law, http://dbpedia.org/resource/Ifni, http://dbpedia.org/resource/Imperial_General_Headquarters, http://dbpedia.org/resource/India_national_cricket_team, http://dbpedia.org/resource/Indian_film, http://dbpedia.org/resource/Indian_national_cricket_team, http://dbpedia.org/resource/Indonesian_cuisine, http://dbpedia.org/resource/International_Association_of_the_Congo, http://dbpedia.org/resource/Intersex, http://dbpedia.org/resource/Ireland_cricket_team, http://dbpedia.org/resource/Irish_citizen, http://dbpedia.org/resource/Israeli_cuisine, http://dbpedia.org/resource/Italian_Libya, http://dbpedia.org/resource/Italian_Somaliland, http://dbpedia.org/resource/Javanese_cuisine, http://dbpedia.org/resource/Jin_(Chinese_state), http://dbpedia.org/resource/Jordanian_cuisine, http://dbpedia.org/resource/Joseon_Dynasty, http://dbpedia.org/resource/Judean_Hills, http://dbpedia.org/resource/Kabardino-Balkaria, http://dbpedia.org/resource/Kenya_national_cricket_team, http://dbpedia.org/resource/Kham, http://dbpedia.org/resource/Khanate_of_Khiva, http://dbpedia.org/resource/Kiautschou_Bay_concession, http://dbpedia.org/resource/Kimberley,_South_Africa, http://dbpedia.org/resource/King_of_Morocco, http://dbpedia.org/resource/Kingdom_of_Finland_(1918), http://dbpedia.org/resource/Kingdom_of_Lithuania, http://dbpedia.org/resource/Kingdom_of_Lithuania_(1918), http://dbpedia.org/resource/Kingdom_of_Rarotonga, http://dbpedia.org/resource/Komi_Republic, http://dbpedia.org/resource/Korean_peninsula, http://dbpedia.org/resource/Kovno_Governorate, http://dbpedia.org/resource/Kuban_People's_Republic, http://dbpedia.org/resource/Kunashir_Island, http://dbpedia.org/resource/Kwantung_Leased_Territory, http://dbpedia.org/resource/Later_Yan, http://dbpedia.org/resource/Leopardstown, http://dbpedia.org/resource/List_of_Canadians, http://dbpedia.org/resource/Louisiana_in_the_American_Civil_War, http://dbpedia.org/resource/Malta_Colony, http://dbpedia.org/resource/Manx_cat, http://dbpedia.org/resource/Mari_El, http://dbpedia.org/resource/Mariana_Islands, http://dbpedia.org/resource/Mauretania_Caesariensis, http://dbpedia.org/resource/Member_states_of_the_European_Union, http://dbpedia.org/resource/Military_history_of_the_United_States_during_World_War_II, http://dbpedia.org/resource/Minyue, http://dbpedia.org/resource/Mississippi_in_the_American_Civil_War, http://dbpedia.org/resource/Mongolian_cuisine, http://dbpedia.org/resource/Mount_Lebanon_Mutasarrifate, http://dbpedia.org/resource/Mutasarrifate_of_Jerusalem, http://dbpedia.org/resource/Nanking, http://dbpedia.org/resource/Nassau_(Cook_Islands), http://dbpedia.org/resource/Natal,_South_Africa, http://dbpedia.org/resource/Nationality, http://dbpedia.org/resource/Neopagan, http://dbpedia.org/resource/Neverland, http://dbpedia.org/resource/New_Westminster,_British_Columbia, http://dbpedia.org/resource/New_York_City_Fire_Department, http://dbpedia.org/resource/New_Zealand_national_cricket_team, http://dbpedia.org/resource/New_Zealand_nationality_law, http://dbpedia.org/resource/Non-citizens_(Latvia), http://dbpedia.org/resource/Northern_Rhodesia, http://dbpedia.org/resource/Nyasaland, http://dbpedia.org/resource/Odawa_people, http://dbpedia.org/resource/Pakistan_national_cricket_team, http://dbpedia.org/resource/Palestinian_cuisine, http://dbpedia.org/resource/Panama_Canal_Zone, http://dbpedia.org/resource/Pannonia_Inferior, http://dbpedia.org/resource/Penrhyn_Island, http://dbpedia.org/resource/Peruvian, http://dbpedia.org/resource/Philippine_nationality_law, http://dbpedia.org/resource/Police_area, http://dbpedia.org/resource/Portuguese-speaking_African_countries, http://dbpedia.org/resource/Portuguese_Cape_Verde, http://dbpedia.org/resource/Portuguese_Gold_Coast, http://dbpedia.org/resource/Portuguese_São_Tomé_and_Príncipe, http://dbpedia.org/resource/Praeneste, http://dbpedia.org/resource/Prince-Bishopric_of_Warmia, http://dbpedia.org/resource/Principality_of_Transylvania_(1570–1711), http://dbpedia.org/resource/Province_of_Silesia, http://dbpedia.org/resource/Republic_of_Communist_China, http://dbpedia.org/resource/Republic_of_South_Africa, http://dbpedia.org/resource/Roman_Syria, http://dbpedia.org/resource/SE_postcode_area, http://dbpedia.org/resource/SW_postcode_area, http://dbpedia.org/resource/Sandomierz_Voivodeship, http://dbpedia.org/resource/Saybrook_Colony, http://dbpedia.org/resource/Second_Hellenic_Republic, http://dbpedia.org/resource/Shanghai_French_Concession, http://dbpedia.org/resource/Shanghai_International_Settlement, http://dbpedia.org/resource/Sichuan_Province, http://dbpedia.org/resource/Somali_Republic, http://dbpedia.org/resource/Spanish_Formosa, http://dbpedia.org/resource/Spanish_Guinea, http://dbpedia.org/resource/Spanish_Main, http://dbpedia.org/resource/Spanish_West_Indies, http://dbpedia.org/resource/Sri_Lanka_national_cricket_team, http://dbpedia.org/resource/State_of_Indiana, http://dbpedia.org/resource/State_of_Lu, http://dbpedia.org/resource/State_of_the_Teutonic_Order, http://dbpedia.org/resource/Sui_generis, http://dbpedia.org/resource/Sultanate_of_Darfur, http://dbpedia.org/resource/Sultanate_of_Sulu, http://dbpedia.org/resource/Swedish_Gold_Coast, http://dbpedia.org/resource/Swedish_overseas_colonies, http://dbpedia.org/resource/Syria_Vilayet, http://dbpedia.org/resource/Taipei_County, http://dbpedia.org/resource/Taiwan_under_Japanese_rule, http://dbpedia.org/resource/Tatar, http://dbpedia.org/resource/Television_in_the_United_States, http://dbpedia.org/resource/Territory_of_Papua, http://dbpedia.org/resource/Tetbury, http://dbpedia.org/resource/The_Protectorate, http://dbpedia.org/resource/Tibetan_Empire, http://dbpedia.org/resource/Togoland, http://dbpedia.org/resource/Tonkin_(French_protectorate), http://dbpedia.org/resource/Trucial_States, http://dbpedia.org/resource/Trust_Territory_of_Somaliland, http://dbpedia.org/resource/Tuva, http://dbpedia.org/resource/US, http://dbpedia.org/resource/Udmurtia, http://dbpedia.org/resource/Ukrainian_State, http://dbpedia.org/resource/Ulaid, http://dbpedia.org/resource/Union_of_Sovereign_States, http://dbpedia.org/resource/Uyghur_Khaganate, http://dbpedia.org/resource/Valparaiso, http://dbpedia.org/resource/Vandal_Kingdom, http://dbpedia.org/resource/Venice,_Italy, http://dbpedia.org/resource/Vinland, http://dbpedia.org/resource/Voronezh_Governorate, http://dbpedia.org/resource/Walt_Disney_World_Resort, http://dbpedia.org/resource/West_Indies_Cricket_Board, http://dbpedia.org/resource/West_Indies_national_cricket_team, http://dbpedia.org/resource/Yangzhou_(ancient_China), http://dbpedia.org/resource/Yemen_Vilayet, http://dbpedia.org/resource/Zhili_Clique, http://dbpedia.org/resource/Zimbabwe_national_cricket_team, http://dbpedia.org/resource/Aceh_Sultanate, http://dbpedia.org/resource/Afghan_cuisine, http://dbpedia.org/resource/Alawite_State, http://dbpedia.org/resource/Albanian_people, http://dbpedia.org/resource/Alodia, http://dbpedia.org/resource/Arab_Federation, http://dbpedia.org/resource/Arab_Republic_of_Egypt, http://dbpedia.org/resource/Arabian, http://dbpedia.org/resource/Article_One_of_the_United_States_Constitution, http://dbpedia.org/resource/Austro-Hungary, http://dbpedia.org/resource/Ava_Kingdom, http://dbpedia.org/resource/Baltic_Assembly, http://dbpedia.org/resource/Bamana_Empire, http://dbpedia.org/resource/Barotseland, http://dbpedia.org/resource/Bosnia-Herzegovina, http://dbpedia.org/resource/Britanny, http://dbpedia.org/resource/British_National_(Overseas), http://dbpedia.org/resource/British_television, http://dbpedia.org/resource/Bunyoro, http://dbpedia.org/resource/Carinthian_Slovenes, http://dbpedia.org/resource/Central_Belt, http://dbpedia.org/resource/Cherokee_Nation, http://dbpedia.org/resource/Chola_Dynasty, http://dbpedia.org/resource/Chūzan, http://dbpedia.org/resource/Cisalpine_Republic, http://dbpedia.org/resource/Cispadane_Republic, http://dbpedia.org/resource/Colony_of_Aden, http://dbpedia.org/resource/Common_Market_for_Eastern_and_Southern_Africa, http://dbpedia.org/resource/Common_Monetary_Area, http://dbpedia.org/resource/Commune_of_Rome, http://dbpedia.org/resource/Confederation_of_the_Rhine, http://dbpedia.org/resource/Cook_Islands_Federation, http://dbpedia.org/resource/Crimean_Autonomous_Soviet_Socialist_Republic, http://dbpedia.org/resource/Darul_Islam_(Indonesia), http://dbpedia.org/resource/Democratic_Republic_of_Yemen, http://dbpedia.org/resource/Demographics_of_Mozambique, http://dbpedia.org/resource/Dendi_Kingdom, http://dbpedia.org/resource/Des_Moines_metropolitan_area, http://dbpedia.org/resource/Dominion_of_Fiji, http://dbpedia.org/resource/Duchy_of_Lucca, http://dbpedia.org/resource/Duchy_of_Massa_and_Carrara, http://dbpedia.org/resource/Duchy_of_Mecklenburg-Strelitz, http://dbpedia.org/resource/Duchy_of_Parma, http://dbpedia.org/resource/East_Frisia, http://dbpedia.org/resource/East_Jersey, http://dbpedia.org/resource/Eastern_Orthodox_liturgical_calendar, http://dbpedia.org/resource/Economy_of_Germany, http://dbpedia.org/resource/Education_in_the_United_States, http://dbpedia.org/resource/Egyptian_nationality_law, http://dbpedia.org/resource/Electorate_of_Hesse, http://dbpedia.org/resource/Emirate_of_Diriyah, http://dbpedia.org/resource/Emirate_of_Nejd, http://dbpedia.org/resource/Emirate_of_Nejd_and_Hasa, http://dbpedia.org/resource/Emirate_of_Sicily, http://dbpedia.org/resource/Empire_of_Great_Fulo, http://dbpedia.org/resource/English_national_identity, http://dbpedia.org/resource/Englishmen, http://dbpedia.org/resource/Enlightenment_in_Spain, http://dbpedia.org/resource/European_Economic_Area, http://dbpedia.org/resource/Euskal_Herria, http://dbpedia.org/resource/Far_Eastern_Republic, http://dbpedia.org/resource/Federal_Government_of_Somalia, http://dbpedia.org/resource/Federated_Malay_States, http://dbpedia.org/resource/Federation_of_Arab_Emirates_of_the_South, http://dbpedia.org/resource/Federation_of_South_Arabia, http://dbpedia.org/resource/Finnish_Democratic_Republic, http://dbpedia.org/resource/Finnish_Socialist_Workers'_Republic, http://dbpedia.org/resource/First_Hellenic_Republic, http://dbpedia.org/resource/First_Hungarian_Republic, http://dbpedia.org/resource/Five_Eyes, http://dbpedia.org/resource/Franceville,_New_Hebrides, http://dbpedia.org/resource/Free_City_of_Kraków, http://dbpedia.org/resource/Free_State_of_Fiume, http://dbpedia.org/resource/Gagauzia, http://dbpedia.org/resource/Galician_Soviet_Socialist_Republic, http://dbpedia.org/resource/German_Samoa, http://dbpedia.org/resource/Government_of_New_Zealand, http://dbpedia.org/resource/Governorate_of_Estonia, http://dbpedia.org/resource/Gozo_(independent_state), http://dbpedia.org/resource/Grand_Duchy_of_Mecklenburg-Schwerin, http://dbpedia.org/resource/Grand_Duchy_of_Mecklenburg-Strelitz, http://dbpedia.org/resource/Grand_Principality_of_Serbia, http://dbpedia.org/resource/Great_Britain_Davis_Cup_team, http://dbpedia.org/resource/Greater_China, http://dbpedia.org/resource/Greater_Republic_of_Central_America, http://dbpedia.org/resource/Habesh_Eyalet, http://dbpedia.org/resource/Himyarite_Kingdom, http://dbpedia.org/resource/Hohenzollern-Hechingen, http://dbpedia.org/resource/Hohenzollern-Sigmaringen, http://dbpedia.org/resource/Huaorani, http://dbpedia.org/resource/Hungarian_minority_in_Romania, http://dbpedia.org/resource/Idrisid_Emirate_of_Asir, http://dbpedia.org/resource/Imamate_of_Futa_Jallon, http://dbpedia.org/resource/Imamate_of_Futa_Toro, http://dbpedia.org/resource/International_Authority_for_the_Ruhr, http://dbpedia.org/resource/Irian_Jaya, http://dbpedia.org/resource/Irish_Australian, http://dbpedia.org/resource/Islands_of_Refreshment, http://dbpedia.org/resource/Italian_Empire, http://dbpedia.org/resource/Italian_Islands_of_the_Aegean, http://dbpedia.org/resource/Italian_Regency_of_Carnaro, http://dbpedia.org/resource/Italian_Republic_(Napoleonic), http://dbpedia.org/resource/Jolof_Empire, http://dbpedia.org/resource/Kaabu, http://dbpedia.org/resource/Kazakh_Khanate, http://dbpedia.org/resource/Khmers, http://dbpedia.org/resource/Kingdom_of_Aksum, http://dbpedia.org/resource/Kingdom_of_Bora_Bora, http://dbpedia.org/resource/Kingdom_of_Brittany, http://dbpedia.org/resource/Kingdom_of_Burundi, http://dbpedia.org/resource/Kingdom_of_Butua, http://dbpedia.org/resource/Kingdom_of_Corsica_(1736), http://dbpedia.org/resource/Kingdom_of_Hejaz, http://dbpedia.org/resource/Kingdom_of_Kurdistan, http://dbpedia.org/resource/Kingdom_of_Mutapa, http://dbpedia.org/resource/Kingdom_of_Nejd_and_Hejaz, http://dbpedia.org/resource/Kingdom_of_Pajang, http://dbpedia.org/resource/Kingdom_of_Rwanda, http://dbpedia.org/resource/Kingdom_of_Tahiti, http://dbpedia.org/resource/Konbaung_Dynasty, http://dbpedia.org/resource/Land_of_Ev, http://dbpedia.org/resource/Landgraviate_of_Hesse-Darmstadt, http://dbpedia.org/resource/Lanfang_Republic, http://dbpedia.org/resource/List_of_countries_in_the_Eurovision_Song_Contest, http://dbpedia.org/resource/Longueuil,_Quebec, http://dbpedia.org/resource/Mahdist_Sudan, http://dbpedia.org/resource/Majeerteen_Sultanate, http://dbpedia.org/resource/Mali_Federation, http://dbpedia.org/resource/Marstrand_Free_Port, http://dbpedia.org/resource/Mataram_Sultanate, http://dbpedia.org/resource/Medieval_Iceland, http://dbpedia.org/resource/Merina_Kingdom, http://dbpedia.org/resource/Middle_Saxons, http://dbpedia.org/resource/Miranda_do_Douro, http://dbpedia.org/resource/Mongolian_People's_Republic, http://dbpedia.org/resource/Morona-Santiago, http://dbpedia.org/resource/Mthwakazi, http://dbpedia.org/resource/Mutawakkilite_Kingdom_of_Yemen, http://dbpedia.org/resource/Netherlands_New_Guinea, http://dbpedia.org/resource/New_Zealanders, http://dbpedia.org/resource/Nicaraguan, http://dbpedia.org/resource/Nigerians, http://dbpedia.org/resource/Nogai_Horde, http://dbpedia.org/resource/North_Cape,_Norway, http://dbpedia.org/resource/North_Sea_Empire, http://dbpedia.org/resource/North_of_England, http://dbpedia.org/resource/Ojibwa, http://dbpedia.org/resource/Orange_River_Sovereignty, http://dbpedia.org/resource/Pacific_Union, http://dbpedia.org/resource/People's_Democratic_Republic_of_Ethiopia, http://dbpedia.org/resource/Philippians, http://dbpedia.org/resource/Portland_metropolitan_area, http://dbpedia.org/resource/Portuguese_East_Africa, http://dbpedia.org/resource/Principality_of_Erfurt, http://dbpedia.org/resource/Principality_of_Lippe, http://dbpedia.org/resource/Principality_of_Serbia_(medieval), http://dbpedia.org/resource/Principality_of_Transylvania_(1571–1711), http://dbpedia.org/resource/Principality_of_Trinidad, http://dbpedia.org/resource/Principality_of_Zeta, http://dbpedia.org/resource/Province_of_Saxony, http://dbpedia.org/resource/Provisional_Government_of_Hawaii, http://dbpedia.org/resource/Provisional_Revolutionary_Government_of_the_Republic_of_South_Vietnam, http://dbpedia.org/resource/Puget_Sound_region, http://dbpedia.org/resource/Rapa_Nui, http://dbpedia.org/resource/Regency_of_Algiers, http://dbpedia.org/resource/Republic_of_Acre, http://dbpedia.org/resource/Republic_of_Central_Lithuania, http://dbpedia.org/resource/Republic_of_Cospaia, http://dbpedia.org/resource/Republic_of_Ezo, http://dbpedia.org/resource/Republic_of_Formosa, http://dbpedia.org/resource/Republic_of_Hungary_(1946–1949), http://dbpedia.org/resource/Republic_of_Indian_Stream, http://dbpedia.org/resource/Republic_of_Kuwait, http://dbpedia.org/resource/Republic_of_Maryland, http://dbpedia.org/resource/Republic_of_Mountainous_Armenia, http://dbpedia.org/resource/Republic_of_North_Peru, http://dbpedia.org/resource/Republic_of_Prekmurje, http://dbpedia.org/resource/Republic_of_Rose_Island, http://dbpedia.org/resource/Republic_of_Sonora, http://dbpedia.org/resource/Republic_of_Vietnam, http://dbpedia.org/resource/Republic_of_Yucatán, http://dbpedia.org/resource/Republic_of_the_Rio_Grande, http://dbpedia.org/resource/Revolutionary_Government_Junta_of_El_Salvador, http://dbpedia.org/resource/Rif_Republic, http://dbpedia.org/resource/Romans, http://dbpedia.org/resource/Royal_Prussia, http://dbpedia.org/resource/Russian_Republic, http://dbpedia.org/resource/Ryukyu_Kingdom, http://dbpedia.org/resource/SADR, http://dbpedia.org/resource/Salento, http://dbpedia.org/resource/Saudeleur_Dynasty, http://dbpedia.org/resource/Saxe-Altenburg, http://dbpedia.org/resource/Saxe-Coburg-Saalfeld, http://dbpedia.org/resource/Saxe-Gotha, http://dbpedia.org/resource/Saxe-Hildburghausen, http://dbpedia.org/resource/Saxe-Meiningen, http://dbpedia.org/resource/Saxe-Weimar, http://dbpedia.org/resource/Saxe-Weimar-Eisenach, http://dbpedia.org/resource/Schwarzburg-Rudolstadt, http://dbpedia.org/resource/Schwarzburg-Sondershausen, http://dbpedia.org/resource/Scottish_cricket_team, http://dbpedia.org/resource/Sennar_(sultanate), http://dbpedia.org/resource/Serbian_Despotate, http://dbpedia.org/resource/Slovak_Soviet_Republic, http://dbpedia.org/resource/Slovak_people, http://dbpedia.org/resource/Socialist_Republic_of_Vietnam, http://dbpedia.org/resource/South-West_Africa, http://dbpedia.org/resource/South_Atlantic, http://dbpedia.org/resource/Southern_Jutland, http://dbpedia.org/resource/Southwell,_Nottinghamshire, http://dbpedia.org/resource/Soviet_Republic_of_Naissaar, http://dbpedia.org/resource/State_of_Aleppo, http://dbpedia.org/resource/State_of_Damascus, http://dbpedia.org/resource/State_of_Muskogee, http://dbpedia.org/resource/Sultanate_of_Cirebon, http://dbpedia.org/resource/Sultanate_of_Hobyo, http://dbpedia.org/resource/Sultanate_of_Maguindanao, http://dbpedia.org/resource/Sultanate_of_Nejd, http://dbpedia.org/resource/Surinam_(Dutch_colony), http://dbpedia.org/resource/São_Tomé, http://dbpedia.org/resource/Tanganyika_Territory, http://dbpedia.org/resource/Taungoo_Dynasty, http://dbpedia.org/resource/Telecorporación_Salvadoreña, http://dbpedia.org/resource/Terra_nullius, http://dbpedia.org/resource/Tibet_(1912–51), http://dbpedia.org/resource/Tidewater_region_of_Virginia, http://dbpedia.org/resource/Toucouleur_Empire, http://dbpedia.org/resource/Transkei, http://dbpedia.org/resource/Transpadane_Republic, http://dbpedia.org/resource/Travancore, http://dbpedia.org/resource/Tuʻi_Tonga_Empire, http://dbpedia.org/resource/United_Arab_Republic, http://dbpedia.org/resource/United_Belgian_States, http://dbpedia.org/resource/United_Colonies_of_Vancouver_Island_and_British_Columbia, http://dbpedia.org/resource/United_Kingdom_in_the_Eurovision_Song_Contest, http://dbpedia.org/resource/United_Mexican_States, http://dbpedia.org/resource/United_Principalities, http://dbpedia.org/resource/United_Suvadive_Republic, http://dbpedia.org/resource/Upper_Nile_State, http://dbpedia.org/resource/Việt_Nam, http://dbpedia.org/resource/Volga_Bulgaria, http://dbpedia.org/resource/Warsangali_Sultanate, http://dbpedia.org/resource/Wassoulou_Empire, http://dbpedia.org/resource/Wattasid_dynasty, http://dbpedia.org/resource/West_Jersey, http://dbpedia.org/resource/West_Prussia, http://dbpedia.org/resource/Western_Balkans, http://dbpedia.org/resource/Wikt:Anglo-French, http://dbpedia.org/resource/Württemberg, http://dbpedia.org/resource/Yeke_Kingdom, http://dbpedia.org/resource/Yemen_Arab_Republic, http://dbpedia.org/resource/Zimbabwe_Rhodesia, http://dbpedia.org/resource/Argentine_Federal_Police, http://dbpedia.org/resource/Avatele, http://dbpedia.org/resource/Bangsamoro_Republik, http://dbpedia.org/resource/Bavaria-Ingolstadt, http://dbpedia.org/resource/Bishopric_of_Merseburg, http://dbpedia.org/resource/British_Arctic_Territories, http://dbpedia.org/resource/Buyeo, http://dbpedia.org/resource/Central_Mexico, http://dbpedia.org/resource/Chernigov_Governorate, http://dbpedia.org/resource/Colville_tribe, http://dbpedia.org/resource/Commonwealth_of_Dominica, http://dbpedia.org/resource/Constitution_of_Namibia, http://dbpedia.org/resource/Curaçao_and_Dependencies, http://dbpedia.org/resource/Dedemsvaart, http://dbpedia.org/resource/Dependencies_of_Norway, http://dbpedia.org/resource/Dutch_nationality_law, http://dbpedia.org/resource/Freetown,_Sierra_Leone, http://dbpedia.org/resource/Government_of_South_Korea, http://dbpedia.org/resource/Gurung, http://dbpedia.org/resource/Hersfeld_Abbey, http://dbpedia.org/resource/History_of_the_Khitans, http://dbpedia.org/resource/Inuk, http://dbpedia.org/resource/Jinsha_(archaeological_site), http://dbpedia.org/resource/Khumbu, http://dbpedia.org/resource/Korean_ethnic_nationalism, http://dbpedia.org/resource/List_of_American_artists, http://dbpedia.org/resource/List_of_Canadians_of_English_descent, http://dbpedia.org/resource/List_of_Roman_Catholic_dioceses_(structured_view), http://dbpedia.org/resource/Lordship_of_Overijssel, http://dbpedia.org/resource/Lordship_of_Utrecht, http://dbpedia.org/resource/Majuro,_Marshall_Islands, http://dbpedia.org/resource/Namibia_national_cricket_team, http://dbpedia.org/resource/New_York_City_metropolitan_area, http://dbpedia.org/resource/Northern_Limit_Line, http://dbpedia.org/resource/Operational_Zone_of_the_Adriatic_Littoral, http://dbpedia.org/resource/Operational_Zone_of_the_Alpine_Foothills, http://dbpedia.org/resource/Ovamboland, http://dbpedia.org/resource/Pozsony_County, http://dbpedia.org/resource/Prince-Bishopric_of_Montenegro, http://dbpedia.org/resource/Provisional_People's_Committee_for_North_Korea, http://dbpedia.org/resource/Rai_dynasty, http://dbpedia.org/resource/Rajahnate_of_Butuan, http://dbpedia.org/resource/Reichskommissariat_Kaukasus, http://dbpedia.org/resource/Reichskommissariat_Moskowien, http://dbpedia.org/resource/Reichskommissariat_Niederlande, http://dbpedia.org/resource/Reichskommissariat_Turkestan, http://dbpedia.org/resource/Republic_of_Bulgaria, http://dbpedia.org/resource/Right_of_abode_issue,_Hong_Kong, http://dbpedia.org/resource/SSS_islands, http://dbpedia.org/resource/Serbians, http://dbpedia.org/resource/Soviet_Civil_Administration, http://dbpedia.org/resource/Subanon_people, http://dbpedia.org/resource/Suriname_(Kingdom_of_the_Netherlands), http://dbpedia.org/resource/Syrian_cuisine, http://dbpedia.org/resource/Tamang, http://dbpedia.org/resource/Taupo_Volcanic_Zone, http://dbpedia.org/resource/Tewa_people, http://dbpedia.org/resource/Thakali_people, http://dbpedia.org/resource/The_United_States, http://dbpedia.org/resource/Uruzgan, http://dbpedia.org/resource/Uzbek_Soviet_Socialist_Republic, http://dbpedia.org/resource/Visayas, http://dbpedia.org/resource/Washington,_D.C._metropolitan_area, http://dbpedia.org/resource/West_bank, http://dbpedia.org/resource/Western_Pomerania, http://dbpedia.org/resource/Western_Satraps, http://dbpedia.org/resource/Whatcom_County, http://dbpedia.org/resource/Yadav, http://dbpedia.org/resource/Yakama_Nation, http://dbpedia.org/resource/Zamboanga_Peninsula_(geographical_region), http://dbpedia.org/resource/Baghdad,_Iraq, http://dbpedia.org/resource/Basilica_of_Saint_John_Lateran, http://dbpedia.org/resource/Beer_in_Ireland, http://dbpedia.org/resource/Bishopric_of_Halberstadt, http://dbpedia.org/resource/Bishopric_of_Metz, http://dbpedia.org/resource/Bishopric_of_Minden, http://dbpedia.org/resource/Bishopric_of_Verdun, http://dbpedia.org/resource/Catepanate_of_Italy, http://dbpedia.org/resource/Congo-Brazzaville, http://dbpedia.org/resource/County_of_Kladsko, http://dbpedia.org/resource/County_of_Savoy, http://dbpedia.org/resource/Cuisine_of_Thailand, http://dbpedia.org/resource/Duchy_of_Florence, http://dbpedia.org/resource/Eastern_Orthodox_Christianity, http://dbpedia.org/resource/Engelberg_Abbey, http://dbpedia.org/resource/Hungarian-American, http://dbpedia.org/resource/Hunza_(princely_state), http://dbpedia.org/resource/Lithuanian-American, http://dbpedia.org/resource/Nabataean_Kingdom, http://dbpedia.org/resource/Norman_England, http://dbpedia.org/resource/PE_postcode_area, http://dbpedia.org/resource/Palatinate-Neuburg, http://dbpedia.org/resource/Palistin, http://dbpedia.org/resource/Posen-West_Prussia, http://dbpedia.org/resource/Principality_of_Piombino, http://dbpedia.org/resource/United_Duchies_of_Jülich-Cleves-Berg, http://dbpedia.org/resource/Vatican_Basilica, http://dbpedia.org/resource/Zanskar, http://dbpedia.org/resource/Benzingerode, http://dbpedia.org/resource/Bishopric_of_Trent, http://dbpedia.org/resource/Canada_men's_national_junior_ice_hockey_team, http://dbpedia.org/resource/Cinema_of_South_Africa, http://dbpedia.org/resource/County_of_Wernigerode, http://dbpedia.org/resource/Danish_ethnic_group, http://dbpedia.org/resource/Darug_people, http://dbpedia.org/resource/Duchy_of_Guastalla, http://dbpedia.org/resource/Duchy_of_Modena_and_Reggio, http://dbpedia.org/resource/English_(people), http://dbpedia.org/resource/Gévaudan, http://dbpedia.org/resource/Hungarian_Republic_(1919–20), http://dbpedia.org/resource/Hyolmo_people, http://dbpedia.org/resource/Ieper, http://dbpedia.org/resource/Koch_dynasty, http://dbpedia.org/resource/Kyrgyz_Republic, http://dbpedia.org/resource/Malayalis, http://dbpedia.org/resource/Marquisate_of_Finale, http://dbpedia.org/resource/New_Zealand_Cricket, http://dbpedia.org/resource/Polish_National_Government_(January_Uprising), http://dbpedia.org/resource/Pottawatomie, http://dbpedia.org/resource/San_Luis_Obispo, http://dbpedia.org/resource/Sandwich,_Ontario, http://dbpedia.org/resource/Sauerland, http://dbpedia.org/resource/Stolberg-Wernigerode, http://dbpedia.org/resource/Te_Āti_Awa, http://dbpedia.org/resource/United_States_Ambassador_to_Colombia, http://dbpedia.org/resource/United_States_Shipping_Board, http://dbpedia.org/resource/Viipuri_Province, http://dbpedia.org/resource/Wallis_(island), http://dbpedia.org/resource/Worms-Pfeddersheim, http://dbpedia.org/resource/Yotvingia, http://dbpedia.org/resource/Armilla,_Granada, http://dbpedia.org/resource/Bishopric_of_Speyer, http://dbpedia.org/resource/Counts_of_Toggenburg, http://dbpedia.org/resource/County_of_Horne, http://dbpedia.org/resource/County_of_Veldenz, http://dbpedia.org/resource/Despotate_of_Arta, http://dbpedia.org/resource/Deva_dynasty, http://dbpedia.org/resource/Duchy_of_Bouillon, http://dbpedia.org/resource/El_Aaiun, http://dbpedia.org/resource/Free_Imperial_City_of_Aachen, http://dbpedia.org/resource/German_Empire_(1848/1849), http://dbpedia.org/resource/Grand_Duchy_of_Berg, http://dbpedia.org/resource/Greater_Hesse, http://dbpedia.org/resource/Hesse-Marburg, http://dbpedia.org/resource/History_of_Burma, http://dbpedia.org/resource/History_of_Speyer, http://dbpedia.org/resource/Ionian_Islands_under_Venetian_rule, http://dbpedia.org/resource/Kadamba_dynasty, http://dbpedia.org/resource/Kagawa_prefecture, http://dbpedia.org/resource/Kalabhra_dynasty, http://dbpedia.org/resource/Kingdom_of_France_(1791–92), http://dbpedia.org/resource/Kingdom_of_Gampola, http://dbpedia.org/resource/Kingdom_of_Upatissa_Nuwara, http://dbpedia.org/resource/Maitraka, http://dbpedia.org/resource/Mamluk_Dynasty_(Delhi), http://dbpedia.org/resource/Maukhari, http://dbpedia.org/resource/Nagas_of_Padmavati, http://dbpedia.org/resource/Nayakas_of_Keladi, http://dbpedia.org/resource/Pale_of_Calais, http://dbpedia.org/resource/Paramara_dynasty, http://dbpedia.org/resource/Principality_of_Regensburg, http://dbpedia.org/resource/Ramnad_estate, http://dbpedia.org/resource/Rascians, http://dbpedia.org/resource/Starhemberg, http://dbpedia.org/resource/State_of_Aden, http://dbpedia.org/resource/Sutiya_Kingdom, http://dbpedia.org/resource/Taifa_of_Granada, http://dbpedia.org/resource/Thanjavur_Nayak_kingdom, http://dbpedia.org/resource/The_People's_Republic_of_China, http://dbpedia.org/resource/Torres_Strait_Islands, http://dbpedia.org/resource/Vishnukundina, http://dbpedia.org/resource/Weimer_Republic, http://dbpedia.org/resource/Amazon_Region_of_Colombia, http://dbpedia.org/resource/Bhopal_State, http://dbpedia.org/resource/Chandela, http://dbpedia.org/resource/Chhattisgarh_Division, http://dbpedia.org/resource/Duchy_of_Saxe-Wittenberg, http://dbpedia.org/resource/Jin_(Korean_state), http://dbpedia.org/resource/New_South_Wales,_Australia, http://dbpedia.org/resource/Palestinian_Authority_passport, http://dbpedia.org/resource/Powys_Wenwynwyn, http://dbpedia.org/resource/Principality_of_Brunswick-Wolfenbüttel, http://dbpedia.org/resource/Province_of_Québec, http://dbpedia.org/resource/Salm_(state), http://dbpedia.org/resource/Sapwuahfik, http://dbpedia.org/resource/Scheduled_Tribes, http://dbpedia.org/resource/Taebong, http://dbpedia.org/resource/Weingarten_Abbey, http://dbpedia.org/resource/Acre_Sanjak, http://dbpedia.org/resource/Argentinian_people, http://dbpedia.org/resource/Auckland_Province, http://dbpedia.org/resource/Cinema_of_Serbia, http://dbpedia.org/resource/Cuisine_of_Malaysia, http://dbpedia.org/resource/Czech_republic, http://dbpedia.org/resource/Danube_Banovina, http://dbpedia.org/resource/Eastern_Xia, http://dbpedia.org/resource/Eiderstedt, http://dbpedia.org/resource/Federation_of_Nigeria, http://dbpedia.org/resource/Free_and_Independent_State_of_Cundinamarca, http://dbpedia.org/resource/Governorate_of_New_Toledo, http://dbpedia.org/resource/Guyana_(Commonwealth_realm), http://dbpedia.org/resource/Interim_Government_of_Somalia, http://dbpedia.org/resource/Kimberley_region_of_Western_Australia, http://dbpedia.org/resource/Køge_Bay, http://dbpedia.org/resource/Malawi_(Commonwealth_realm), http://dbpedia.org/resource/Mauritius_(Commonwealth_realm), http://dbpedia.org/resource/People's_Revolutionary_Government_(Grenada), http://dbpedia.org/resource/Principality_of_Mingrelia, http://dbpedia.org/resource/Queen_of_New_Zealand, http://dbpedia.org/resource/Realm_of_Stefan_Dragutin, http://dbpedia.org/resource/Rosebud_Sioux_Tribe, http://dbpedia.org/resource/Sans_Arc, http://dbpedia.org/resource/Santa_Rosa_Island,_California, http://dbpedia.org/resource/Sayn-Wittgenstein-Berleburg, http://dbpedia.org/resource/Sayn-Wittgenstein-Hohenstein, http://dbpedia.org/resource/Sierra_Leone_(Commonwealth_realm), http://dbpedia.org/resource/Tainan_County, http://dbpedia.org/resource/The_Gambia_(Commonwealth_realm), http://dbpedia.org/resource/Third_Republic_of_Venezuela, http://dbpedia.org/resource/Trinidad_and_Tobago_(Commonwealth_realm), http://dbpedia.org/resource/Valle_di_Susa, http://dbpedia.org/resource/Venezuela_Province, http://dbpedia.org/resource/Xinjing, http://dbpedia.org/resource/British_Formula_Renault_Championship, http://dbpedia.org/resource/El_Seibo_Province, http://dbpedia.org/resource/European_Rental_Association, http://dbpedia.org/resource/Glassport,_Pennsylvania, http://dbpedia.org/resource/Matmâta, http://dbpedia.org/resource/Northern_Syria, http://dbpedia.org/resource/Rugiland, http://dbpedia.org/resource/San_José_de_Ocoa_Province, http://dbpedia.org/resource/Television_in_the_Soviet_Union, http://dbpedia.org/resource/United_States_men's_national_volleyball_team, http://dbpedia.org/resource/Anglo-Scottish_Border, http://dbpedia.org/resource/Armenians_in_Abkhazia, http://dbpedia.org/resource/Cassel,_France, http://dbpedia.org/resource/Flint/Tri-Cities, http://dbpedia.org/resource/Hui'an, http://dbpedia.org/resource/Indian_British, http://dbpedia.org/resource/Islamic_denominations, http://dbpedia.org/resource/Pakistani_British, http://dbpedia.org/resource/Qiong_Wen, http://dbpedia.org/resource/Syrian_Government, http://dbpedia.org/resource/Wikt:British, http://dbpedia.org/resource/Älvsborg_County, http://dbpedia.org/resource/Bolivian_American, http://dbpedia.org/resource/Cangnan, http://dbpedia.org/resource/Former_Zhao, http://dbpedia.org/resource/Hauraki_Plains, http://dbpedia.org/resource/Ngāti_Kahungunu, http://dbpedia.org/resource/Old_Peru,_Iowa, http://dbpedia.org/resource/Swiss-American, http://dbpedia.org/resource/Argentine_American, http://dbpedia.org/resource/Bendery, http://dbpedia.org/resource/CHINA, http://dbpedia.org/resource/Carriacou_and_Petite_Martinique, http://dbpedia.org/resource/County_of_Tecklenburg, http://dbpedia.org/resource/Dena'ina_people, http://dbpedia.org/resource/España, http://dbpedia.org/resource/Glen_Affric, http://dbpedia.org/resource/Inverpolly, http://dbpedia.org/resource/Jolo_Island, http://dbpedia.org/resource/Lake_Chaohu, http://dbpedia.org/resource/Lake_Iliamna, http://dbpedia.org/resource/Stegeborg_Castle, http://dbpedia.org/resource/Fifth_Republic_of_South_Korea, http://dbpedia.org/resource/United_States_cricket_team, http://dbpedia.org/resource/Venezuelan, http://dbpedia.org/resource/Venezuelan_American, http://dbpedia.org/resource/A_Shau_Valley, http://dbpedia.org/resource/Correctional_Service_of_Canada, http://dbpedia.org/resource/Gothenburg_and_Bohus_County, http://dbpedia.org/resource/Kristianstad_County, http://dbpedia.org/resource/List_of_reportedly_haunted_locations_in_the_United_States, http://dbpedia.org/resource/Malmöhus_County, http://dbpedia.org/resource/Aberdeenshire,_Scotland, http://dbpedia.org/resource/Dutch_cricket_team, http://dbpedia.org/resource/Prince_Rupert's_Land, http://dbpedia.org/resource/Reutje, http://dbpedia.org/resource/Cinema_of_Austria, http://dbpedia.org/resource/David_C.H._Austin, http://dbpedia.org/resource/Dominican-American, http://dbpedia.org/resource/Hrodna_Voblast, http://dbpedia.org/resource/New_Zealand_people, http://dbpedia.org/resource/Newark_Basin, http://dbpedia.org/resource/Sierra_Leonean, http://dbpedia.org/resource/Kiowa_Indian_Tribe_of_Oklahoma, http://dbpedia.org/resource/Krylbo, http://dbpedia.org/resource/Trångsund, http://dbpedia.org/resource/American_Jew, http://dbpedia.org/resource/Arenberg, http://dbpedia.org/resource/Prince-Bishopric_of_Paderborn, http://dbpedia.org/resource/United_States_Ambassador_to_Denmark, http://dbpedia.org/resource/2013_Australian_motorcycle_Grand_Prix, http://dbpedia.org/resource/Alter_Botanischer_Garten_der_Universität_Göttingen, http://dbpedia.org/resource/Arkhangelsk_Governorate, http://dbpedia.org/resource/Cangnan_County, http://dbpedia.org/resource/Columbian_Viceroyalty, http://dbpedia.org/resource/Desaguadero_River, http://dbpedia.org/resource/Kamiina_District,_Nagano, http://dbpedia.org/resource/Kolathunadu, http://dbpedia.org/resource/Northern_Krai, http://dbpedia.org/resource/Schöntal_Abbey, http://dbpedia.org/resource/U.S.A, http://dbpedia.org/resource/Victoria_County,_Ontario, http://dbpedia.org/resource/African_National_Congress_of_South_Africa, http://dbpedia.org/resource/Danava_dynasty, http://dbpedia.org/resource/England_national_rugby_sevens_team, http://dbpedia.org/resource/Japan_Sevens, http://dbpedia.org/resource/Kachari_Kingdom, http://dbpedia.org/resource/Naraka_dynasty, http://dbpedia.org/resource/Olonets_Governorate, http://dbpedia.org/resource/Rionegro,_Antioquia, http://dbpedia.org/resource/Taringa,_Queensland, http://dbpedia.org/resource/Tasmanian, http://dbpedia.org/resource/Uruguayan, http://dbpedia.org/resource/Alleppey, http://dbpedia.org/resource/Ciudad_de_Guatemala, http://dbpedia.org/resource/Duchy_of_Bytom, http://dbpedia.org/resource/Firuzkuh, http://dbpedia.org/resource/Free_Republic_of_Schwarzenberg, http://dbpedia.org/resource/Guyana_cricket_team, http://dbpedia.org/resource/KSA, http://dbpedia.org/resource/Kastamonu_Vilayet, http://dbpedia.org/resource/Orio, http://dbpedia.org/resource/Phuoc_Tuy_Province, http://dbpedia.org/resource/Saxe-Zeitz, http://dbpedia.org/resource/Torne_Valley, http://dbpedia.org/resource/Arakan_State, http://dbpedia.org/resource/German_passport, http://dbpedia.org/resource/Hsipaw_State, http://dbpedia.org/resource/Phayao_Kingdom, http://dbpedia.org/resource/Phillippines]... [INFO] P: 0.0018933417481855476, R: 1.0, F: 0.0037795275590551186 -[INFO] Cumulative metrics, sample: 74, P: 0.331, R: 0.370, macro F1: 0.328, macro F1*: 0.349 -[INFO] QALD Cumulative metrics, sample: 74, P: 0.547, R: 0.370, macro F1: 0.328, QALD macro F1: 0.441 -[INFO] Current time: 410 ms, average: 3307.54 ms/ques, entity detection: 712.55 ms/ques (linking tool: 626.22 ms/ques), relation detection: 1045.15 ms/ques (similarity: 605.03 ms/ques), EDG generation: 73.00 ms/ques +[INFO] Cumulative metrics, sample: 74, P: 0.331, R: 0.370, macro F1: 0.349 +[INFO] QALD Cumulative metrics, sample: 74, P: 0.547, R: 0.370, average F1: 0.328, QALD macro F1: 0.441 +[INFO] Current question consumed time: 388 ms, average: 3096.24 ms/ques, entity detection: 651.49 ms/ques (linking tool: 573.85 ms/ques), relation detection: 1001.34 ms/ques (similarity: 608.18 ms/ques), EDG generation: 73.47 ms/ques [INFO] Entity Linking P: 0.2702702702702703, R: 0.2702702702702703, micro F1: 0.2702702702702703, macro F1: 0.2702702702702703 [INFO] Relation Linking P: 0.2972972972972973, R: 0.2972972972972973, micro F1: 0.2972972972972973, macro F1: 0.2972972972972973 [INFO] Type Linking P: 0.918918918918919, R: 0.918918918918919, micro F1: 0.918918918918919, macro F1: 0.918918918918919 @@ -1117,9 +1117,9 @@ Question 74: Give me all movies with Tom Cruise. [INFO] Golden Answer: [http://dbpedia.org/resource/Cocktail_(1988_film), http://dbpedia.org/resource/Mission:_Impossible_(film), http://dbpedia.org/resource/A_Few_Good_Men, http://dbpedia.org/resource/Rain_Man, http://dbpedia.org/resource/Jerry_Maguire, http://dbpedia.org/resource/Minority_Report_(film), http://dbpedia.org/resource/Lions_for_Lambs, http://dbpedia.org/resource/Mission:_Impossible_(film_series), http://dbpedia.org/resource/Magnolia_(film), http://dbpedia.org/resource/Risky_Business, http://dbpedia.org/resource/The_Firm_(1993_film), http://dbpedia.org/resource/Top_Gun, http://dbpedia.org/resource/The_Color_of_Money, http://dbpedia.org/resource/Collateral_(film), http://dbpedia.org/resource/The_Outsiders_(film), http://dbpedia.org/resource/Valkyrie_(film), http://dbpedia.org/resource/Born_on_the_Fourth_of_July_(film), http://dbpedia.org/resource/Legend_(1985_film), http://dbpedia.org/resource/Interview_with_the_Vampire_(film), http://dbpedia.org/resource/Losin'_It, http://dbpedia.org/resource/All_the_Right_Moves_(film), http://dbpedia.org/resource/Far_and_Away, http://dbpedia.org/resource/Knight_and_Day, http://dbpedia.org/resource/The_Mummy_(2017_film), http://dbpedia.org/resource/Vanilla_Sky, http://dbpedia.org/resource/The_Last_Samurai] [INFO] Predicted Answer: [http://dbpedia.org/resource/Cocktail_(1988_film), http://dbpedia.org/resource/Mission:_Impossible_(film), http://dbpedia.org/resource/A_Few_Good_Men, http://dbpedia.org/resource/Vanilla_Sky, http://dbpedia.org/resource/Rain_Man, http://dbpedia.org/resource/The_Last_Samurai, http://dbpedia.org/resource/Jerry_Maguire, http://dbpedia.org/resource/Minority_Report_(film), http://dbpedia.org/resource/Lions_for_Lambs, http://dbpedia.org/resource/Mission:_Impossible_(film_series), http://dbpedia.org/resource/Magnolia_(film), http://dbpedia.org/resource/Risky_Business, http://dbpedia.org/resource/The_Firm_(1993_film), http://dbpedia.org/resource/Top_Gun, http://dbpedia.org/resource/The_Color_of_Money, http://dbpedia.org/resource/Collateral_(film), http://dbpedia.org/resource/The_Outsiders_(film), http://dbpedia.org/resource/Valkyrie_(film), http://dbpedia.org/resource/Born_on_the_Fourth_of_July_(film), http://dbpedia.org/resource/Legend_(1985_film), http://dbpedia.org/resource/Interview_with_the_Vampire_(film), http://dbpedia.org/resource/Losin'_It, http://dbpedia.org/resource/All_the_Right_Moves_(film), http://dbpedia.org/resource/Far_and_Away, http://dbpedia.org/resource/Knight_and_Day, http://dbpedia.org/resource/The_Mummy_(2017_film)] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 75, P: 0.340, R: 0.378, macro F1: 0.337, macro F1*: 0.358 -[INFO] QALD Cumulative metrics, sample: 75, P: 0.553, R: 0.378, macro F1: 0.337, QALD macro F1: 0.449 -[INFO] Current time: 1469 ms, average: 3283.03 ms/ques, entity detection: 706.68 ms/ques (linking tool: 620.05 ms/ques), relation detection: 1036.79 ms/ques (similarity: 599.91 ms/ques), EDG generation: 72.16 ms/ques +[INFO] Cumulative metrics, sample: 75, P: 0.340, R: 0.378, macro F1: 0.358 +[INFO] QALD Cumulative metrics, sample: 75, P: 0.553, R: 0.378, average F1: 0.337, QALD macro F1: 0.449 +[INFO] Current question consumed time: 1364 ms, average: 3073.15 ms/ques, entity detection: 645.97 ms/ques (linking tool: 568.23 ms/ques), relation detection: 993.29 ms/ques (similarity: 603.04 ms/ques), EDG generation: 72.64 ms/ques [INFO] Entity Linking P: 0.26666666666666666, R: 0.26666666666666666, micro F1: 0.26666666666666666, macro F1: 0.26666666666666666 [INFO] Relation Linking P: 0.29333333333333333, R: 0.29333333333333333, micro F1: 0.29333333333333333, macro F1: 0.29333333333333333 [INFO] Type Linking P: 0.92, R: 0.92, micro F1: 0.92, macro F1: 0.92 @@ -1132,9 +1132,9 @@ Question 75: When did Dracula's creator die? [INFO] Golden Answer: [1912-04-20] [INFO] Predicted Answer: [1897-05-26] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 76, P: 0.335, R: 0.373, macro F1: 0.333, macro F1*: 0.353 -[INFO] QALD Cumulative metrics, sample: 76, P: 0.546, R: 0.373, macro F1: 0.333, QALD macro F1: 0.443 -[INFO] Current time: 1370 ms, average: 3257.86 ms/ques, entity detection: 703.24 ms/ques (linking tool: 615.25 ms/ques), relation detection: 1027.09 ms/ques (similarity: 594.09 ms/ques), EDG generation: 71.32 ms/ques +[INFO] Cumulative metrics, sample: 76, P: 0.335, R: 0.373, macro F1: 0.353 +[INFO] QALD Cumulative metrics, sample: 76, P: 0.546, R: 0.373, average F1: 0.333, QALD macro F1: 0.443 +[INFO] Current question consumed time: 1209 ms, average: 3048.62 ms/ques, entity detection: 642.51 ms/ques (linking tool: 563.68 ms/ques), relation detection: 983.93 ms/ques (similarity: 597.17 ms/ques), EDG generation: 71.79 ms/ques [INFO] Entity Linking P: 0.2631578947368421, R: 0.2631578947368421, micro F1: 0.2631578947368421, macro F1: 0.2631578947368421 [INFO] Relation Linking P: 0.2894736842105263, R: 0.2894736842105263, micro F1: 0.2894736842105263, macro F1: 0.2894736842105263 [INFO] Type Linking P: 0.9210526315789473, R: 0.9210526315789473, micro F1: 0.9210526315789473, macro F1: 0.9210526315789473 @@ -1147,9 +1147,9 @@ Question 76: Who created English Wikipedia? [INFO] Golden Answer: [http://dbpedia.org/resource/Larry_Sanger, http://dbpedia.org/resource/Jimmy_Wales] [INFO] Predicted Answer: [http://dbpedia.org/resource/Larry_Sanger, http://dbpedia.org/resource/Jimmy_Wales] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 77, P: 0.344, R: 0.381, macro F1: 0.342, macro F1*: 0.362 -[INFO] QALD Cumulative metrics, sample: 77, P: 0.552, R: 0.381, macro F1: 0.342, QALD macro F1: 0.451 -[INFO] Current time: 947 ms, average: 3227.86 ms/ques, entity detection: 695.64 ms/ques (linking tool: 608.35 ms/ques), relation detection: 1017.44 ms/ques (similarity: 588.39 ms/ques), EDG generation: 70.47 ms/ques +[INFO] Cumulative metrics, sample: 77, P: 0.344, R: 0.381, macro F1: 0.362 +[INFO] QALD Cumulative metrics, sample: 77, P: 0.552, R: 0.381, average F1: 0.342, QALD macro F1: 0.451 +[INFO] Current question consumed time: 847 ms, average: 3020.03 ms/ques, entity detection: 635.34 ms/ques (linking tool: 557.22 ms/ques), relation detection: 974.56 ms/ques (similarity: 591.43 ms/ques), EDG generation: 70.92 ms/ques [INFO] Entity Linking P: 0.2597402597402597, R: 0.2597402597402597, micro F1: 0.2597402597402597, macro F1: 0.2597402597402597 [INFO] Relation Linking P: 0.2857142857142857, R: 0.2857142857142857, micro F1: 0.2857142857142857, macro F1: 0.2857142857142857 [INFO] Type Linking P: 0.922077922077922, R: 0.922077922077922, micro F1: 0.922077922077922, macro F1: 0.922077922077922 @@ -1162,9 +1162,9 @@ Question 77: Give me all female German chancellors. [INFO] Golden Answer: [http://dbpedia.org/resource/Angela_Merkel] [INFO] Predicted Answer: [] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 78, P: 0.339, R: 0.376, macro F1: 0.337, macro F1*: 0.357 -[INFO] QALD Cumulative metrics, sample: 78, P: 0.557, R: 0.376, macro F1: 0.337, QALD macro F1: 0.449 -[INFO] Current time: 831 ms, average: 3197.13 ms/ques, entity detection: 688.58 ms/ques (linking tool: 601.81 ms/ques), relation detection: 1007.19 ms/ques (similarity: 582.19 ms/ques), EDG generation: 69.68 ms/ques +[INFO] Cumulative metrics, sample: 78, P: 0.339, R: 0.376, macro F1: 0.357 +[INFO] QALD Cumulative metrics, sample: 78, P: 0.557, R: 0.376, average F1: 0.337, QALD macro F1: 0.449 +[INFO] Current question consumed time: 775 ms, average: 2991.24 ms/ques, entity detection: 628.95 ms/ques (linking tool: 551.29 ms/ques), relation detection: 964.78 ms/ques (similarity: 585.15 ms/ques), EDG generation: 70.13 ms/ques [INFO] Entity Linking P: 0.2692307692307692, R: 0.2692307692307692, micro F1: 0.2692307692307692, macro F1: 0.2692307692307692 [INFO] Relation Linking P: 0.2948717948717949, R: 0.2948717948717949, micro F1: 0.2948717948717949, macro F1: 0.2948717948717949 [INFO] Type Linking P: 0.9230769230769231, R: 0.9230769230769231, micro F1: 0.9230769230769231, macro F1: 0.9230769230769231 @@ -1177,9 +1177,9 @@ Question 78: Who owns Aldi? [INFO] Golden Answer: [http://dbpedia.org/resource/Theo_Albrecht, http://dbpedia.org/resource/Karl_Albrecht] [INFO] Predicted Answer: [http://dbpedia.org/resource/Aldi_Talk] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 79, P: 0.335, R: 0.372, macro F1: 0.333, macro F1*: 0.352 -[INFO] QALD Cumulative metrics, sample: 79, P: 0.550, R: 0.372, macro F1: 0.333, QALD macro F1: 0.444 -[INFO] Current time: 914 ms, average: 3168.24 ms/ques, entity detection: 682.34 ms/ques (linking tool: 595.49 ms/ques), relation detection: 998.10 ms/ques (similarity: 576.85 ms/ques), EDG generation: 68.86 ms/ques +[INFO] Cumulative metrics, sample: 79, P: 0.335, R: 0.372, macro F1: 0.352 +[INFO] QALD Cumulative metrics, sample: 79, P: 0.550, R: 0.372, average F1: 0.333, QALD macro F1: 0.444 +[INFO] Current question consumed time: 812 ms, average: 2963.66 ms/ques, entity detection: 623.20 ms/ques (linking tool: 545.54 ms/ques), relation detection: 956.00 ms/ques (similarity: 579.73 ms/ques), EDG generation: 69.29 ms/ques [INFO] Entity Linking P: 0.26582278481012656, R: 0.26582278481012656, micro F1: 0.26582278481012656, macro F1: 0.26582278481012656 [INFO] Relation Linking P: 0.2911392405063291, R: 0.2911392405063291, micro F1: 0.2911392405063291, macro F1: 0.2911392405063291 [INFO] Type Linking P: 0.9240506329113924, R: 0.9240506329113924, micro F1: 0.9240506329113924, macro F1: 0.9240506329113924 @@ -1192,9 +1192,9 @@ Question 79: Which books were written by Danielle Steel? [INFO] Golden Answer: [http://dbpedia.org/resource/Bungalow_2, http://dbpedia.org/resource/Toxic_Bachelors, http://dbpedia.org/resource/Friends_Forever_(novel), http://dbpedia.org/resource/Honor_Thyself, http://dbpedia.org/resource/Matters_of_the_Heart_(novel), http://dbpedia.org/resource/Betrayal_(Steel_novel), http://dbpedia.org/resource/Echoes_(Steel_novel), http://dbpedia.org/resource/Five_Days_in_Paris, http://dbpedia.org/resource/Message_from_Nam, http://dbpedia.org/resource/A_Good_Woman_(novel), http://dbpedia.org/resource/A_Perfect_Stranger, http://dbpedia.org/resource/Accident_(novel), http://dbpedia.org/resource/Amazing_Grace_(novel), http://dbpedia.org/resource/Big_Girl_(novel), http://dbpedia.org/resource/Coming_Out_(novel), http://dbpedia.org/resource/Crossings_(Steel_novel), http://dbpedia.org/resource/Daddy_(novel), http://dbpedia.org/resource/Family_Album_(novel), http://dbpedia.org/resource/Family_Ties_(novel), http://dbpedia.org/resource/Fine_Things, http://dbpedia.org/resource/Full_Circle_(novel), http://dbpedia.org/resource/H.R.H._(novel), http://dbpedia.org/resource/Hotel_Vendome_(novel), http://dbpedia.org/resource/Impossible_(novel), http://dbpedia.org/resource/Jewels_(novel), http://dbpedia.org/resource/Kaleidoscope_(novel), http://dbpedia.org/resource/Miracle_(novel), http://dbpedia.org/resource/Mixed_Blessings_(novel), http://dbpedia.org/resource/No_Greater_Love_(novel), http://dbpedia.org/resource/Now_and_Forever_(novel), http://dbpedia.org/resource/One_Day_at_a_Time_(novel), http://dbpedia.org/resource/Passion's_Promise, http://dbpedia.org/resource/Ransom_(Steel_novel), http://dbpedia.org/resource/Rogue_(novel), http://dbpedia.org/resource/Safe_Harbour_(novel), http://dbpedia.org/resource/Season_of_Passion, http://dbpedia.org/resource/Second_Chance_(Steel_novel), http://dbpedia.org/resource/Silent_Honor, http://dbpedia.org/resource/Sisters_(Steel_novel), http://dbpedia.org/resource/Southern_Lights_(novel), http://dbpedia.org/resource/Special_Delivery_(novel), http://dbpedia.org/resource/Sunset_in_St._Tropez, http://dbpedia.org/resource/The_Gift_(Steel_novel), http://dbpedia.org/resource/The_House_(novel), http://dbpedia.org/resource/Thurston_House_(novel), http://dbpedia.org/resource/Wanderlust_(Steel_novel), http://dbpedia.org/resource/Zoya_(novel)] [INFO] Predicted Answer: [http://dbpedia.org/resource/No_Greater_Love_(1996_film), http://dbpedia.org/resource/Once_in_a_Lifetime_(1994_film), http://dbpedia.org/resource/Danielle_Steel's_A_Perfect_Stranger, http://dbpedia.org/resource/Daddy_(1991_film), http://dbpedia.org/resource/Fine_Things_(film), http://dbpedia.org/resource/Heartbeat_(1993_film), http://dbpedia.org/resource/Mixed_Blessings_(film), http://dbpedia.org/resource/Safe_Harbour_(film), http://dbpedia.org/resource/Vanished_(1995_film), http://dbpedia.org/resource/Changes_(1991_film), http://dbpedia.org/resource/Danielle_Steel's_Star, http://dbpedia.org/resource/Kaleidoscope_(1990_film), http://dbpedia.org/resource/Secrets_(1992_American_film)] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 80, P: 0.331, R: 0.367, macro F1: 0.329, macro F1*: 0.348 -[INFO] QALD Cumulative metrics, sample: 80, P: 0.543, R: 0.367, macro F1: 0.329, QALD macro F1: 0.438 -[INFO] Current time: 1491 ms, average: 3147.28 ms/ques, entity detection: 677.38 ms/ques (linking tool: 590.30 ms/ques), relation detection: 991.06 ms/ques (similarity: 574.99 ms/ques), EDG generation: 68.14 ms/ques +[INFO] Cumulative metrics, sample: 80, P: 0.331, R: 0.367, macro F1: 0.348 +[INFO] QALD Cumulative metrics, sample: 80, P: 0.543, R: 0.367, average F1: 0.329, QALD macro F1: 0.438 +[INFO] Current question consumed time: 1355 ms, average: 2943.55 ms/ques, entity detection: 618.42 ms/ques (linking tool: 540.70 ms/ques), relation detection: 949.10 ms/ques (similarity: 577.88 ms/ques), EDG generation: 68.58 ms/ques [INFO] Entity Linking P: 0.2625, R: 0.2625, micro F1: 0.2625, macro F1: 0.2625 [INFO] Relation Linking P: 0.2875, R: 0.2875, micro F1: 0.2875, macro F1: 0.2875 [INFO] Type Linking P: 0.925, R: 0.925, micro F1: 0.925, macro F1: 0.925 @@ -1207,9 +1207,9 @@ Question 80: Who was influenced by Socrates? [INFO] Golden Answer: [http://dbpedia.org/resource/Aristotle, http://dbpedia.org/resource/Diogenes_of_Sinope, http://dbpedia.org/resource/Edmund_Burke, http://dbpedia.org/resource/Georg_Wilhelm_Friedrich_Hegel, http://dbpedia.org/resource/Karl_Popper, http://dbpedia.org/resource/Lucian, http://dbpedia.org/resource/Peter_Kreeft, http://dbpedia.org/resource/Plato, http://dbpedia.org/resource/Pyrrho, http://dbpedia.org/resource/Søren_Kierkegaard, http://dbpedia.org/resource/Hannah_Arendt, http://dbpedia.org/resource/Niccolò_Machiavelli, http://dbpedia.org/resource/Antisthenes, http://dbpedia.org/resource/Aristippus, http://dbpedia.org/resource/Epictetus, http://dbpedia.org/resource/Euclid_of_Megara, http://dbpedia.org/resource/Leo_Strauss, http://dbpedia.org/resource/Allan_Bloom, http://dbpedia.org/resource/Seyhan_Kurt, http://dbpedia.org/resource/Traian_Brăileanu, http://dbpedia.org/resource/Yaşar_Nuri_Öztürk, http://dbpedia.org/resource/John_Henry_Newman__Western_philosophy__1] [INFO] Predicted Answer: [http://dbpedia.org/resource/Aristotle, http://dbpedia.org/resource/Diogenes_of_Sinope, http://dbpedia.org/resource/Edmund_Burke, http://dbpedia.org/resource/Georg_Wilhelm_Friedrich_Hegel, http://dbpedia.org/resource/Karl_Popper, http://dbpedia.org/resource/Lucian, http://dbpedia.org/resource/Peter_Kreeft, http://dbpedia.org/resource/Plato, http://dbpedia.org/resource/Pyrrho, http://dbpedia.org/resource/Søren_Kierkegaard, http://dbpedia.org/resource/Hannah_Arendt, http://dbpedia.org/resource/Niccolò_Machiavelli, http://dbpedia.org/resource/Antisthenes, http://dbpedia.org/resource/Aristippus, http://dbpedia.org/resource/Epictetus, http://dbpedia.org/resource/Euclid_of_Megara, http://dbpedia.org/resource/Leo_Strauss, http://dbpedia.org/resource/Allan_Bloom, http://dbpedia.org/resource/Seyhan_Kurt, http://dbpedia.org/resource/Traian_Brăileanu, http://dbpedia.org/resource/Yaşar_Nuri_Öztürk, http://dbpedia.org/resource/John_Henry_Newman__Western_philosophy__1, http://dbpedia.org/resource/Anaxagoras, http://dbpedia.org/resource/Archelaus_(philosopher), http://dbpedia.org/resource/Prodicus] [INFO] P: 0.88, R: 1.0, F: 0.9361702127659575 -[INFO] Cumulative metrics, sample: 81, P: 0.338, R: 0.375, macro F1: 0.336, macro F1*: 0.355 -[INFO] QALD Cumulative metrics, sample: 81, P: 0.548, R: 0.375, macro F1: 0.336, QALD macro F1: 0.445 -[INFO] Current time: 1471 ms, average: 3126.58 ms/ques, entity detection: 672.90 ms/ques (linking tool: 585.41 ms/ques), relation detection: 985.38 ms/ques (similarity: 571.43 ms/ques), EDG generation: 67.38 ms/ques +[INFO] Cumulative metrics, sample: 81, P: 0.338, R: 0.375, macro F1: 0.355 +[INFO] QALD Cumulative metrics, sample: 81, P: 0.548, R: 0.375, average F1: 0.336, QALD macro F1: 0.445 +[INFO] Current question consumed time: 1294 ms, average: 2923.19 ms/ques, entity detection: 614.22 ms/ques (linking tool: 536.19 ms/ques), relation detection: 943.17 ms/ques (similarity: 574.30 ms/ques), EDG generation: 67.81 ms/ques [INFO] Entity Linking P: 0.25925925925925924, R: 0.25925925925925924, micro F1: 0.25925925925925924, macro F1: 0.25925925925925924 [INFO] Relation Linking P: 0.2839506172839506, R: 0.2839506172839506, micro F1: 0.2839506172839506, macro F1: 0.2839506172839506 [INFO] Type Linking P: 0.9259259259259259, R: 0.9259259259259259, micro F1: 0.9259259259259259, macro F1: 0.9259259259259259 @@ -1222,9 +1222,9 @@ Question 81: How deep is Lake Chiemsee? [INFO] Golden Answer: [72.7] [INFO] Predicted Answer: [] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 82, P: 0.334, R: 0.370, macro F1: 0.332, macro F1*: 0.351 -[INFO] QALD Cumulative metrics, sample: 82, P: 0.553, R: 0.370, macro F1: 0.332, QALD macro F1: 0.444 -[INFO] Current time: 1398 ms, average: 3105.50 ms/ques, entity detection: 667.65 ms/ques (linking tool: 580.30 ms/ques), relation detection: 979.50 ms/ques (similarity: 567.23 ms/ques), EDG generation: 66.66 ms/ques +[INFO] Cumulative metrics, sample: 82, P: 0.334, R: 0.370, macro F1: 0.351 +[INFO] QALD Cumulative metrics, sample: 82, P: 0.553, R: 0.370, average F1: 0.332, QALD macro F1: 0.444 +[INFO] Current question consumed time: 1244 ms, average: 2902.71 ms/ques, entity detection: 609.45 ms/ques (linking tool: 531.52 ms/ques), relation detection: 937.06 ms/ques (similarity: 570.00 ms/ques), EDG generation: 67.26 ms/ques [INFO] Entity Linking P: 0.2682926829268293, R: 0.2682926829268293, micro F1: 0.2682926829268293, macro F1: 0.2682926829268293 [INFO] Relation Linking P: 0.2926829268292683, R: 0.2926829268292683, micro F1: 0.2926829268292683, macro F1: 0.2926829268292683 [INFO] Type Linking P: 0.926829268292683, R: 0.926829268292683, micro F1: 0.926829268292683, macro F1: 0.926829268292683 @@ -1237,9 +1237,9 @@ Question 82: Which companies work in the aerospace industry as well as in medici [INFO] Golden Answer: [http://dbpedia.org/resource/Radiall, http://dbpedia.org/resource/ATI_425_Titanium_Alloy] [INFO] Predicted Answer: [http://dbpedia.org/resource/3i, http://dbpedia.org/resource/3i_Infrastructure, http://dbpedia.org/resource/4_Pines_Brewing_Company, http://dbpedia.org/resource/AEA_Technology, http://dbpedia.org/resource/AEG, http://dbpedia.org/resource/Abita_Brewing_Company, http://dbpedia.org/resource/Adtranz, http://dbpedia.org/resource/Aeritalia, http://dbpedia.org/resource/Aggregate_Industries, http://dbpedia.org/resource/AirBaltic, http://dbpedia.org/resource/Air_Europa, http://dbpedia.org/resource/Air_Liberté, http://dbpedia.org/resource/Air_Mauritius, http://dbpedia.org/resource/Airways_Flying_Club, http://dbpedia.org/resource/Al-Naser_Airlines, http://dbpedia.org/resource/Alembic_Inc, http://dbpedia.org/resource/Alliance_Française, http://dbpedia.org/resource/Alstom, http://dbpedia.org/resource/American_Arithmometer_Company, http://dbpedia.org/resource/American_Automobile_Association, http://dbpedia.org/resource/American_Locomotive_Company, http://dbpedia.org/resource/American_Media_(publisher), http://dbpedia.org/resource/American_Megatrends, http://dbpedia.org/resource/Ampeg, http://dbpedia.org/resource/Amstel_Brewery, http://dbpedia.org/resource/Angel_Trains, http://dbpedia.org/resource/Anglia_Railways, http://dbpedia.org/resource/Animax, http://dbpedia.org/resource/Ansett_Australia, http://dbpedia.org/resource/App_Store, http://dbpedia.org/resource/Apple_Bank_for_Savings, http://dbpedia.org/resource/Arctic_Cat, http://dbpedia.org/resource/Arena_(swimwear), http://dbpedia.org/resource/Arri, http://dbpedia.org/resource/Arts_&_Crafts_México, http://dbpedia.org/resource/Asiana_Airlines, http://dbpedia.org/resource/Assicurazioni_Generali, http://dbpedia.org/resource/Association_of_Train_Operating_Companies, http://dbpedia.org/resource/AstroTurf, http://dbpedia.org/resource/Audiovox, http://dbpedia.org/resource/Autograph_Records, http://dbpedia.org/resource/BA_CityFlyer, http://dbpedia.org/resource/BA_Connect, http://dbpedia.org/resource/BBC_Northern_Ireland, http://dbpedia.org/resource/BBC_Radio, http://dbpedia.org/resource/BBC_Switch, http://dbpedia.org/resource/BBC_Television, http://dbpedia.org/resource/BBC_Wales, http://dbpedia.org/resource/BBC_Weather, http://dbpedia.org/resource/BBN_Technologies, http://dbpedia.org/resource/BCE_Inc., http://dbpedia.org/resource/BC_Card, http://dbpedia.org/resource/BHP_Billiton, http://dbpedia.org/resource/BT_Business, http://dbpedia.org/resource/BT_Consumer, http://dbpedia.org/resource/BT_Global_Services, http://dbpedia.org/resource/BT_Italia, http://dbpedia.org/resource/BT_Mobile, http://dbpedia.org/resource/BT_MyDonate, http://dbpedia.org/resource/BT_TV, http://dbpedia.org/resource/BT_Wholesale, http://dbpedia.org/resource/BT_Wi-fi, http://dbpedia.org/resource/Bajaj_Auto, http://dbpedia.org/resource/Balch_&_Bingham, http://dbpedia.org/resource/Banca_CR_Firenze, http://dbpedia.org/resource/Banca_Carige, http://dbpedia.org/resource/Banca_Carim, http://dbpedia.org/resource/Banca_Carime, http://dbpedia.org/resource/Banca_Caripe, http://dbpedia.org/resource/Banca_Cassa_di_Risparmio_di_Savigliano, http://dbpedia.org/resource/Banca_Monte_dei_Paschi_di_Siena, http://dbpedia.org/resource/Banca_Nazionale_del_Lavoro, http://dbpedia.org/resource/Banca_Popolare_dell'Emilia_Romagna, http://dbpedia.org/resource/Banca_Popolare_di_Bari, http://dbpedia.org/resource/Banca_Popolare_di_Milano, http://dbpedia.org/resource/Banca_Popolare_di_Vicenza, http://dbpedia.org/resource/Banca_Regionale_Europea, http://dbpedia.org/resource/Banca_Tercas, http://dbpedia.org/resource/Banca_del_Monte_di_Lucca, http://dbpedia.org/resource/Banca_dell'Adriatico, http://dbpedia.org/resource/Banca_di_Credito_Cooperativo_di_Roma, http://dbpedia.org/resource/Banco_Popolare, http://dbpedia.org/resource/Bandai_Visual, http://dbpedia.org/resource/Bangkok_Airways, http://dbpedia.org/resource/Bank_of_America, http://dbpedia.org/resource/Barrick_Gold, http://dbpedia.org/resource/Base_(company), http://dbpedia.org/resource/Base_CRM, http://dbpedia.org/resource/Base_FX, http://dbpedia.org/resource/BassLab, http://dbpedia.org/resource/Bear_Family_Records, http://dbpedia.org/resource/Beaver_Lumber, http://dbpedia.org/resource/Bee-Line_Bus_System, http://dbpedia.org/resource/Beez_Entertainment, http://dbpedia.org/resource/Bella_Center, http://dbpedia.org/resource/Benchmark_(venture_capital_firm), http://dbpedia.org/resource/Benchmark_Electronics, http://dbpedia.org/resource/Benchmark_Recordings, http://dbpedia.org/resource/Big_Finish_Productions, http://dbpedia.org/resource/Billion_(company), http://dbpedia.org/resource/Binglin_Stable, http://dbpedia.org/resource/Biograph_Records, http://dbpedia.org/resource/Birra_Moretti, http://dbpedia.org/resource/Biscúter, http://dbpedia.org/resource/Bitter_Cars, http://dbpedia.org/resource/Black_Swan_Records, http://dbpedia.org/resource/Blitz_(software), http://dbpedia.org/resource/Blitz_Games, http://dbpedia.org/resource/Blockbuster_LLC, http://dbpedia.org/resource/Bluesville_Records, http://dbpedia.org/resource/Bob_Albrecht, http://dbpedia.org/resource/Boeing_Airplane_Company, http://dbpedia.org/resource/Boeing_Commercial_Airplanes, http://dbpedia.org/resource/Boeing_Defense,_Space_&_Security, http://dbpedia.org/resource/Boeing_Integrated_Defense_Systems, http://dbpedia.org/resource/Boeing_Rotorcraft_Systems, http://dbpedia.org/resource/Bombardier_Recreational_Products, http://dbpedia.org/resource/Bostik, http://dbpedia.org/resource/Boulevard_Brewing_Company, http://dbpedia.org/resource/Brasserie_d'Achouffe, http://dbpedia.org/resource/Brasseries_du_Cameroun, http://dbpedia.org/resource/Breitling_SA, http://dbpedia.org/resource/Britannia_Airways, http://dbpedia.org/resource/British_Airways_Helicopters, http://dbpedia.org/resource/British_Airways_Ltd_(2012–15), http://dbpedia.org/resource/British_Asia_Airways, http://dbpedia.org/resource/British_Caledonian, http://dbpedia.org/resource/British_Council, http://dbpedia.org/resource/British_Midland_International, http://dbpedia.org/resource/British_Rail_Engineering_Limited, http://dbpedia.org/resource/British_Rail_Research_Division, http://dbpedia.org/resource/British_Railways, http://dbpedia.org/resource/British_Railways_Board, http://dbpedia.org/resource/British_Transport_Commission, http://dbpedia.org/resource/British_Transport_Hotels, http://dbpedia.org/resource/Brookfield_Asset_Management, http://dbpedia.org/resource/Brookfield_Office_Properties, http://dbpedia.org/resource/Brother_Records, http://dbpedia.org/resource/Brouwerij_'t_IJ, http://dbpedia.org/resource/Brunswick_Corporation, http://dbpedia.org/resource/Brunswick_Records, http://dbpedia.org/resource/Brymon_Airways, http://dbpedia.org/resource/Buc-ee's, http://dbpedia.org/resource/Buck_Knives, http://dbpedia.org/resource/Budd_Company, http://dbpedia.org/resource/Buddah_Records, http://dbpedia.org/resource/Buffet_Crampon, http://dbpedia.org/resource/Bug_AS, http://dbpedia.org/resource/Bugatti_Automobiles, http://dbpedia.org/resource/BuzzFeed, http://dbpedia.org/resource/CBS_All_Access, http://dbpedia.org/resource/CMC_International, http://dbpedia.org/resource/COMSAT, http://dbpedia.org/resource/CSL_Limited, http://dbpedia.org/resource/Caisse_de_dépôt_et_placement_du_Québec, http://dbpedia.org/resource/Calvin_Klein, http://dbpedia.org/resource/Cambrian_Airways, http://dbpedia.org/resource/Canadian_Bioinformatics_Workshops, http://dbpedia.org/resource/CanalSat, http://dbpedia.org/resource/Candix_Records, http://dbpedia.org/resource/Capital_Brewery, http://dbpedia.org/resource/Cargolux, http://dbpedia.org/resource/Caribou_Records, http://dbpedia.org/resource/Cariparma, http://dbpedia.org/resource/Carispezia, http://dbpedia.org/resource/Carlton_Video, http://dbpedia.org/resource/Cassa_dei_Risparmi_di_Forlì_e_della_Romagna, http://dbpedia.org/resource/Cassa_di_Risparmio_del_Friuli_Venezia_Giulia, http://dbpedia.org/resource/Cassa_di_Risparmio_del_Veneto, http://dbpedia.org/resource/Cassa_di_Risparmio_della_Repubblica_di_San_Marino, http://dbpedia.org/resource/Cassa_di_Risparmio_di_Asti, http://dbpedia.org/resource/Cassa_di_Risparmio_di_Biella_e_Vercelli, http://dbpedia.org/resource/Cassa_di_Risparmio_di_Cento, http://dbpedia.org/resource/Cassa_di_Risparmio_di_Cesena, http://dbpedia.org/resource/Cassa_di_Risparmio_di_Fermo, http://dbpedia.org/resource/Cassa_di_Risparmio_di_Fossano, http://dbpedia.org/resource/Cassa_di_Risparmio_di_Loreto, http://dbpedia.org/resource/Cassa_di_Risparmio_di_Orvieto, http://dbpedia.org/resource/Cassa_di_Risparmio_di_Pistoia_e_della_Lucchesia, http://dbpedia.org/resource/Cassa_di_Risparmio_di_Ravenna, http://dbpedia.org/resource/Cassa_di_Risparmio_di_Saluzzo, http://dbpedia.org/resource/Cassa_di_Risparmio_di_San_Miniato, http://dbpedia.org/resource/Cassa_di_Risparmio_di_Volterra, http://dbpedia.org/resource/Cassa_di_Risparmio_in_Bologna, http://dbpedia.org/resource/Casse_di_Risparmio_dell'Umbria, http://dbpedia.org/resource/Cdiscount, http://dbpedia.org/resource/Cengage_Learning, http://dbpedia.org/resource/Central_Independent_Television, http://dbpedia.org/resource/Centrica, http://dbpedia.org/resource/Cessna, http://dbpedia.org/resource/Chaosium, http://dbpedia.org/resource/Chatime, http://dbpedia.org/resource/Chiltern_Railways, http://dbpedia.org/resource/Chimay_Brewery, http://dbpedia.org/resource/Château_Haut-Brion, http://dbpedia.org/resource/Château_d'Yquem, http://dbpedia.org/resource/Citadel_Broadcasting, http://dbpedia.org/resource/City_Sightseeing, http://dbpedia.org/resource/CodeGear, http://dbpedia.org/resource/Colas_Rail, http://dbpedia.org/resource/Collectables_Records, http://dbpedia.org/resource/Comair_(South_Africa), http://dbpedia.org/resource/Commodore_Records, http://dbpedia.org/resource/Commonwealth_Bank, http://dbpedia.org/resource/Community_Newspaper_Company, http://dbpedia.org/resource/Computershare, http://dbpedia.org/resource/Concarril, http://dbpedia.org/resource/Conn-Selmer, http://dbpedia.org/resource/Conrad_Hotels, http://dbpedia.org/resource/Consett_Iron_Company, http://dbpedia.org/resource/Convergent_Technologies_(Unisys), http://dbpedia.org/resource/Coral_Records, http://dbpedia.org/resource/Corel, http://dbpedia.org/resource/Counterpoint_(publisher), http://dbpedia.org/resource/Coursera, http://dbpedia.org/resource/Cowboy_Records, http://dbpedia.org/resource/Credit_Suisse, http://dbpedia.org/resource/Credito_Valtellinese, http://dbpedia.org/resource/Creemore_Springs, http://dbpedia.org/resource/Crown_Estate, http://dbpedia.org/resource/Crédit_Agricole, http://dbpedia.org/resource/Cumulus_Media, http://dbpedia.org/resource/Curtiss-Wright, http://dbpedia.org/resource/D._C._Thomson_&_Co., http://dbpedia.org/resource/DB_Cargo_UK, http://dbpedia.org/resource/DC_Entertainment, http://dbpedia.org/resource/DC_Thomson, http://dbpedia.org/resource/DSW,_Inc., http://dbpedia.org/resource/DStv, http://dbpedia.org/resource/Dabs.com, http://dbpedia.org/resource/Dan-Air, http://dbpedia.org/resource/Dandelion_Records, http://dbpedia.org/resource/Danelectro, http://dbpedia.org/resource/Dassault_Group, http://dbpedia.org/resource/De_Havilland_Canada, http://dbpedia.org/resource/De_La_Rue, http://dbpedia.org/resource/De_Laurentiis_Entertainment_Group, http://dbpedia.org/resource/Debt_Management_Office, http://dbpedia.org/resource/Derbi, http://dbpedia.org/resource/Deseret_Book, http://dbpedia.org/resource/Dharma_Productions, http://dbpedia.org/resource/Diamond_Comic_Distributors, http://dbpedia.org/resource/Digit-Alb, http://dbpedia.org/resource/Dish_Network, http://dbpedia.org/resource/Document_Records, http://dbpedia.org/resource/Doing_business_as, http://dbpedia.org/resource/Dorman_Long, http://dbpedia.org/resource/Dot_Records, http://dbpedia.org/resource/Douglas_Comer, http://dbpedia.org/resource/Dublin_Bus, http://dbpedia.org/resource/Dupont_Brewery, http://dbpedia.org/resource/E.ON, http://dbpedia.org/resource/EADS, http://dbpedia.org/resource/ESPN_Inc., http://dbpedia.org/resource/ESP_Guitars, http://dbpedia.org/resource/East_Coast_(train_operating_company), http://dbpedia.org/resource/Einbecker_Brewery, http://dbpedia.org/resource/Electric_Honey_(label), http://dbpedia.org/resource/Elektra_Records, http://dbpedia.org/resource/Elmer's_Products,_Inc., http://dbpedia.org/resource/Embarcadero_Technologies, http://dbpedia.org/resource/Entercom, http://dbpedia.org/resource/Era_Records, http://dbpedia.org/resource/Eros_International, http://dbpedia.org/resource/Eurosport, http://dbpedia.org/resource/Eurostar_International_Limited, http://dbpedia.org/resource/Evergreen_Group, http://dbpedia.org/resource/Eversholt_Rail_Group, http://dbpedia.org/resource/Faber-Castell, http://dbpedia.org/resource/Fairway_Market, http://dbpedia.org/resource/Fenway_Sports_Group, http://dbpedia.org/resource/Ferrocarrils_de_la_Generalitat_de_Catalunya, http://dbpedia.org/resource/FetLife, http://dbpedia.org/resource/Fidelity_Investments, http://dbpedia.org/resource/Fifth_Avenue_Coach_Company, http://dbpedia.org/resource/Film_Four, http://dbpedia.org/resource/First_Niagara_Bank, http://dbpedia.org/resource/Fodera, http://dbpedia.org/resource/Fort_Garry_Brewing_Company, http://dbpedia.org/resource/Foursquare, http://dbpedia.org/resource/Fox_Television_Stations, http://dbpedia.org/resource/Franconia_Brewing_Company, http://dbpedia.org/resource/Franklin_Templeton_Investments, http://dbpedia.org/resource/Frederick_Warne_&_Co, http://dbpedia.org/resource/Free_software_movement, http://dbpedia.org/resource/Freesat, http://dbpedia.org/resource/Freeview_(UK), http://dbpedia.org/resource/Freightliner_Group, http://dbpedia.org/resource/Freshpet, http://dbpedia.org/resource/Fuji_Heavy_Industries, http://dbpedia.org/resource/G._Schneider_&_Sohn, http://dbpedia.org/resource/G4S, http://dbpedia.org/resource/GB_Railfreight, http://dbpedia.org/resource/GE_Aviation, http://dbpedia.org/resource/GKN, http://dbpedia.org/resource/GNU_Project, http://dbpedia.org/resource/GateHouse_Media, http://dbpedia.org/resource/Gate_Gourmet, http://dbpedia.org/resource/Genentech, http://dbpedia.org/resource/General_Magic, http://dbpedia.org/resource/Generalitat_of_Catalonia, http://dbpedia.org/resource/Gigabyte_Technology, http://dbpedia.org/resource/Gimbels, http://dbpedia.org/resource/Girobank, http://dbpedia.org/resource/Global_Crossing, http://dbpedia.org/resource/Global_Language_Monitor, http://dbpedia.org/resource/Global_Supply_Systems, http://dbpedia.org/resource/Globosat, http://dbpedia.org/resource/Go_(airline), http://dbpedia.org/resource/Gone_Records, http://dbpedia.org/resource/Government_of_Spain, http://dbpedia.org/resource/Grand_Central_Railway, http://dbpedia.org/resource/Great_Basin_Brewing_Company, http://dbpedia.org/resource/Great_Lakes_Brewing_Company, http://dbpedia.org/resource/Grosset_&_Dunlap, http://dbpedia.org/resource/H&M, http://dbpedia.org/resource/Hallmark_Channel, http://dbpedia.org/resource/Hammer_Films, http://dbpedia.org/resource/Harley-Davidson, http://dbpedia.org/resource/Harmony_Company, http://dbpedia.org/resource/Harry_Needle_Railroad_Company, http://dbpedia.org/resource/Hawk_Films, http://dbpedia.org/resource/Hawker_Beechcraft, http://dbpedia.org/resource/Hawker_Siddeley_Canada, http://dbpedia.org/resource/Hecht-Lancaster_&_Buzzell_Music, http://dbpedia.org/resource/Henkel, http://dbpedia.org/resource/Hillman, http://dbpedia.org/resource/Hispano-Suiza, http://dbpedia.org/resource/Hitwise, http://dbpedia.org/resource/Home_Box_Office, http://dbpedia.org/resource/Hot_(Israel), http://dbpedia.org/resource/Hull_Trains, http://dbpedia.org/resource/Huntingdon_Life_Sciences, http://dbpedia.org/resource/Hutchinson_(publisher), http://dbpedia.org/resource/Höfner, http://dbpedia.org/resource/I.R.S._Records, http://dbpedia.org/resource/IAG_Cargo, http://dbpedia.org/resource/IBM_Corporation, http://dbpedia.org/resource/ICM_Partners, http://dbpedia.org/resource/IMAX, http://dbpedia.org/resource/ITV_(TV_Network), http://dbpedia.org/resource/ITV_Digital, http://dbpedia.org/resource/Ibanez, http://dbpedia.org/resource/Icon_Productions, http://dbpedia.org/resource/Impala, http://dbpedia.org/resource/Imperial_Chemical_Industries, http://dbpedia.org/resource/Imperial_Porcelain_Factory, http://dbpedia.org/resource/Imperial_Theatre, http://dbpedia.org/resource/Incorporation_(business), http://dbpedia.org/resource/Infobase_Publishing, http://dbpedia.org/resource/Insurance_Company_of_North_America, http://dbpedia.org/resource/Insyde_Software, http://dbpedia.org/resource/Inter-Varsity_Press, http://dbpedia.org/resource/Intesa_Sanpaolo, http://dbpedia.org/resource/Inventel, http://dbpedia.org/resource/Iran_Air, http://dbpedia.org/resource/ItalDesign, http://dbpedia.org/resource/Iveco_Bus, http://dbpedia.org/resource/J._G._Brill_and_Company, http://dbpedia.org/resource/JAM_Creative_Productions, http://dbpedia.org/resource/JSP_Records, http://dbpedia.org/resource/Jay_Cashman, http://dbpedia.org/resource/Jeepster_Records, http://dbpedia.org/resource/Johnny_&_Associates, http://dbpedia.org/resource/KCOM_Group, http://dbpedia.org/resource/KDE, http://dbpedia.org/resource/Kawasaki_Heavy_Industries, http://dbpedia.org/resource/Kay_Musical_Instrument_Company, http://dbpedia.org/resource/Ken_Thompson_(computer_programmer), http://dbpedia.org/resource/Kenya_Wildlife_Service, http://dbpedia.org/resource/Keolis, http://dbpedia.org/resource/King_Features_Syndicate, http://dbpedia.org/resource/Kink.com, http://dbpedia.org/resource/Kleiner_Perkins_Caufield_&_Byers, http://dbpedia.org/resource/Knickerbocker_Club, http://dbpedia.org/resource/Kodansha, http://dbpedia.org/resource/Koninklijke_Hoogovens, http://dbpedia.org/resource/LWT, http://dbpedia.org/resource/Lambie-Nairn, http://dbpedia.org/resource/Lancôme, http://dbpedia.org/resource/Landor_Associates, http://dbpedia.org/resource/Learjet, http://dbpedia.org/resource/Leslie_E._Robertson_Associates, http://dbpedia.org/resource/Lexicon_Branding, http://dbpedia.org/resource/Line_6_(company), http://dbpedia.org/resource/Linux_Foundation, http://dbpedia.org/resource/Lithuanian_National_Radio_and_Television, http://dbpedia.org/resource/Liverpool_(store), http://dbpedia.org/resource/Loganair, http://dbpedia.org/resource/London,_Midland_and_Scottish_Railway, http://dbpedia.org/resource/London_General_Omnibus_Company, http://dbpedia.org/resource/London_Records, http://dbpedia.org/resource/M&T_Bank, http://dbpedia.org/resource/MCA_Records, http://dbpedia.org/resource/MCI_Communications, http://dbpedia.org/resource/MEO_(Portugal), http://dbpedia.org/resource/MTR_Corporation, http://dbpedia.org/resource/Macmillan_Publishers_(United_States), http://dbpedia.org/resource/Madman_Entertainment, http://dbpedia.org/resource/Man_Group, http://dbpedia.org/resource/Marie_Brizard_et_Roger_International, http://dbpedia.org/resource/Marks_&_Spencer, http://dbpedia.org/resource/MassDOT, http://dbpedia.org/resource/Massey_Hall, http://dbpedia.org/resource/Matador_Records, http://dbpedia.org/resource/McKesson_Corporation, http://dbpedia.org/resource/Mediaset_Premium, http://dbpedia.org/resource/Medibank, http://dbpedia.org/resource/Melodeon_Records, http://dbpedia.org/resource/Memorex, http://dbpedia.org/resource/Mental_Floss, http://dbpedia.org/resource/Mercure_de_France, http://dbpedia.org/resource/Mercury_Communications, http://dbpedia.org/resource/Merrill_Lynch, http://dbpedia.org/resource/Merseyrail, http://dbpedia.org/resource/Merseyside_Passenger_Transport_Executive, http://dbpedia.org/resource/Merseytravel, http://dbpedia.org/resource/Messier-Bugatti-Dowty, http://dbpedia.org/resource/Micro_Focus, http://dbpedia.org/resource/Micro_Focus_International, http://dbpedia.org/resource/Middle_East_Broadcasting_Center, http://dbpedia.org/resource/Mines_Advisory_Group, http://dbpedia.org/resource/Ministry_of_Public_Enlightenment_and_Propaganda, http://dbpedia.org/resource/Minute_Maid, http://dbpedia.org/resource/Mitsubishi_Fuso_Truck_and_Bus_Corporation, http://dbpedia.org/resource/Molinos_Río_de_la_Plata, http://dbpedia.org/resource/Molson_Brewery, http://dbpedia.org/resource/Momentive_Specialty_Chemicals, http://dbpedia.org/resource/Monte_Davidoff, http://dbpedia.org/resource/Monteith's, http://dbpedia.org/resource/Montreal_Locomotive_Works, http://dbpedia.org/resource/Mowlem, http://dbpedia.org/resource/Mozilla, http://dbpedia.org/resource/Mozilla_Foundation, http://dbpedia.org/resource/Music_Man_(company), http://dbpedia.org/resource/NCR_Corporation, http://dbpedia.org/resource/National_Australia_Bank, http://dbpedia.org/resource/National_Center_for_Biotechnology_Information, http://dbpedia.org/resource/National_Pescara, http://dbpedia.org/resource/National_Savings_and_Investments, http://dbpedia.org/resource/National_Telephone_Company, http://dbpedia.org/resource/National_Weather_Service, http://dbpedia.org/resource/Natural_Balance_Pet_Foods, http://dbpedia.org/resource/Nerdist_Industries, http://dbpedia.org/resource/Network_Rail, http://dbpedia.org/resource/New_Belgium_Brewing_Company, http://dbpedia.org/resource/New_Era_Cap_Company, http://dbpedia.org/resource/New_Holland_Publishers, http://dbpedia.org/resource/New_York_Mercantile_Exchange, http://dbpedia.org/resource/New_York_Power_Authority, http://dbpedia.org/resource/Nielsen_Media_Research, http://dbpedia.org/resource/Nissan_Motors, http://dbpedia.org/resource/Nomura_Group, http://dbpedia.org/resource/Nordic_Games, http://dbpedia.org/resource/Northeast_Airlines_(UK), http://dbpedia.org/resource/Northrop_Corporation, http://dbpedia.org/resource/Nova_Science_Publishers, http://dbpedia.org/resource/Npower_(UK), http://dbpedia.org/resource/Nuova_Banca_delle_Marche, http://dbpedia.org/resource/Nuova_Cassa_di_Risparmio_di_Chieti, http://dbpedia.org/resource/Nuova_Cassa_di_Risparmio_di_Ferrara, http://dbpedia.org/resource/ORF_(broadcaster), http://dbpedia.org/resource/Obayashi_Corporation, http://dbpedia.org/resource/Odeon_Cinemas, http://dbpedia.org/resource/Office_of_Rail_and_Road, http://dbpedia.org/resource/Oil_and_Gas_Development_Company, http://dbpedia.org/resource/Olfa, http://dbpedia.org/resource/Olympia_and_York, http://dbpedia.org/resource/Ontario_Power_Generation, http://dbpedia.org/resource/OpenSkies, http://dbpedia.org/resource/OpenTV, http://dbpedia.org/resource/Open_Court_Publishing_Company, http://dbpedia.org/resource/Openreach, http://dbpedia.org/resource/Orval_Brewery, http://dbpedia.org/resource/Oxbow_(surfwear), http://dbpedia.org/resource/PNY_Technologies, http://dbpedia.org/resource/PT_Kereta_Api, http://dbpedia.org/resource/Panavision, http://dbpedia.org/resource/Pantheon_Books, http://dbpedia.org/resource/Paper_Mate, http://dbpedia.org/resource/Park_Gate_Iron_and_Steel_Company, http://dbpedia.org/resource/Parsons_Brinckerhoff, http://dbpedia.org/resource/Paul_Scherrer_Institute, http://dbpedia.org/resource/Peermusic, http://dbpedia.org/resource/Pegaso, http://dbpedia.org/resource/Pendleton_Woolen_Mills, http://dbpedia.org/resource/Penn_Brewery, http://dbpedia.org/resource/People's_Computer_Company, http://dbpedia.org/resource/Perini_Building_Company, http://dbpedia.org/resource/Pernod_Ricard, http://dbpedia.org/resource/Phantom_Films, http://dbpedia.org/resource/Pharsalia_Technologies, http://dbpedia.org/resource/Phelon_&_Moore, http://dbpedia.org/resource/Philco, http://dbpedia.org/resource/Philips_Records, http://dbpedia.org/resource/Phorm, http://dbpedia.org/resource/Pioneer_Corporation, http://dbpedia.org/resource/Plusnet, http://dbpedia.org/resource/Polaris_Industries, http://dbpedia.org/resource/Porterbrook, http://dbpedia.org/resource/Post_Office_Research_Station, http://dbpedia.org/resource/Pratt_&_Whitney, http://dbpedia.org/resource/Pritt, http://dbpedia.org/resource/Project_MAC, http://dbpedia.org/resource/PublicAffairs, http://dbpedia.org/resource/Pullman_Company, http://dbpedia.org/resource/RBC_Capital_Markets, http://dbpedia.org/resource/RCA, http://dbpedia.org/resource/RTBF, http://dbpedia.org/resource/RTL_Group, http://dbpedia.org/resource/Racal, http://dbpedia.org/resource/Rail_Europe,_Inc., http://dbpedia.org/resource/Railfreight_Distribution, http://dbpedia.org/resource/Railtrack, http://dbpedia.org/resource/Rajshri_Productions, http://dbpedia.org/resource/Ranker, http://dbpedia.org/resource/Rational_Software, http://dbpedia.org/resource/Raytheon_Aircraft_Company, http://dbpedia.org/resource/Reader's_Digest_Association, http://dbpedia.org/resource/Real_property, http://dbpedia.org/resource/Red_Barrels, http://dbpedia.org/resource/Red_Bee_Media, http://dbpedia.org/resource/Red_Hat, http://dbpedia.org/resource/Red_Star_Parcels, http://dbpedia.org/resource/Regal_Entertainment_Group, http://dbpedia.org/resource/Regal_Records_(1949), http://dbpedia.org/resource/Reliance_Entertainment, http://dbpedia.org/resource/Rhino_Entertainment, http://dbpedia.org/resource/Rhythm_Zone, http://dbpedia.org/resource/Ribbit_(telecommunications_company), http://dbpedia.org/resource/Rich_Products, http://dbpedia.org/resource/Rickenbacker, http://dbpedia.org/resource/Riverside_Records, http://dbpedia.org/resource/Riviera_Trains, http://dbpedia.org/resource/Rocksteady_Studios, http://dbpedia.org/resource/Rogue_Ales, http://dbpedia.org/resource/Romano_Artioli, http://dbpedia.org/resource/Roots_Canada, http://dbpedia.org/resource/Roski, http://dbpedia.org/resource/Rough_Trade_Records, http://dbpedia.org/resource/Roulette_Records, http://dbpedia.org/resource/Rover_Company, http://dbpedia.org/resource/Rowman_&_Littlefield, http://dbpedia.org/resource/Royal_Jordanian, http://dbpedia.org/resource/Royal_Society_for_the_Protection_of_Birds, http://dbpedia.org/resource/Ruby_Tuesday_(restaurant), http://dbpedia.org/resource/S7_Airlines, http://dbpedia.org/resource/SEAT, http://dbpedia.org/resource/SKF, http://dbpedia.org/resource/SKY_México, http://dbpedia.org/resource/ST_Aerospace, http://dbpedia.org/resource/Saatchi_&_Saatchi, http://dbpedia.org/resource/Salem_Red_Sox, http://dbpedia.org/resource/SanDisk, http://dbpedia.org/resource/Sanctuary_Records_Group, http://dbpedia.org/resource/Sanofi_Pasteur, http://dbpedia.org/resource/Sara_Lee_Corporation, http://dbpedia.org/resource/Schecter_Guitar_Research, http://dbpedia.org/resource/Schroders, http://dbpedia.org/resource/Scottish_Government, http://dbpedia.org/resource/Scottish_Region_of_British_Railways, http://dbpedia.org/resource/Sea_of_Tunes, http://dbpedia.org/resource/Seagate_Technology, http://dbpedia.org/resource/Sealink, http://dbpedia.org/resource/Seaspeed, http://dbpedia.org/resource/Segue, http://dbpedia.org/resource/Select_Bus_Service, http://dbpedia.org/resource/Sentai_Filmworks, http://dbpedia.org/resource/Setra, http://dbpedia.org/resource/Shake_Shack, http://dbpedia.org/resource/Short_Brothers, http://dbpedia.org/resource/Signature_Books, http://dbpedia.org/resource/Silver_Line_(MBTA), http://dbpedia.org/resource/Silverjet, http://dbpedia.org/resource/Singha, http://dbpedia.org/resource/Sirius_Canada, http://dbpedia.org/resource/Sirius_Satellite_Radio, http://dbpedia.org/resource/Sirius_XM_Radio, http://dbpedia.org/resource/Skanska, http://dbpedia.org/resource/SkyCable, http://dbpedia.org/resource/Sky_(United_Kingdom), http://dbpedia.org/resource/Sky_Deutschland, http://dbpedia.org/resource/Sky_Italia, http://dbpedia.org/resource/Sky_Television_plc, http://dbpedia.org/resource/Skymark_Airlines, http://dbpedia.org/resource/Skyriver, http://dbpedia.org/resource/Sly_Fox_Brewery, http://dbpedia.org/resource/Smarthistory, http://dbpedia.org/resource/Smithsonian_Folkways, http://dbpedia.org/resource/Smurfit_Kappa_Group, http://dbpedia.org/resource/Snapper_Music, http://dbpedia.org/resource/Society_for_Worldwide_Interbank_Financial_Telecommunication, http://dbpedia.org/resource/Société_Générale, http://dbpedia.org/resource/Software_Arts, http://dbpedia.org/resource/Software_in_the_Public_Interest, http://dbpedia.org/resource/Solectron, http://dbpedia.org/resource/Solid_Gold_(pet_food), http://dbpedia.org/resource/Sony_Music_Entertainment, http://dbpedia.org/resource/South_Yorkshire_Passenger_Transport_Executive, http://dbpedia.org/resource/Southern_Region_of_British_Railways, http://dbpedia.org/resource/Southern_Television, http://dbpedia.org/resource/Spirit_AeroSystems, http://dbpedia.org/resource/Sprint_Corporation, http://dbpedia.org/resource/Stagecoach_Group, http://dbpedia.org/resource/State-owned_enterprise, http://dbpedia.org/resource/Stearman_Aircraft, http://dbpedia.org/resource/Steinberger, http://dbpedia.org/resource/Stone_&_Wood_Brewing_Co., http://dbpedia.org/resource/Strategic_Rail_Authority, http://dbpedia.org/resource/Strathclyde_Partnership_for_Transport, http://dbpedia.org/resource/Stream_TV, http://dbpedia.org/resource/Stutz_Motor_Company, http://dbpedia.org/resource/Suez_Canal_Company, http://dbpedia.org/resource/Sumitomo_Mitsui_Banking_Corporation, http://dbpedia.org/resource/Super_Bock, http://dbpedia.org/resource/Swiss_Federal_Railways, http://dbpedia.org/resource/Swisscom, http://dbpedia.org/resource/Syngenta, http://dbpedia.org/resource/Südtiroler_Sparkasse_–_Cassa_di_Risparmio_di_Bolzano, http://dbpedia.org/resource/TK_Records, http://dbpedia.org/resource/Tachyon_Publications, http://dbpedia.org/resource/Taco_Bell, http://dbpedia.org/resource/TalkTalk_Group, http://dbpedia.org/resource/Talpa_(TV), http://dbpedia.org/resource/Target_Games, http://dbpedia.org/resource/Tartan, http://dbpedia.org/resource/Tastykake, http://dbpedia.org/resource/Tata_Steel, http://dbpedia.org/resource/Tata_Steel_Europe, http://dbpedia.org/resource/Taylor_&_Francis, http://dbpedia.org/resource/Ted's_Montana_Grill, http://dbpedia.org/resource/Telereal_Trillium, http://dbpedia.org/resource/Televisa, http://dbpedia.org/resource/Television_South, http://dbpedia.org/resource/Televisió_de_Catalunya, http://dbpedia.org/resource/Telewizja_Polska, http://dbpedia.org/resource/Textron, http://dbpedia.org/resource/Textron_Aviation, http://dbpedia.org/resource/Thames_&_Hudson, http://dbpedia.org/resource/The_Public_Theater, http://dbpedia.org/resource/The_Royal_Bank_of_Scotland_Group, http://dbpedia.org/resource/Tiger_Aspect_Productions, http://dbpedia.org/resource/Time_Inc., http://dbpedia.org/resource/Time_Warner_Cable, http://dbpedia.org/resource/Timken_Company, http://dbpedia.org/resource/Tobias_(bass_guitar_company), http://dbpedia.org/resource/Todd-AO, http://dbpedia.org/resource/Toei_Company, http://dbpedia.org/resource/Tops_Friendly_Markets, http://dbpedia.org/resource/Townsquare_Media, http://dbpedia.org/resource/Trainload_Freight, http://dbpedia.org/resource/Transbrasil, http://dbpedia.org/resource/Transports_Metropolitans_de_Barcelona, http://dbpedia.org/resource/Travel_Air, http://dbpedia.org/resource/Trolza, http://dbpedia.org/resource/Tröegs_Brewing_Company, http://dbpedia.org/resource/Tudou, http://dbpedia.org/resource/Twentieth_Century_Fox, http://dbpedia.org/resource/Tyan, http://dbpedia.org/resource/Tyne_and_Wear_Passenger_Transport_Executive, http://dbpedia.org/resource/Tzadik_Records, http://dbpedia.org/resource/UBI_Banca, http://dbpedia.org/resource/UK_Rail_Leasing, http://dbpedia.org/resource/Unified_EFI_Forum, http://dbpedia.org/resource/Union_Carbide, http://dbpedia.org/resource/UnipolSai, http://dbpedia.org/resource/United_Artists_Records, http://dbpedia.org/resource/United_Steel_Companies, http://dbpedia.org/resource/United_Western_Recorders, http://dbpedia.org/resource/Universal_Pictures, http://dbpedia.org/resource/Universal_Uclick, http://dbpedia.org/resource/University_of_North_Texas_Press, http://dbpedia.org/resource/Urban_Transportation_Development_Corporation, http://dbpedia.org/resource/VMware, http://dbpedia.org/resource/Vaccinogen, http://dbpedia.org/resource/Vanacore_Music, http://dbpedia.org/resource/Veneto_Banca, http://dbpedia.org/resource/Veolia_Water, http://dbpedia.org/resource/Verbatim_Corporation, http://dbpedia.org/resource/Vertigo_Records, http://dbpedia.org/resource/Verve_Records, http://dbpedia.org/resource/Vibroplex, http://dbpedia.org/resource/Victor_Entertainment, http://dbpedia.org/resource/Victory_Brewing_Company, http://dbpedia.org/resource/Viessmann, http://dbpedia.org/resource/Vinod_Chopra_Films, http://dbpedia.org/resource/Virgin_CrossCountry, http://dbpedia.org/resource/Virgin_EMI_Records, http://dbpedia.org/resource/Virgin_Media, http://dbpedia.org/resource/Virgin_Media_Ireland, http://dbpedia.org/resource/Virgin_Trains, http://dbpedia.org/resource/Viz_Media, http://dbpedia.org/resource/Vocalion_Records, http://dbpedia.org/resource/Volga-Dnepr_Airlines, http://dbpedia.org/resource/Volga-Dnepr_Group, http://dbpedia.org/resource/Volkswagen_AG, http://dbpedia.org/resource/Vueling, http://dbpedia.org/resource/WB_Games_Montréal, http://dbpedia.org/resource/WIR_Bank, http://dbpedia.org/resource/War_on_Want, http://dbpedia.org/resource/Warner_Bros._Animation, http://dbpedia.org/resource/Warner_Brothers, http://dbpedia.org/resource/Warwick_(company), http://dbpedia.org/resource/West_Coast_Railway_Company, http://dbpedia.org/resource/West_Yorkshire_Metro, http://dbpedia.org/resource/Western_Region_of_British_Railways, http://dbpedia.org/resource/Westinghouse_Electric_Corporation, http://dbpedia.org/resource/Westmalle_Brewery, http://dbpedia.org/resource/Wienerberger, http://dbpedia.org/resource/Wiley-VCH, http://dbpedia.org/resource/Wiss,_Janney,_Elstner_Associates,_Inc., http://dbpedia.org/resource/Wissotzky_Tea, http://dbpedia.org/resource/Woodhead_Publishing, http://dbpedia.org/resource/World_Music_Network, http://dbpedia.org/resource/World_Scientific, http://dbpedia.org/resource/Wyeth, http://dbpedia.org/resource/X-Acto, http://dbpedia.org/resource/Yamaha_Corporation, http://dbpedia.org/resource/Yash_Raj_Films, http://dbpedia.org/resource/Yes_(Israel), http://dbpedia.org/resource/Yleisradio, http://dbpedia.org/resource/Youku, http://dbpedia.org/resource/Zaha_Hadid_Architects, http://dbpedia.org/resource/Škoda_Auto, http://dbpedia.org/resource/1-800-GOT-JUNK%3F, http://dbpedia.org/resource/3Dconnexion, http://dbpedia.org/resource/A&W_(Canada), http://dbpedia.org/resource/ADK_(company), http://dbpedia.org/resource/ANEK_Lines, http://dbpedia.org/resource/ATO_Records, http://dbpedia.org/resource/A_Buck_or_Two, http://dbpedia.org/resource/Aardman_Animations, http://dbpedia.org/resource/Abbott_Laboratories, http://dbpedia.org/resource/Abu_Dhabi_Investment_Council, http://dbpedia.org/resource/Access_Communications, http://dbpedia.org/resource/Accolade_(game_company), http://dbpedia.org/resource/Ace_Hardware, http://dbpedia.org/resource/Ache_Records, http://dbpedia.org/resource/Actavis, http://dbpedia.org/resource/Adam_Kennedy_(programmer), http://dbpedia.org/resource/Addition_Elle, http://dbpedia.org/resource/Advance_Gold, http://dbpedia.org/resource/Advanced_Cyclotron_Systems, http://dbpedia.org/resource/Aena, http://dbpedia.org/resource/Aeronautical_Development_Establishment, http://dbpedia.org/resource/Aeroports_de_Catalunya, http://dbpedia.org/resource/Affinity_Credit_Union, http://dbpedia.org/resource/Agence_France-Presse, http://dbpedia.org/resource/Alfa_Romeo, http://dbpedia.org/resource/Allders, http://dbpedia.org/resource/Allen_&_Overy, http://dbpedia.org/resource/Allstate, http://dbpedia.org/resource/Allyn_&_Bacon, http://dbpedia.org/resource/Alta_Newspaper_Group, http://dbpedia.org/resource/Alterra_Power, http://dbpedia.org/resource/American_Health_Connection, http://dbpedia.org/resource/American_Motors_Corporation, http://dbpedia.org/resource/Amey_plc, http://dbpedia.org/resource/Anchor_(brand), http://dbpedia.org/resource/Andor_Technology, http://dbpedia.org/resource/Anglia_Television, http://dbpedia.org/resource/Angoss, http://dbpedia.org/resource/Animal_Logic, http://dbpedia.org/resource/Anime_Limited, http://dbpedia.org/resource/Ann_Summers, http://dbpedia.org/resource/Aon_Corporation, http://dbpedia.org/resource/Appnovation, http://dbpedia.org/resource/Arc'teryx, http://dbpedia.org/resource/Arcelor, http://dbpedia.org/resource/Archway_Cookies, http://dbpedia.org/resource/Areva_Resources_Canada, http://dbpedia.org/resource/Aritzia, http://dbpedia.org/resource/Army_&_Navy_Stores_(Canada), http://dbpedia.org/resource/Arriva, http://dbpedia.org/resource/Arriva_London, http://dbpedia.org/resource/Arsenal_Pulp_Press, http://dbpedia.org/resource/Aryzta, http://dbpedia.org/resource/Associated-Rediffusion, http://dbpedia.org/resource/Atimi_Software_Inc, http://dbpedia.org/resource/Augmentation_Research_Center, http://dbpedia.org/resource/Aviation_Industry_Corporation_of_China, http://dbpedia.org/resource/B612_Foundation, http://dbpedia.org/resource/BETDAQ, http://dbpedia.org/resource/BGC_Partners, http://dbpedia.org/resource/Bain_&_Company, http://dbpedia.org/resource/Balfour_Beatty, http://dbpedia.org/resource/Bandai_Entertainment, http://dbpedia.org/resource/Bandai_Namco_Games, http://dbpedia.org/resource/Bank_of_Montreal, http://dbpedia.org/resource/Barnes_&_Noble, http://dbpedia.org/resource/Barr_Pharmaceuticals, http://dbpedia.org/resource/Bauer_Radio, http://dbpedia.org/resource/Baxter_International, http://dbpedia.org/resource/Becker's, http://dbpedia.org/resource/Beijing_Genomics_Institute, http://dbpedia.org/resource/Bellator_MMA, http://dbpedia.org/resource/Belron, http://dbpedia.org/resource/Bendix_Corporation, http://dbpedia.org/resource/Bennett_Environmental, http://dbpedia.org/resource/Beretta, http://dbpedia.org/resource/BigPark, http://dbpedia.org/resource/Big_Blue_Bubble, http://dbpedia.org/resource/BioWare, http://dbpedia.org/resource/Biovail, http://dbpedia.org/resource/Bird_Studio, http://dbpedia.org/resource/Birlinn_Limited, http://dbpedia.org/resource/Bit_Corporation, http://dbpedia.org/resource/BlackBerry_(company), http://dbpedia.org/resource/Black_Hen_Music, http://dbpedia.org/resource/Blackburn_Radio, http://dbpedia.org/resource/Blake,_Cassels_&_Graydon, http://dbpedia.org/resource/Blank_Rome, http://dbpedia.org/resource/Bleeding_Art_Industries, http://dbpedia.org/resource/Blenz_Coffee, http://dbpedia.org/resource/Blippar, http://dbpedia.org/resource/Blue_Cross_and_Blue_Shield_Association, http://dbpedia.org/resource/Blue_Sky_Studios, http://dbpedia.org/resource/Bluenotes, http://dbpedia.org/resource/Boeing_Canada, http://dbpedia.org/resource/Border_Television, http://dbpedia.org/resource/Boston_Musical_Instrument_Company, http://dbpedia.org/resource/Boston_Pizza, http://dbpedia.org/resource/Bowring_Brothers, http://dbpedia.org/resource/Boyd_Group_Income_Fund, http://dbpedia.org/resource/Brick_Brewing_Company, http://dbpedia.org/resource/Bridgewater_Associates, http://dbpedia.org/resource/Bridgewater_Bank, http://dbpedia.org/resource/BrightSide_Technologies, http://dbpedia.org/resource/Brightlight_Pictures, http://dbpedia.org/resource/British_American_Tobacco, http://dbpedia.org/resource/British_Broadcasting_Company, http://dbpedia.org/resource/British_Caledonian_in_the_1970s, http://dbpedia.org/resource/Brock_Solutions, http://dbpedia.org/resource/Bruce_Power, http://dbpedia.org/resource/Brunet_(pharmacy), http://dbpedia.org/resource/Buick, http://dbpedia.org/resource/Bulk_Barn, http://dbpedia.org/resource/Bullfrog_Power, http://dbpedia.org/resource/Bungie, http://dbpedia.org/resource/Bureau_of_Indian_Affairs, http://dbpedia.org/resource/CDC_Group, http://dbpedia.org/resource/CHUM_Limited, http://dbpedia.org/resource/CIT_Group, http://dbpedia.org/resource/CME_Group, http://dbpedia.org/resource/CTfastrak, http://dbpedia.org/resource/Cabot_Creamery, http://dbpedia.org/resource/Cadbury, http://dbpedia.org/resource/Cadillac_Fairview, http://dbpedia.org/resource/Café_Coffee_Day, http://dbpedia.org/resource/Caisse_des_dépôts_et_consignations, http://dbpedia.org/resource/California_Wolf_Center, http://dbpedia.org/resource/CallidusCloud, http://dbpedia.org/resource/Calm_Air, http://dbpedia.org/resource/Caltex, http://dbpedia.org/resource/Calypso_Technology, http://dbpedia.org/resource/Campbell_Soup_Company, http://dbpedia.org/resource/CanJet, http://dbpedia.org/resource/Canadian_Bank_Note_Company, http://dbpedia.org/resource/Canadian_Hydro_Developers, http://dbpedia.org/resource/Canadian_Imperial_Bank_of_Commerce, http://dbpedia.org/resource/Canadian_Salt_Company, http://dbpedia.org/resource/Canadian_Tire_Financial_Services, http://dbpedia.org/resource/Candente_Copper, http://dbpedia.org/resource/Canpotex, http://dbpedia.org/resource/Canterbury_of_New_Zealand, http://dbpedia.org/resource/Canucks_Sports_&_Entertainment, http://dbpedia.org/resource/CapeRay, http://dbpedia.org/resource/Capitec_Bank, http://dbpedia.org/resource/Cara_Operations, http://dbpedia.org/resource/Carfax_(company), http://dbpedia.org/resource/Carroll_&_Graf_Publishers, http://dbpedia.org/resource/Cartoon_Network_Studios, http://dbpedia.org/resource/Casavant_Frères, http://dbpedia.org/resource/Case_Construction_Equipment, http://dbpedia.org/resource/Case_Corporation, http://dbpedia.org/resource/Case_IH, http://dbpedia.org/resource/Catalyst_Paper, http://dbpedia.org/resource/Celestica, http://dbpedia.org/resource/Cellcom_Communications, http://dbpedia.org/resource/Central_Mountain_Air, http://dbpedia.org/resource/Cerberus_Capital_Management, http://dbpedia.org/resource/Chapters, http://dbpedia.org/resource/Chase_(bank), http://dbpedia.org/resource/Chicago_Board_Options_Exchange, http://dbpedia.org/resource/Chicago_Board_of_Trade, http://dbpedia.org/resource/Chicago_Mercantile_Exchange, http://dbpedia.org/resource/Chicago_Theatre, http://dbpedia.org/resource/Chicago_Transit_Authority, http://dbpedia.org/resource/Chimera_Entertainment, http://dbpedia.org/resource/China_Central_Television, http://dbpedia.org/resource/Chiquita_Brands_International, http://dbpedia.org/resource/Choices_Market, http://dbpedia.org/resource/Christian_Television_Network, http://dbpedia.org/resource/Christiania_Bank, http://dbpedia.org/resource/Christiania_Theatre, http://dbpedia.org/resource/Chrysler_(brand), http://dbpedia.org/resource/Chrysler_(division), http://dbpedia.org/resource/Chrysler_Canada, http://dbpedia.org/resource/Chrysler_Europe, http://dbpedia.org/resource/Churchill_Investments_Group, http://dbpedia.org/resource/Cigna, http://dbpedia.org/resource/Cineplex_Entertainment, http://dbpedia.org/resource/Cingular_Wireless, http://dbpedia.org/resource/Cirque_du_Soleil, http://dbpedia.org/resource/Citizens_Bank_of_Canada, http://dbpedia.org/resource/Cleopatra_Records, http://dbpedia.org/resource/Club_Monaco, http://dbpedia.org/resource/Clyde_&_Co, http://dbpedia.org/resource/Coachman_Insurance_Company, http://dbpedia.org/resource/Coca-Cola_Amatil, http://dbpedia.org/resource/Coca-Cola_Enterprises, http://dbpedia.org/resource/Coca-Cola_FEMSA, http://dbpedia.org/resource/Coca-Cola_HBC_AG, http://dbpedia.org/resource/Coeur_Mining, http://dbpedia.org/resource/Coffee_Time, http://dbpedia.org/resource/Cognos, http://dbpedia.org/resource/Colt's_Manufacturing_Company, http://dbpedia.org/resource/Colt_Canada, http://dbpedia.org/resource/Columbia_Aircraft, http://dbpedia.org/resource/Columbia_Insurance_Group, http://dbpedia.org/resource/ComfortDelGro_Cabcharge, http://dbpedia.org/resource/Comisión_Federal_de_Electricidad, http://dbpedia.org/resource/Comm100, http://dbpedia.org/resource/Commer, http://dbpedia.org/resource/Commonwealth_Edison, http://dbpedia.org/resource/Comparethemarket.com, http://dbpedia.org/resource/Computacenter, http://dbpedia.org/resource/Concentra_Financial, http://dbpedia.org/resource/Concord_Area_Transit, http://dbpedia.org/resource/Concord_Coach_Lines, http://dbpedia.org/resource/Concord_Pacific_Developments, http://dbpedia.org/resource/Conestoga-Rovers_&_Associates, http://dbpedia.org/resource/Conexus_Credit_Union, http://dbpedia.org/resource/Connecticut_Transit, http://dbpedia.org/resource/Conspiracy_Entertainment, http://dbpedia.org/resource/Construction_Data_Company, http://dbpedia.org/resource/Continental_Newspapers, http://dbpedia.org/resource/Copper_Canyon_Press, http://dbpedia.org/resource/Corby_Distilleries, http://dbpedia.org/resource/Coronation_Fund_Managers, http://dbpedia.org/resource/Corporation_for_National_and_Community_Service, http://dbpedia.org/resource/Coteau_Books, http://dbpedia.org/resource/Cott, http://dbpedia.org/resource/Country_Style, http://dbpedia.org/resource/Courage_International, http://dbpedia.org/resource/Crackle_(company), http://dbpedia.org/resource/Crate_&_Barrel, http://dbpedia.org/resource/Cray, http://dbpedia.org/resource/Cray_Research, http://dbpedia.org/resource/Creation_Entertainment, http://dbpedia.org/resource/Creation_Records, http://dbpedia.org/resource/Crestline_Coach, http://dbpedia.org/resource/Crosley_Broadcasting_Corporation, http://dbpedia.org/resource/Crown_Prosecution_Service, http://dbpedia.org/resource/Cummins, http://dbpedia.org/resource/Curtiss-Wright_Corporation, http://dbpedia.org/resource/Customer1, http://dbpedia.org/resource/Customers, http://dbpedia.org/resource/Cygnus_Solutions, http://dbpedia.org/resource/Cyrillus, http://dbpedia.org/resource/D-Wave_Systems, http://dbpedia.org/resource/DLA_Piper, http://dbpedia.org/resource/DTS_(sound_system), http://dbpedia.org/resource/Daily_Mail_and_General_Trust, http://dbpedia.org/resource/DaimlerChrysler_AG, http://dbpedia.org/resource/Dairyland_Canada, http://dbpedia.org/resource/Daiya, http://dbpedia.org/resource/Dan_Ingalls, http://dbpedia.org/resource/Dassault, http://dbpedia.org/resource/Datasoft, http://dbpedia.org/resource/DeMoulas_Market_Basket, http://dbpedia.org/resource/DeSoto_(automobile), http://dbpedia.org/resource/Dean_Guitars, http://dbpedia.org/resource/Delivery.com, http://dbpedia.org/resource/Dell_Comics, http://dbpedia.org/resource/Delta_Hotels, http://dbpedia.org/resource/Devil's_Due_Publishing, http://dbpedia.org/resource/DiC_Entertainment, http://dbpedia.org/resource/Diamond-Star_Motors, http://dbpedia.org/resource/Diamond_Energy, http://dbpedia.org/resource/Digital_Extremes, http://dbpedia.org/resource/Discovery_Air, http://dbpedia.org/resource/Dole_Food_Company, http://dbpedia.org/resource/Dollarama, http://dbpedia.org/resource/Dominion_Voting_Systems, http://dbpedia.org/resource/Domtar, http://dbpedia.org/resource/Donner_Metals, http://dbpedia.org/resource/Douglas_Aircraft, http://dbpedia.org/resource/Dr_Pepper_Snapple_Group, http://dbpedia.org/resource/DreamWorks_Animation, http://dbpedia.org/resource/Dubai_Ports_World, http://dbpedia.org/resource/DynaVenture, http://dbpedia.org/resource/Dynamite_Entertainment, http://dbpedia.org/resource/Dynamotive_Energy_Systems, http://dbpedia.org/resource/Dynamsoft, http://dbpedia.org/resource/EA_Black_Box, http://dbpedia.org/resource/EA_Canada, http://dbpedia.org/resource/EA_Sports, http://dbpedia.org/resource/EDF_Energy, http://dbpedia.org/resource/Eagle_(automobile)]... [INFO] P: 2.195124628200766E-5, R: 1.0, F: 4.390152887074292E-5 -[INFO] Cumulative metrics, sample: 83, P: 0.330, R: 0.378, macro F1: 0.328, macro F1*: 0.352 -[INFO] QALD Cumulative metrics, sample: 83, P: 0.546, R: 0.378, macro F1: 0.328, QALD macro F1: 0.447 -[INFO] Current time: 5151 ms, average: 3130.14 ms/ques, entity detection: 661.46 ms/ques (linking tool: 574.39 ms/ques), relation detection: 972.54 ms/ques (similarity: 563.16 ms/ques), EDG generation: 67.10 ms/ques +[INFO] Cumulative metrics, sample: 83, P: 0.330, R: 0.378, macro F1: 0.352 +[INFO] QALD Cumulative metrics, sample: 83, P: 0.546, R: 0.378, average F1: 0.328, QALD macro F1: 0.447 +[INFO] Current question consumed time: 4805 ms, average: 2925.63 ms/ques, entity detection: 603.71 ms/ques (linking tool: 526.05 ms/ques), relation detection: 930.23 ms/ques (similarity: 565.86 ms/ques), EDG generation: 67.76 ms/ques [INFO] Entity Linking P: 0.27710843373493976, R: 0.27710843373493976, micro F1: 0.27710843373493976, macro F1: 0.27710843373493976 [INFO] Relation Linking P: 0.30120481927710846, R: 0.30120481927710846, micro F1: 0.30120481927710846, macro F1: 0.30120481927710846 [INFO] Type Linking P: 0.9156626506024096, R: 0.9156626506024096, micro F1: 0.9156626506024096, macro F1: 0.9156626506024096 @@ -1252,9 +1252,9 @@ Question 83: Which professional surfers were born on the Philippines? [INFO] Golden Answer: [http://dbpedia.org/resource/Marcus_Adoro] [INFO] Predicted Answer: [http://dbpedia.org/resource/Bianca_Gonzalez, http://dbpedia.org/resource/Ejay_Falcon, http://dbpedia.org/resource/Enchong_Dee, http://dbpedia.org/resource/Jimboy_Martin, http://dbpedia.org/resource/John_Prats, http://dbpedia.org/resource/Keanna_Reeves, http://dbpedia.org/resource/Mariel_Rodriguez, http://dbpedia.org/resource/Melisa_Cantiveros, http://dbpedia.org/resource/Myrtle_Sarrosa, http://dbpedia.org/resource/Nonito_Donaire, http://dbpedia.org/resource/Rico_Hizon, http://dbpedia.org/resource/Slater_Young, http://dbpedia.org/resource/Willie_Revillame, http://dbpedia.org/resource/Ehra_Madrigal, http://dbpedia.org/resource/Jamby_Madrigal, http://dbpedia.org/resource/Jessica_Soho, http://dbpedia.org/resource/Michelle_Madrigal, http://dbpedia.org/resource/Ma-Anne_Dionisio, http://dbpedia.org/resource/Monique_Wilson, http://dbpedia.org/resource/Ruben_Enaje, http://dbpedia.org/resource/Cheryl_Diaz_Meyer, http://dbpedia.org/resource/Cristeta_Comerford, http://dbpedia.org/resource/Melissa_de_la_Cruz, http://dbpedia.org/resource/Billy_Hinsche, http://dbpedia.org/resource/Gregorio_F._Zaide, http://dbpedia.org/resource/Margarita_Moran-Floirendo, http://dbpedia.org/resource/Robb_White, http://dbpedia.org/resource/Karen_McCullah, http://dbpedia.org/resource/Reggie_Lee_(actor), http://dbpedia.org/resource/Sanjay_Beach, http://dbpedia.org/resource/Hazel_Mae, http://dbpedia.org/resource/Mig_Macario, http://dbpedia.org/resource/Dindin_Moreno, http://dbpedia.org/resource/Jesus_Salud, http://dbpedia.org/resource/Manuel_Ocampo, http://dbpedia.org/resource/Raimund_Marasigan, http://dbpedia.org/resource/Kitkat_(comedian), http://dbpedia.org/resource/Pooh_(comedian), http://dbpedia.org/resource/Viktoria_(singer), http://dbpedia.org/resource/Bong_Revilla, http://dbpedia.org/resource/Celeste_Legaspi, http://dbpedia.org/resource/Diether_Ocampo, http://dbpedia.org/resource/Joel_Lamangan, http://dbpedia.org/resource/Panfilo_Lacson, http://dbpedia.org/resource/Roxanne_Guinoo, http://dbpedia.org/resource/Sugar_Mercado, http://dbpedia.org/resource/Yancy_de_Ocampo, http://dbpedia.org/resource/Allan_Caidic, http://dbpedia.org/resource/Caloy_Garcia, http://dbpedia.org/resource/Chito_Narvasa, http://dbpedia.org/resource/Chito_Salud, http://dbpedia.org/resource/Chito_Victolero, http://dbpedia.org/resource/Chot_Reyes, http://dbpedia.org/resource/Danny_Ildefonso, http://dbpedia.org/resource/Francis_Arnaiz, http://dbpedia.org/resource/Jayjay_Helterbrand, http://dbpedia.org/resource/Johnny_Abarrientos, http://dbpedia.org/resource/Leo_Austria, http://dbpedia.org/resource/Mark_Caguioa, http://dbpedia.org/resource/Ramon_Fernandez, http://dbpedia.org/resource/Vergel_Meneses, http://dbpedia.org/resource/Wilfred_Steven_Uytengsu, http://dbpedia.org/resource/Yeng_Guiao, http://dbpedia.org/resource/Karla_Henry, http://dbpedia.org/resource/Lorraine_Schuck, http://dbpedia.org/resource/Alfredo_Razon_Gonzalez, http://dbpedia.org/resource/Charee_Pineda, http://dbpedia.org/resource/Jazz_Ocampo, http://dbpedia.org/resource/Magtanggol_Gunigundo_I, http://dbpedia.org/resource/Oscar_Malapitan, http://dbpedia.org/resource/Paolo_Duterte, http://dbpedia.org/resource/Raymart_Santiago, http://dbpedia.org/resource/Rex_Gatchalian, http://dbpedia.org/resource/Rochelle_Pangilinan, http://dbpedia.org/resource/Beauty_Gonzalez, http://dbpedia.org/resource/Buddy_Zabala, http://dbpedia.org/resource/Erich_Gonzales, http://dbpedia.org/resource/Marcus_Adoro, http://dbpedia.org/resource/Mark_Meily, http://dbpedia.org/resource/Sonny_Angara, http://dbpedia.org/resource/Charmane_Star, http://dbpedia.org/resource/Tita_Duran, http://dbpedia.org/resource/Misagh_Bahadoran, http://dbpedia.org/resource/Raphy_Reyes, http://dbpedia.org/resource/Dennis_Laurente, http://dbpedia.org/resource/Wynne_Arboleda, http://dbpedia.org/resource/Fitch_Arboleda, http://dbpedia.org/resource/Jovin_Bedic, http://dbpedia.org/resource/Norman_Fegidero, http://dbpedia.org/resource/Paolo_Bugas, http://dbpedia.org/resource/Patrick_Deyto, http://dbpedia.org/resource/Rodolfo_Alicante, http://dbpedia.org/resource/Simone_Rota, http://dbpedia.org/resource/Helen_Vela, http://dbpedia.org/resource/Maria_Teresa_Carlson, http://dbpedia.org/resource/Emelio_Caligdong, http://dbpedia.org/resource/Felix_Roco, http://dbpedia.org/resource/Ina_Raymundo, http://dbpedia.org/resource/Jiro_Manio, http://dbpedia.org/resource/Romnick_Sarmenta, http://dbpedia.org/resource/Carl_Benito, http://dbpedia.org/resource/Zaira_Cosico, http://dbpedia.org/resource/Benjamin_Evangelista, http://dbpedia.org/resource/Maricar_Balagtas, http://dbpedia.org/resource/Alfie_Almario, http://dbpedia.org/resource/John_Geesnell_Yap, http://dbpedia.org/resource/Kai_(singer), http://dbpedia.org/resource/Anna_Theresa_Licaros, http://dbpedia.org/resource/Bianca_Guidotti, http://dbpedia.org/resource/Carlene_Aguilar, http://dbpedia.org/resource/Czarina_Gatbonton, http://dbpedia.org/resource/Dianne_Necio, http://dbpedia.org/resource/Francine_Prieto, http://dbpedia.org/resource/Gionna_Cabrera, http://dbpedia.org/resource/Janicel_Lubina, http://dbpedia.org/resource/Janina_San_Miguel, http://dbpedia.org/resource/Jillian_Ward, http://dbpedia.org/resource/Joanna_Bacalso, http://dbpedia.org/resource/Joanne_Quintas, http://dbpedia.org/resource/Katrina_Jayne_Dimaranan, http://dbpedia.org/resource/Kylie_Verzosa, http://dbpedia.org/resource/Lia_Andrea_Ramos, http://dbpedia.org/resource/Liza_Lorena, http://dbpedia.org/resource/Maggie_Wilson, http://dbpedia.org/resource/Maxine_Medina, http://dbpedia.org/resource/Melody_Gersbach, http://dbpedia.org/resource/Paolo_Contis, http://dbpedia.org/resource/Patricia_Fernandez_(actress), http://dbpedia.org/resource/Polo_Ravales, http://dbpedia.org/resource/Rio_Locsin, http://dbpedia.org/resource/Vincent_Crisologo, http://dbpedia.org/resource/Alberto_Reynoso, http://dbpedia.org/resource/Enrique_Beech, http://dbpedia.org/resource/Jong_Uichico, http://dbpedia.org/resource/Leopoldo_Ang, http://dbpedia.org/resource/Olsen_Racela, http://dbpedia.org/resource/Paterno_Miranda, http://dbpedia.org/resource/Roberto_Roxas, http://dbpedia.org/resource/Rolando_Guaves, http://dbpedia.org/resource/Ahron_Villena, http://dbpedia.org/resource/Al_Tantay, http://dbpedia.org/resource/Alex_Castro, http://dbpedia.org/resource/Aubrey_Miles, http://dbpedia.org/resource/Cogie_Domingo, http://dbpedia.org/resource/Cris_Villanueva, http://dbpedia.org/resource/Cristina_Gonzales, http://dbpedia.org/resource/Devon_Seron, http://dbpedia.org/resource/Dick_Israel, http://dbpedia.org/resource/Dominic_Zapata, http://dbpedia.org/resource/Edgar_Allan_Guzman, http://dbpedia.org/resource/Efren_Reyes_Jr., http://dbpedia.org/resource/Frank_G._Rivera, http://dbpedia.org/resource/Isabel_Granada, http://dbpedia.org/resource/Jaime_Fabregas, http://dbpedia.org/resource/James_Blanco, http://dbpedia.org/resource/Jennica_Garcia, http://dbpedia.org/resource/Keempee_de_Leon, http://dbpedia.org/resource/Lito_Pimentel, http://dbpedia.org/resource/Macky_Escalona, http://dbpedia.org/resource/Malou_de_Guzman, http://dbpedia.org/resource/Maricar_Reyes, http://dbpedia.org/resource/Martin_Escudero, http://dbpedia.org/resource/Matet_de_Leon, http://dbpedia.org/resource/Mike_Tan, http://dbpedia.org/resource/Rey_%22PJ%22_Abellana, http://dbpedia.org/resource/Sandy_Andolong, http://dbpedia.org/resource/Shay_Jordan, http://dbpedia.org/resource/Sofia_Andres, http://dbpedia.org/resource/Tanya_Garcia, http://dbpedia.org/resource/Tonton_Gutierrez, http://dbpedia.org/resource/Tyron_Perez, http://dbpedia.org/resource/Vhong_Navarro, http://dbpedia.org/resource/Yam_Laranas, http://dbpedia.org/resource/Zoren_Legaspi, http://dbpedia.org/resource/Freddie_Abuda, http://dbpedia.org/resource/Jewel_Ponferada, http://dbpedia.org/resource/Francis_Magundayao, http://dbpedia.org/resource/Rose_Fostanes, http://dbpedia.org/resource/Bernard_Palanca, http://dbpedia.org/resource/Juanito_Victor_Remulla, http://dbpedia.org/resource/Sandy_Talag, http://dbpedia.org/resource/JR_Cawaling, http://dbpedia.org/resource/Jervy_Cruz, http://dbpedia.org/resource/Marvin_Cruz, http://dbpedia.org/resource/Pocholo_Villanueva, http://dbpedia.org/resource/RR_Garcia, http://dbpedia.org/resource/Simon_Atkins_(basketball), http://dbpedia.org/resource/J.R._Quiñahan, http://dbpedia.org/resource/Ivan_Mayrina, http://dbpedia.org/resource/Liezl_Martinez, http://dbpedia.org/resource/Mariz_Umali, http://dbpedia.org/resource/Carlo_Gonzales, http://dbpedia.org/resource/Boy_Cabahug, http://dbpedia.org/resource/Chris_Exciminiano, http://dbpedia.org/resource/Mark_Borboran, http://dbpedia.org/resource/Teddy_Alfarero, http://dbpedia.org/resource/Vic_Manuel, http://dbpedia.org/resource/Kanakan_Balintagos, http://dbpedia.org/resource/Carlo_Piccio, http://dbpedia.org/resource/Dennis_Lajola, http://dbpedia.org/resource/Celso_Lobregat, http://dbpedia.org/resource/Erico_Basilio_Fabian, http://dbpedia.org/resource/Rufa_Mi, http://dbpedia.org/resource/Ryan_Ramos, http://dbpedia.org/resource/Zia_Marquez, http://dbpedia.org/resource/Arnold_Atienza, http://dbpedia.org/resource/Julius_Caesar_Herrera, http://dbpedia.org/resource/Mikee_Romero, http://dbpedia.org/resource/Glaiza_Herradura, http://dbpedia.org/resource/Lawrence_Bonus, http://dbpedia.org/resource/Sigfrido_Tiñga, http://dbpedia.org/resource/Koreen_Medina, http://dbpedia.org/resource/Lourence_Ilagan, http://dbpedia.org/resource/Ric_Manrique,_Jr., http://dbpedia.org/resource/Vince_Gamad, http://dbpedia.org/resource/Gretchen_Abaniel, http://dbpedia.org/resource/Monika_Santa_Maria, http://dbpedia.org/resource/Galo_Ador,_Jr., http://dbpedia.org/resource/Larry_Cruz, http://dbpedia.org/resource/Athena_Lee_(marksman), http://dbpedia.org/resource/Mary_Ejercito, http://dbpedia.org/resource/Barbara_Miguel, http://dbpedia.org/resource/Ogie_Escanilla, http://dbpedia.org/resource/Garvo_Lañete, http://dbpedia.org/resource/Kara_Magsanoc-Alikpala, http://dbpedia.org/resource/Benito_F._Reyes, http://dbpedia.org/resource/Jejomar_Binay_Jr., http://dbpedia.org/resource/Nimfa_C._Vilches, http://dbpedia.org/resource/Aika_Robredo, http://dbpedia.org/resource/Fhrancis_Lopez, http://dbpedia.org/resource/Karlo_Maquinto, http://dbpedia.org/resource/Tamara_Benitez, http://dbpedia.org/resource/John_Pinto_(basketball), http://dbpedia.org/resource/Paolo_Taha, http://dbpedia.org/resource/Philip_Paniamogan, http://dbpedia.org/resource/Prince_Caperal, http://dbpedia.org/resource/Cardinal_Zen_(musician), http://dbpedia.org/resource/Carl_Balita, http://dbpedia.org/resource/Keith_Agovida, http://dbpedia.org/resource/CJ_Tañedo, http://dbpedia.org/resource/JC_Santos, http://dbpedia.org/resource/Harlene_Raguin, http://dbpedia.org/resource/Rafael_Arenillo_Cusi, http://dbpedia.org/resource/Gerardo_Roxas_Jr., http://dbpedia.org/resource/Tonisito_Umali, http://dbpedia.org/resource/Yeng_Guiao__Joseller_Guiao__1, http://dbpedia.org/resource/Dingdong_Dantes, http://dbpedia.org/resource/Gary_Valenciano, http://dbpedia.org/resource/Inday_Badiday, http://dbpedia.org/resource/Ogie_Alcasid, http://dbpedia.org/resource/Pepe_Diokno, http://dbpedia.org/resource/Abdurajik_Abubakar_Janjalani, http://dbpedia.org/resource/Albert_Martinez, http://dbpedia.org/resource/Alyssa_Alano, http://dbpedia.org/resource/Ben_Rich, http://dbpedia.org/resource/Ces_Drilon, http://dbpedia.org/resource/Eliseo_Soriano, http://dbpedia.org/resource/Gaudencio_Rosales, http://dbpedia.org/resource/Isnilon_Totoni_Hapilon, http://dbpedia.org/resource/KC_Concepcion, http://dbpedia.org/resource/Mel_Tiangco, http://dbpedia.org/resource/Rabeh_Al-Hussaini, http://dbpedia.org/resource/Ricardo_Visaya, http://dbpedia.org/resource/Richard_Gomez, http://dbpedia.org/resource/Rommel_Adducul, http://dbpedia.org/resource/Sheryl_Cruz, http://dbpedia.org/resource/Francis_Magalona, http://dbpedia.org/resource/Geoff_Eigenmann, http://dbpedia.org/resource/Gloria_Diaz, http://dbpedia.org/resource/Gretchen_Barretto, http://dbpedia.org/resource/Isabelle_Daza, http://dbpedia.org/resource/Jeric_Teng, http://dbpedia.org/resource/Juris_Fernandez, http://dbpedia.org/resource/Marilou_Diaz-Abaya, http://dbpedia.org/resource/Nico_Salva, http://dbpedia.org/resource/Onofre_Corpuz, http://dbpedia.org/resource/Rico_Blanco, http://dbpedia.org/resource/Ryan_Araña, http://dbpedia.org/resource/Bembol_Roco, http://dbpedia.org/resource/Kuh_Ledesma, http://dbpedia.org/resource/Jose_Ramon_Villarin, http://dbpedia.org/resource/Nelbert_Omolon, http://dbpedia.org/resource/Nicanor_Perlas, http://dbpedia.org/resource/Roberto_Yap, http://dbpedia.org/resource/Tomas_Osmeña, http://dbpedia.org/resource/Leopoldo_Serantes, http://dbpedia.org/resource/Mocha_Uson, http://dbpedia.org/resource/Narima_dela_Cruz, http://dbpedia.org/resource/Paulina_Constancia, http://dbpedia.org/resource/Luisito_Espinosa, http://dbpedia.org/resource/RJ_Rosales, http://dbpedia.org/resource/Carlos_Celdran, http://dbpedia.org/resource/Tito_Sotto, http://dbpedia.org/resource/Douglas_Quijano, http://dbpedia.org/resource/Eugene_Domingo, http://dbpedia.org/resource/Gelli_de_Belen, http://dbpedia.org/resource/Jaclyn_Jose, http://dbpedia.org/resource/Riego_Gamalinda, http://dbpedia.org/resource/Eden_Sonsona, http://dbpedia.org/resource/Glenn_Donaire, http://dbpedia.org/resource/Alicia_Alonzo, http://dbpedia.org/resource/Eddy_King, http://dbpedia.org/resource/Jessica_Zafra, http://dbpedia.org/resource/Maritoni_Fernandez, http://dbpedia.org/resource/Tita_Muñoz, http://dbpedia.org/resource/Victoria_Tauli-Corpuz, http://dbpedia.org/resource/Wilma_Galvante, http://dbpedia.org/resource/Francis_Reyes, http://dbpedia.org/resource/Frederic_Bartter, http://dbpedia.org/resource/Karl_Roy, http://dbpedia.org/resource/Fely_Irvine, http://dbpedia.org/resource/Kathleen_de_Leon_Jones, http://dbpedia.org/resource/Bela_Padilla, http://dbpedia.org/resource/Corky_Trinidad, http://dbpedia.org/resource/Christopher_Camat, http://dbpedia.org/resource/Eduardo_Buenavista, http://dbpedia.org/resource/Hidilyn_Diaz, http://dbpedia.org/resource/Miguel_Mendoza, http://dbpedia.org/resource/Romeo_Brin, http://dbpedia.org/resource/Rustico_Torrecampo, http://dbpedia.org/resource/Jimbo_Aquino, http://dbpedia.org/resource/Louise_delos_Reyes, http://dbpedia.org/resource/Roland_Dantes, http://dbpedia.org/resource/Joshua_Garcia, http://dbpedia.org/resource/Junrey_Balawing, http://dbpedia.org/resource/Voltaire_Y._Rosales, http://dbpedia.org/resource/Jerwin_Ancajas, http://dbpedia.org/resource/Sanya_Lopez, http://dbpedia.org/resource/Bryan_Termulo, http://dbpedia.org/resource/Alfred_Labatos, http://dbpedia.org/resource/Belinda_Mariano, http://dbpedia.org/resource/Bugoy_Cariño, http://dbpedia.org/resource/CJ_Navato, http://dbpedia.org/resource/CX_Navarro, http://dbpedia.org/resource/Carl_John_Barrameda, http://dbpedia.org/resource/Coney_Reyes, http://dbpedia.org/resource/DJ_Durano, http://dbpedia.org/resource/DM_Sevilla, http://dbpedia.org/resource/EJ_Jallorina, http://dbpedia.org/resource/Eunice_Lagusad, http://dbpedia.org/resource/Freddie_M._Garcia, http://dbpedia.org/resource/Gretchen_Fullido, http://dbpedia.org/resource/Harvey_Bautista, http://dbpedia.org/resource/John_Manalo, http://dbpedia.org/resource/Joseph_Andre_Garcia, http://dbpedia.org/resource/Lance_Lucido, http://dbpedia.org/resource/Louise_Abuel, http://dbpedia.org/resource/Lovely_Rivero, http://dbpedia.org/resource/Marc_Logan_(broadcast_journalist), http://dbpedia.org/resource/Marco_Alcaraz, http://dbpedia.org/resource/Marianne_dela_Riva, http://dbpedia.org/resource/Maris_Racal, http://dbpedia.org/resource/Mikee_Lee, http://dbpedia.org/resource/Miles_Ocampo, http://dbpedia.org/resource/Mylene_Dizon, http://dbpedia.org/resource/Nathan_Lopez, http://dbpedia.org/resource/Niña_Corpuz, http://dbpedia.org/resource/Pamu_Pamorada, http://dbpedia.org/resource/JC_Intal, http://dbpedia.org/resource/Carlo_Muñoz, http://dbpedia.org/resource/Empress_Schuck, http://dbpedia.org/resource/Lito_Legaspi, http://dbpedia.org/resource/Mickey_Ferriols, http://dbpedia.org/resource/Pekto, http://dbpedia.org/resource/Ping_Medina, http://dbpedia.org/resource/Tittsworth, http://dbpedia.org/resource/Brent_Chua, http://dbpedia.org/resource/Sonny_Cabatu, http://dbpedia.org/resource/Johnny_Midnight_(broadcaster), http://dbpedia.org/resource/Jason_Misolas, http://dbpedia.org/resource/Clodualdo_del_Mundo,_Jr., http://dbpedia.org/resource/Liz_Wahl, http://dbpedia.org/resource/Madam_Auring, http://dbpedia.org/resource/Josephine_Roberto, http://dbpedia.org/resource/Estong_Ballesteros, http://dbpedia.org/resource/Neptali_Gonzales_II, http://dbpedia.org/resource/Marlon_Stöckinger, http://dbpedia.org/resource/Malu_Maglutac, http://dbpedia.org/resource/Albert_Tiu, http://dbpedia.org/resource/Leandro_Muñoz, http://dbpedia.org/resource/Nicole_Abe, http://dbpedia.org/resource/Marion_Andres, http://dbpedia.org/resource/Emmanuel_Gonzalez, http://dbpedia.org/resource/Ashley_Ortega, http://dbpedia.org/resource/Krissel_Valdez, http://dbpedia.org/resource/Hillarie_Parungao, http://dbpedia.org/resource/Regie_Sahali-Generale, http://dbpedia.org/resource/Jon_Jaylo, http://dbpedia.org/resource/Robby_Celiz, http://dbpedia.org/resource/Jujeath_Nagaowa, http://dbpedia.org/resource/Rey_Aquino, http://dbpedia.org/resource/Leon_Miguel, http://dbpedia.org/resource/Joey_Romasanta, http://dbpedia.org/resource/Ed_Daquioag, http://dbpedia.org/resource/Britney_Dolonius, http://dbpedia.org/resource/Rolando_Magbanua, http://dbpedia.org/resource/Cesar_Conde_(artist), http://dbpedia.org/resource/Aga_Muhlach, http://dbpedia.org/resource/Angel_Locsin, http://dbpedia.org/resource/Apl.de.ap, http://dbpedia.org/resource/Aurora_Quezon, http://dbpedia.org/resource/Chito_S._Roño, http://dbpedia.org/resource/Derrick_Monasterio, http://dbpedia.org/resource/Dino_Imperial, http://dbpedia.org/resource/Francis_Pangilinan, http://dbpedia.org/resource/George_Canseco, http://dbpedia.org/resource/Gerry_Peñalosa, http://dbpedia.org/resource/Lydia_Yu-Jose, http://dbpedia.org/resource/Martin_Andanar, http://dbpedia.org/resource/Nikki_Gil, http://dbpedia.org/resource/Peque_Gallaga, http://dbpedia.org/resource/Samboy_Lim, http://dbpedia.org/resource/Tim_Tebow, http://dbpedia.org/resource/Apollo_Quiboloy, http://dbpedia.org/resource/Manny_Pacquiao, http://dbpedia.org/resource/Kenn_Navarro, http://dbpedia.org/resource/Leon_O._Chua, http://dbpedia.org/resource/Chris_Calaguio, http://dbpedia.org/resource/César_Basa, http://dbpedia.org/resource/Joonee_Gamboa, http://dbpedia.org/resource/Kenny_Kwan, http://dbpedia.org/resource/Krista_Ranillo, http://dbpedia.org/resource/Meryll_Soriano, http://dbpedia.org/resource/Rhea_Santos, http://dbpedia.org/resource/Brice_Mack, http://dbpedia.org/resource/Julia_Clarete, http://dbpedia.org/resource/Betong_Sumaya, http://dbpedia.org/resource/JC_Tiuseco, http://dbpedia.org/resource/Lucio_and_Simplicio_Godina, http://dbpedia.org/resource/Ignatius_Jones, http://dbpedia.org/resource/Angelo_Reyes, http://dbpedia.org/resource/E.R._Ejercito, http://dbpedia.org/resource/Joel_Torre, http://dbpedia.org/resource/Rachel_Grant, http://dbpedia.org/resource/William_Fox_(actor), http://dbpedia.org/resource/Kristine_Johnson, http://dbpedia.org/resource/Jhong_Hilario, http://dbpedia.org/resource/Sara_Duterte, http://dbpedia.org/resource/Terrence_Romeo, http://dbpedia.org/resource/Rodel_Mayol, http://dbpedia.org/resource/Benjamin_Dy, http://dbpedia.org/resource/James_Leonard_Gordon, http://dbpedia.org/resource/Sebastian_Benedict, http://dbpedia.org/resource/Arjo_Atayde, http://dbpedia.org/resource/Benjamin_Abalos,_Jr., http://dbpedia.org/resource/Carla_Abellana, http://dbpedia.org/resource/Chanda_Romero, http://dbpedia.org/resource/Jackie_Lou_Blanco, http://dbpedia.org/resource/Jimmy_Santos_(actor), http://dbpedia.org/resource/Kean_Cipriano, http://dbpedia.org/resource/Maricel_Laxa, http://dbpedia.org/resource/Matteo_Guidicelli, http://dbpedia.org/resource/Pilar_Pilapil, http://dbpedia.org/resource/Ruben_Habito, http://dbpedia.org/resource/Suzette_Ranillo, http://dbpedia.org/resource/Roberto_Pagdanganan, http://dbpedia.org/resource/Siot_Tanquingcen, http://dbpedia.org/resource/Tikoy_Aguiluz, http://dbpedia.org/resource/Arnel_Cerafica, http://dbpedia.org/resource/Bobby_Andrews, http://dbpedia.org/resource/Boy_Alano, http://dbpedia.org/resource/Ervic_Vijandre, http://dbpedia.org/resource/IC_Mendoza, http://dbpedia.org/resource/Jean_Garcia, http://dbpedia.org/resource/Maria_Isabel_Lopez, http://dbpedia.org/resource/Maricel_Soriano, http://dbpedia.org/resource/Marjorie_Barretto, http://dbpedia.org/resource/Mona_Louise_Rey, http://dbpedia.org/resource/Perla_Bautista, http://dbpedia.org/resource/Rica_Peralejo, http://dbpedia.org/resource/Valerie_Concepcion, http://dbpedia.org/resource/Rob_Bonta, http://dbpedia.org/resource/Gio_Alvarez, http://dbpedia.org/resource/Shehyee, http://dbpedia.org/resource/Cherie_Gil, http://dbpedia.org/resource/Hilda_Koronel, http://dbpedia.org/resource/Rodolfo_Luat, http://dbpedia.org/resource/Ian_Veneracion, http://dbpedia.org/resource/Dominic_Ochoa, http://dbpedia.org/resource/Josiah_Ng, http://dbpedia.org/resource/Victor_Basa, http://dbpedia.org/resource/Michael_Cinco, http://dbpedia.org/resource/Sassa_Jimenez, http://dbpedia.org/resource/Zarah_(television_personality), http://dbpedia.org/resource/Jay_Washington, http://dbpedia.org/resource/Camille_Prats, http://dbpedia.org/resource/Dondon_Hontiveros, http://dbpedia.org/resource/Elita_Loresca, http://dbpedia.org/resource/Hernando_Iriberri, http://dbpedia.org/resource/Paul_Alvarez, http://dbpedia.org/resource/Ronnie_Magsanoc, http://dbpedia.org/resource/Mico_Palanca, http://dbpedia.org/resource/Fort_Acuña, http://dbpedia.org/resource/Val_Acuña, http://dbpedia.org/resource/Danny_Florencio, http://dbpedia.org/resource/Denver_Cuello, http://dbpedia.org/resource/Bob_dela_Cruz, http://dbpedia.org/resource/Christian_Coronel, http://dbpedia.org/resource/Joseph_Bitangcol, http://dbpedia.org/resource/Mark_Macapagal, http://dbpedia.org/resource/Michael_de_Mesa, http://dbpedia.org/resource/Sunday_Salvacion, http://dbpedia.org/resource/Cathy_Yap-Yang, http://dbpedia.org/resource/Claire_Celdran, http://dbpedia.org/resource/Connie_Sison, http://dbpedia.org/resource/Dyan_Castillejo, http://dbpedia.org/resource/Paolo_Abrera, http://dbpedia.org/resource/Pia_Hontiveros, http://dbpedia.org/resource/Drian_Francisco, http://dbpedia.org/resource/Desiree_del_Valle, http://dbpedia.org/resource/Doris_Bigornia, http://dbpedia.org/resource/Gian_Barbarona, http://dbpedia.org/resource/Jan_Marini, http://dbpedia.org/resource/Kier_Legaspi, http://dbpedia.org/resource/Nikka_Valencia, http://dbpedia.org/resource/Alicia_Vergel, http://dbpedia.org/resource/Nick_Deocampo, http://dbpedia.org/resource/Onemig_Bondoc, http://dbpedia.org/resource/Nelson_Mariano_II, http://dbpedia.org/resource/Diosdado_Gabi, http://dbpedia.org/resource/Harris_Sarmiento, http://dbpedia.org/resource/J._Neil_Garcia, http://dbpedia.org/resource/Zean_Cabangis, http://dbpedia.org/resource/Spazzkid, http://dbpedia.org/resource/Junji_Arias, http://dbpedia.org/resource/Bituin_Escalante, http://dbpedia.org/resource/Eduardo_Zialcita, http://dbpedia.org/resource/Pryde_Henry_Teves, http://dbpedia.org/resource/Angelito_Sarmiento, http://dbpedia.org/resource/Sergio_Garcia_(actor), http://dbpedia.org/resource/Al_Solis, http://dbpedia.org/resource/Francis_Pasion, http://dbpedia.org/resource/Francis_Casey_Alcantara, http://dbpedia.org/resource/Ashley_Cabrera, http://dbpedia.org/resource/Kristelle_Lazaro, http://dbpedia.org/resource/Khavn_De_La_Cruz, http://dbpedia.org/resource/Milan_Melindo, http://dbpedia.org/resource/Dennis_Apuan, http://dbpedia.org/resource/Edna_Agravante, http://dbpedia.org/resource/Joey_Paras, http://dbpedia.org/resource/Jane_De_Leon, http://dbpedia.org/resource/Ronnie_Alonte, http://dbpedia.org/resource/Martha_Cecilia, http://dbpedia.org/resource/Franchesca_Salcedo, http://dbpedia.org/resource/R._D._Alba, http://dbpedia.org/resource/Enzo_Pastor, http://dbpedia.org/resource/Kenji_Roa, http://dbpedia.org/resource/Zandra_Summer, http://dbpedia.org/resource/Datu_Yusoph_Boyog_Mama, http://dbpedia.org/resource/Mayton_Eugenio, http://dbpedia.org/resource/J._P._Alcaraz, http://dbpedia.org/resource/Andres_Barrioquinto, http://dbpedia.org/resource/Robert_Bernardo, http://dbpedia.org/resource/Kenneth_Ighalo, http://dbpedia.org/resource/Janine_Nicandro, http://dbpedia.org/resource/Juan_Martin_Elorde, http://dbpedia.org/resource/Juan_Miguel_Elorde, http://dbpedia.org/resource/Ruel_Antipuesto, http://dbpedia.org/resource/Braulia_de_la_Santa_Cruz, http://dbpedia.org/resource/Caleb_Punzalan, http://dbpedia.org/resource/Negi_(comedian), http://dbpedia.org/resource/Renz_Ongkiko, http://dbpedia.org/resource/Charee_Crest, http://dbpedia.org/resource/Jorell_Crisostomo, http://dbpedia.org/resource/Carlo_J._Caparas, http://dbpedia.org/resource/Renaldo_Lapuz, http://dbpedia.org/resource/Benjamin_Diokno, http://dbpedia.org/resource/Jai_Reyes, http://dbpedia.org/resource/Kris_Aquino, http://dbpedia.org/resource/Manuel_V._Pangilinan, http://dbpedia.org/resource/Ralph_Recto, http://dbpedia.org/resource/Wenn_V._Deramas, http://dbpedia.org/resource/Shelley_Jackson, http://dbpedia.org/resource/Anja_Aguilar, http://dbpedia.org/resource/Brenda_Marshall, http://dbpedia.org/resource/Epy_Quizon, http://dbpedia.org/resource/Frank_Golla, http://dbpedia.org/resource/Justin_Chua, http://dbpedia.org/resource/Kim_Atienza, http://dbpedia.org/resource/LA_Tenorio, http://dbpedia.org/resource/Paolo_Bugia, http://dbpedia.org/resource/Wesley_Gonzales, http://dbpedia.org/resource/Alfred_Vargas, http://dbpedia.org/resource/Antonio_Carpio, http://dbpedia.org/resource/Atom_Araullo, http://dbpedia.org/resource/Atoy_Co, http://dbpedia.org/resource/Bea_Alonzo, http://dbpedia.org/resource/Benjie_Paras, http://dbpedia.org/resource/Christine_Bersola-Babao, http://dbpedia.org/resource/Christopher_de_Leon, http://dbpedia.org/resource/Ed_Ocampo, http://dbpedia.org/resource/Francis_Escudero, http://dbpedia.org/resource/Janine_Tugonon, http://dbpedia.org/resource/Joey_Marquez, http://dbpedia.org/resource/Joey_de_Leon, http://dbpedia.org/resource/Jun_Lana, http://dbpedia.org/resource/Kiefer_Ravena, http://dbpedia.org/resource/Marlou_Aquino, http://dbpedia.org/resource/Marrion_Gopez, http://dbpedia.org/resource/Nadine_Lustre, http://dbpedia.org/resource/Niña_Dolino, http://dbpedia.org/resource/Pia_Guanio, http://dbpedia.org/resource/Robert_Jaworski_Jr., http://dbpedia.org/resource/Rodjun_Cruz, http://dbpedia.org/resource/Vina_Morales, http://dbpedia.org/resource/Gerard_Salonga, http://dbpedia.org/resource/Jose_Llana, http://dbpedia.org/resource/Israel_Cruz, http://dbpedia.org/resource/Francis_Tolentino, http://dbpedia.org/resource/Mariqueen_Maandig, http://dbpedia.org/resource/Ara_Mina, http://dbpedia.org/resource/Benigno_G._Tabora, http://dbpedia.org/resource/Ellona_Santiago, http://dbpedia.org/resource/Jose_Calugas, http://dbpedia.org/resource/Chacha_Cañete, http://dbpedia.org/resource/Jose_Mari_Chan, http://dbpedia.org/resource/Kitchie_Nadal, http://dbpedia.org/resource/Pops_Fernandez, http://dbpedia.org/resource/Rachelle_Ann_Go, http://dbpedia.org/resource/Simeon_V._Marcelo, http://dbpedia.org/resource/Small_Montana, http://dbpedia.org/resource/Yael_Yuzon, http://dbpedia.org/resource/Jonathan_Uyloan, http://dbpedia.org/resource/Rida_Cabanilla, http://dbpedia.org/resource/Fredenil_Castro, http://dbpedia.org/resource/Mercedes_Alvarez_(politician), http://dbpedia.org/resource/Disappearance_of_Sherlyn_Cadapan_and_Karen_Empeño, http://dbpedia.org/resource/Bernardo_O._Dagala, http://dbpedia.org/resource/Canuto_E._Celestino, http://dbpedia.org/resource/Clint_Geronimo, http://dbpedia.org/resource/Froilan_Baguion, http://dbpedia.org/resource/Gabby_Concepcion, http://dbpedia.org/resource/Gerald_Santos, http://dbpedia.org/resource/John_Rey_Tiangco, http://dbpedia.org/resource/Joross_Gamboa, http://dbpedia.org/resource/Paul_Artadi, http://dbpedia.org/resource/Wendy_Valdez, http://dbpedia.org/resource/John_Wilson_(basketball), http://dbpedia.org/resource/Anthony_Clark_(powerlifter), http://dbpedia.org/resource/Daniel_Razon, http://dbpedia.org/resource/Iain_Macnab, http://dbpedia.org/resource/Bearwin_Meily, http://dbpedia.org/resource/Candy_Pangilinan, http://dbpedia.org/resource/Jade_Lopez, http://dbpedia.org/resource/Joyce_E._Bernal, http://dbpedia.org/resource/Gil_Portes, http://dbpedia.org/resource/Richard_Bordeaux_Parker, http://dbpedia.org/resource/Lucas_Bersamin, http://dbpedia.org/resource/Elmer_Bedia, http://dbpedia.org/resource/Ian_Araneta, http://dbpedia.org/resource/Quest_(singer), http://dbpedia.org/resource/Winston_Damarillo, http://dbpedia.org/resource/Bayani_Agbayani, http://dbpedia.org/resource/Mosang, http://dbpedia.org/resource/Ruel_S._Bayani, http://dbpedia.org/resource/Alex_Cabagnot, http://dbpedia.org/resource/Alex_Nuyles, http://dbpedia.org/resource/Ardy_Larong, http://dbpedia.org/resource/Dennis_Espino, http://dbpedia.org/resource/Don_Allado, http://dbpedia.org/resource/Dylan_Ababou, http://dbpedia.org/resource/Eliud_Poligrates, http://dbpedia.org/resource/Harold_Arboleda, http://dbpedia.org/resource/Jayson_Castro, http://dbpedia.org/resource/John_Ferriols, http://dbpedia.org/resource/Jun_Limpot, http://dbpedia.org/resource/Mark_Andaya, http://dbpedia.org/resource/Mark_Isip, http://dbpedia.org/resource/Mark_Telan, http://dbpedia.org/resource/Mark_Yee, http://dbpedia.org/resource/Matt_Ganuelas-Rosser, http://dbpedia.org/resource/Merlito_Sabillo, http://dbpedia.org/resource/Niño_Canaleta, http://dbpedia.org/resource/Robert_Labagala, http://dbpedia.org/resource/Ryan_Gregorio, http://dbpedia.org/resource/Ryan_Reyes, http://dbpedia.org/resource/Troy_Rosario, http://dbpedia.org/resource/Yousif_Aljamal, http://dbpedia.org/resource/Johan_Santos, http://dbpedia.org/resource/Joyce_Ching, http://dbpedia.org/resource/Chris_Tsuper, http://dbpedia.org/resource/Lota_Delgado, http://dbpedia.org/resource/Pacita_Abad, http://dbpedia.org/resource/Adolovni_Acosta, http://dbpedia.org/resource/Czar_Amonsot, http://dbpedia.org/resource/Dennis_Padilla, http://dbpedia.org/resource/Jed_Montero, http://dbpedia.org/resource/Jiggy_Manicad, http://dbpedia.org/resource/Kelly_Misa, http://dbpedia.org/resource/Aiko_Climaco, http://dbpedia.org/resource/Liz_Alindogan, http://dbpedia.org/resource/Mcoy_Fundales_(musician), http://dbpedia.org/resource/Andre_Paras, http://dbpedia.org/resource/Kobe_Paras, http://dbpedia.org/resource/Richie_Ticzon, http://dbpedia.org/resource/Rita_De_Guzman, http://dbpedia.org/resource/Rubilen_Amit, http://dbpedia.org/resource/Chito_Loyzaga, http://dbpedia.org/resource/Lizza_Danila, http://dbpedia.org/resource/Vicki_Belo, http://dbpedia.org/resource/Ramon_Mitra_III, http://dbpedia.org/resource/Robert_%22Ace%22_Barbers, http://dbpedia.org/resource/Cesar_Catli, http://dbpedia.org/resource/Junjun_Cabatu, http://dbpedia.org/resource/Khasim_Mirza, http://dbpedia.org/resource/M.C._Caceres, http://dbpedia.org/resource/Mac_Baracael, http://dbpedia.org/resource/Neil_Rañeses, http://dbpedia.org/resource/R.J._Rizada, http://dbpedia.org/resource/Reil_Cervantes, http://dbpedia.org/resource/Iggy_Pintado, http://dbpedia.org/resource/Gwendolyn_Ecleo, http://dbpedia.org/resource/Dale_Abenojar, http://dbpedia.org/resource/David_Remo, http://dbpedia.org/resource/Tonette_Lopez, http://dbpedia.org/resource/Adonis_Santa_Maria, http://dbpedia.org/resource/Don_Trollano, http://dbpedia.org/resource/Gilbert_Lao, http://dbpedia.org/resource/Jireh_Ibañes, http://dbpedia.org/resource/Josan_Nimes, http://dbpedia.org/resource/Niño_Gelig, http://dbpedia.org/resource/April_Ross_Perez, http://dbpedia.org/resource/Jonavi_Raisa_Quiray, http://dbpedia.org/resource/Marian_Michelle_Oblea, http://dbpedia.org/resource/Marie_Razel_Eguia, http://dbpedia.org/resource/Murielle_Adrienne_Orais, http://dbpedia.org/resource/Tessa_Cam_Rowe, http://dbpedia.org/resource/Warren_Ybañez, http://dbpedia.org/resource/Ciso_Morales, http://dbpedia.org/resource/Jannelle_So, http://dbpedia.org/resource/Jeff_Chan_(basketball), http://dbpedia.org/resource/Mhyco_Aquino, http://dbpedia.org/resource/Mark_Villon, http://dbpedia.org/resource/Rudy_Lingganay, http://dbpedia.org/resource/Neil_Perez, http://dbpedia.org/resource/Nathalie_Hart, http://dbpedia.org/resource/Tin_Patrimonio, http://dbpedia.org/resource/Miko_Manguba, http://dbpedia.org/resource/Kata_Inocencio, http://dbpedia.org/resource/Jose_Antonio_N._Carrion, http://dbpedia.org/resource/David_Basa, http://dbpedia.org/resource/Jerry_Barbaso, http://dbpedia.org/resource/Carmina_Villarroel, http://dbpedia.org/resource/Shamcey_Supsup-Lee, http://dbpedia.org/resource/Stanley_Chi, http://dbpedia.org/resource/Renato_Diaz, http://dbpedia.org/resource/Dominic_Meily, http://dbpedia.org/resource/Stephanie_Sol, http://dbpedia.org/resource/Jam_Sebastian, http://dbpedia.org/resource/Richie_D'Horsie, http://dbpedia.org/resource/Dimasangcay_Pundato, http://dbpedia.org/resource/Marlo_R._Cruz, http://dbpedia.org/resource/Diego_Castro_III, http://dbpedia.org/resource/Amado_Espino_Jr., http://dbpedia.org/resource/Juan_Carlo_Calma, http://dbpedia.org/resource/Aljo_Bendijo, http://dbpedia.org/resource/Alwyn_Uytingco, http://dbpedia.org/resource/Ana_Roces, http://dbpedia.org/resource/Benigno_Aquino_III, http://dbpedia.org/resource/Bernadette_Sembrano, http://dbpedia.org/resource/Freddie_Aguilar, http://dbpedia.org/resource/Gloria_Macapagal_Arroyo, http://dbpedia.org/resource/James_Yap, http://dbpedia.org/resource/Johnny_Manahan, http://dbpedia.org/resource/Juan_Karlos_Labajo, http://dbpedia.org/resource/Katy_de_la_Cruz, http://dbpedia.org/resource/Lea_Salonga, http://dbpedia.org/resource/Lou_Diamond_Phillips, http://dbpedia.org/resource/Ricardo_Vidal, http://dbpedia.org/resource/Tado_(comedian), http://dbpedia.org/resource/Joey_Manahan, http://dbpedia.org/resource/Laidback_Luke, http://dbpedia.org/resource/Jewel_Mische, http://dbpedia.org/resource/Alexander_B._Yano, http://dbpedia.org/resource/Bongbong_Marcos, http://dbpedia.org/resource/Delfin_Lorenzana, http://dbpedia.org/resource/Leo_Echegaray, http://dbpedia.org/resource/Pantaleon_Alvarez, http://dbpedia.org/resource/Presidency_of_Corazon_Aquino, http://dbpedia.org/resource/Peter_Solis_Nery, http://dbpedia.org/resource/Vicky_Morales, http://dbpedia.org/resource/Hermogenes_Esperon, http://dbpedia.org/resource/Murad_Ebrahim, http://dbpedia.org/resource/Jose_Antonio_Vargas, http://dbpedia.org/resource/Rolando_Navarrete, http://dbpedia.org/resource/Karen_Moe, http://dbpedia.org/resource/Geena_Rocero, http://dbpedia.org/resource/Alonzo_Muhlach, http://dbpedia.org/resource/Andrew_E., http://dbpedia.org/resource/Anjo_Yllana, http://dbpedia.org/resource/Bianca_Umali, http://dbpedia.org/resource/Charlene_Gonzales, http://dbpedia.org/resource/Chloe_Dauden, http://dbpedia.org/resource/Dale_Baldillo, http://dbpedia.org/resource/Dimples_Romana, http://dbpedia.org/resource/Gina_Pareño, http://dbpedia.org/resource/Gwendoline_Ruais, http://dbpedia.org/resource/JC_de_Vera, http://dbpedia.org/resource/Jake_Roxas, http://dbpedia.org/resource/Japs_Sergio, http://dbpedia.org/resource/Jason_Webb_(basketball), http://dbpedia.org/resource/Jondan_Salvador, http://dbpedia.org/resource/Karel_Marquez, http://dbpedia.org/resource/Kyla, http://dbpedia.org/resource/Ram_Revilla, http://dbpedia.org/resource/Reynante_Jamili, http://dbpedia.org/resource/Ronnie_Ricketts, http://dbpedia.org/resource/Ruffy_Biazon, http://dbpedia.org/resource/Wendell_Ramos, http://dbpedia.org/resource/Erbito_Salavarria, http://dbpedia.org/resource/Joey_Salceda, http://dbpedia.org/resource/Ryan_Eigenmann, http://dbpedia.org/resource/Sharmaine_Arnaiz, http://dbpedia.org/resource/Tating_Pasilan, http://dbpedia.org/resource/Eddie_Peregrina, http://dbpedia.org/resource/Howie_Severino, http://dbpedia.org/resource/Jed_Madela, http://dbpedia.org/resource/Joey_Albert, http://dbpedia.org/resource/Joy_Nilo, http://dbpedia.org/resource/Lani_Misalucha, http://dbpedia.org/resource/Lilet, http://dbpedia.org/resource/Maki_Pulido, http://dbpedia.org/resource/Manilyn_Reynes, http://dbpedia.org/resource/Pia_Arcangel, http://dbpedia.org/resource/Raffy_Tima, http://dbpedia.org/resource/Raymond_Lauchengco, http://dbpedia.org/resource/Ronnie_Liang, http://dbpedia.org/resource/Abet_Guidaben, http://dbpedia.org/resource/Glaiza_de_Castro, http://dbpedia.org/resource/Kris_Bernal, http://dbpedia.org/resource/José_Wendell_Capili, http://dbpedia.org/resource/Ruffa_Gutierrez, http://dbpedia.org/resource/Carmi_Martin, http://dbpedia.org/resource/Glydel_Mercado, http://dbpedia.org/resource/Iza_Calzado, http://dbpedia.org/resource/Jay_Aquitania, http://dbpedia.org/resource/Junix_Inocian, http://dbpedia.org/resource/Lorraine_Kendrickson, http://dbpedia.org/resource/Mark_Javier, http://dbpedia.org/resource/Marlon_Piñero, http://dbpedia.org/resource/May_Ling_Su, http://dbpedia.org/resource/Mimi_Miyagi, http://dbpedia.org/resource/Mig_Ayesa, http://dbpedia.org/resource/Vinci_Montaner, http://dbpedia.org/resource/Donald_Geisler, http://dbpedia.org/resource/Mon_Ilagan, http://dbpedia.org/resource/Eula_Valdez, http://dbpedia.org/resource/Coleen_Garcia, http://dbpedia.org/resource/Ellen_Adarna, http://dbpedia.org/resource/Joel_Tabora, http://dbpedia.org/resource/Jojo_Duncil, http://dbpedia.org/resource/Bryan_Benedict, http://dbpedia.org/resource/Abdusakur_Mahail_Tan, http://dbpedia.org/resource/Buda_Bautista, http://dbpedia.org/resource/Marlon_Maro, http://dbpedia.org/resource/Marnelli_Dimzon, http://dbpedia.org/resource/Cassandra_Ponti, http://dbpedia.org/resource/Bea_Nicolas, http://dbpedia.org/resource/Diana_Zubiri, http://dbpedia.org/resource/Jenny_Miller, http://dbpedia.org/resource/Neil_Ryan_Sese, http://dbpedia.org/resource/Marcy_Arellano, http://dbpedia.org/resource/Mutya_Orquia, http://dbpedia.org/resource/Chad_Alonzo, http://dbpedia.org/resource/Arthur_dela_Cruz, http://dbpedia.org/resource/Baser_Amer, http://dbpedia.org/resource/Bong_Hawkins, http://dbpedia.org/resource/Boyet_Bautista, http://dbpedia.org/resource/Carlo_Lastimosa, http://dbpedia.org/resource/Dave_Marcelo, http://dbpedia.org/resource/Gabby_Espinas, http://dbpedia.org/resource/Homer_Se, http://dbpedia.org/resource/Jake_Pascual, http://dbpedia.org/resource/Jay-R_Reyes, http://dbpedia.org/resource/Kelvin_dela_Peña, http://dbpedia.org/resource/RJ_Jazul, http://dbpedia.org/resource/Rodney_Santos, http://dbpedia.org/resource/Ronjay_Enrile, http://dbpedia.org/resource/Ryan_Buenafe, http://dbpedia.org/resource/Scottie_Thompson_(basketball), http://dbpedia.org/resource/Topex_Robinson, http://dbpedia.org/resource/Alexis_Belonio, http://dbpedia.org/resource/Alfonso_A._Uy, http://dbpedia.org/resource/Darius_Semaña, http://dbpedia.org/resource/Gab_Chee_Kee, http://dbpedia.org/resource/Lex_Tupas, http://dbpedia.org/resource/Otoniel_Gonzaga_(tenor_singer), http://dbpedia.org/resource/Louie_Mar_Gangcuangco, http://dbpedia.org/resource/Leo_Oracion, http://dbpedia.org/resource/Melissa_Mendez, http://dbpedia.org/resource/Stella_Ruiz_White, http://dbpedia.org/resource/Kenneth_Medrano, http://dbpedia.org/resource/Al_Vergara, http://dbpedia.org/resource/Aldrech_Ramos, http://dbpedia.org/resource/Allein_Maliksi, http://dbpedia.org/resource/Billy_Mamaril, http://dbpedia.org/resource/Bonel_Balingit, http://dbpedia.org/resource/Celino_Cruz, http://dbpedia.org/resource/Chico_Lañete, http://dbpedia.org/resource/E.J._Feihl, http://dbpedia.org/resource/Egay_Billones, http://dbpedia.org/resource/Ervin_Sotto, http://dbpedia.org/resource/Jeric_Fortuna, http://dbpedia.org/resource/Jerwin_Gaco, http://dbpedia.org/resource/Jonas_Villanueva, http://dbpedia.org/resource/Jorge_Gallent, http://dbpedia.org/resource/Justin_Melton, http://dbpedia.org/resource/Mark_Barroca, http://dbpedia.org/resource/Mark_Cruz, http://dbpedia.org/resource/Mike_Hrabak, http://dbpedia.org/resource/Peter_June_Simon, http://dbpedia.org/resource/Renren_Ritualo, http://dbpedia.org/resource/Rey_Evangelista, http://dbpedia.org/resource/Rey_Guevarra, http://dbpedia.org/resource/Rico_Maierhofer, http://dbpedia.org/resource/Roger_Yap, http://dbpedia.org/resource/Ronald_Pascual, http://dbpedia.org/resource/Ronnie_Matias, http://dbpedia.org/resource/Coy_Placido, http://dbpedia.org/resource/Ace_Espinosa, http://dbpedia.org/resource/Alex_Medina, http://dbpedia.org/resource/Benedict_Campos, http://dbpedia.org/resource/Byron_Ortile, http://dbpedia.org/resource/Ching_Arellano, http://dbpedia.org/resource/Chuckie_Dreyfus, http://dbpedia.org/resource/Cris_de_Vera, http://dbpedia.org/resource/Dar_Bernardo, http://dbpedia.org/resource/Dominic_Roque, http://dbpedia.org/resource/Edwin_San_Juan, http://dbpedia.org/resource/Freddie_Quizon, http://dbpedia.org/resource/Jak_Roberto, http://dbpedia.org/resource/Jay-R_Siaboc, http://dbpedia.org/resource/Jess_Lapid,_Jr., http://dbpedia.org/resource/John_Apacible, http://dbpedia.org/resource/Lance_Serrano, http://dbpedia.org/resource/Larry_Silva, http://dbpedia.org/resource/Lloyd_Zaragoza, http://dbpedia.org/resource/Manuel_Chua, http://dbpedia.org/resource/Miko_Sotto, http://dbpedia.org/resource/Mikoy_Morales, http://dbpedia.org/resource/Minióng_Álvarez, http://dbpedia.org/resource/Orlando_Nadres, http://dbpedia.org/resource/Pancho_Magno, http://dbpedia.org/resource/Paolo_Serrano, http://dbpedia.org/resource/Prince_Stefan_(actor), http://dbpedia.org/resource/Renz_Fernandez, http://dbpedia.org/resource/Ritche_Lago_Bautista, http://dbpedia.org/resource/Roldan_Aquino, http://dbpedia.org/resource/Ronald_Humarang, http://dbpedia.org/resource/Ross_Rival, http://dbpedia.org/resource/Roy_Rodrigo, http://dbpedia.org/resource/Teejay_Marquez, http://dbpedia.org/resource/Terence_Baylon, http://dbpedia.org/resource/Thou_Reyes, http://dbpedia.org/resource/Weng_Weng, http://dbpedia.org/resource/Yves_Flores, http://dbpedia.org/resource/Mark_Magsumbol, http://dbpedia.org/resource/Jopay, http://dbpedia.org/resource/Angelica_Barcelo, http://dbpedia.org/resource/Danny_Sillada, http://dbpedia.org/resource/Emmanuel_F._Lacaba, http://dbpedia.org/resource/Frederick_Adams_(basketball), http://dbpedia.org/resource/Antonio_Lagdameo,_Jr., http://dbpedia.org/resource/Pax_Mangudadatu, http://dbpedia.org/resource/Isabel_Rivas, http://dbpedia.org/resource/Toni_Rivero, http://dbpedia.org/resource/Zosimo_Paredes, http://dbpedia.org/resource/Don_Pepot, http://dbpedia.org/resource/Rommel_Asenjo, http://dbpedia.org/resource/Jacob_Benedicto, http://dbpedia.org/resource/Mark_Luz, http://dbpedia.org/resource/Ron_Henley_(rapper), http://dbpedia.org/resource/Heart_Yngrid, http://dbpedia.org/resource/Mohagher_Iqbal, http://dbpedia.org/resource/Petite_(comedian), http://dbpedia.org/resource/Divine_Aucina, http://dbpedia.org/resource/Dennice_Villamor, http://dbpedia.org/resource/Paulo_Hubalde, http://dbpedia.org/resource/Leo_Gaje, http://dbpedia.org/resource/Honeylet_Avanceña, http://dbpedia.org/resource/Paolo_lofamia, http://dbpedia.org/resource/Bruce_Tolentino, http://dbpedia.org/resource/Maria_Rita_Teresa_Batalla-Laforteza, http://dbpedia.org/resource/Teodoro_Escanilla, http://dbpedia.org/resource/Arianne_Caoili, http://dbpedia.org/resource/Eugenio_Torre, http://dbpedia.org/resource/Jayson_Gonzales, http://dbpedia.org/resource/Mark_Paragua, http://dbpedia.org/resource/Rodolfo_Tan_Cardoso, http://dbpedia.org/resource/Rogelio_Antonio_Jr., http://dbpedia.org/resource/Wesley_So, http://dbpedia.org/resource/Abraham_Sarmiento,_Jr., http://dbpedia.org/resource/Alex_Santos_(newscaster), http://dbpedia.org/resource/Andi_Eigenmann, http://dbpedia.org/resource/Bryan_Callen, http://dbpedia.org/resource/Cholo_Laurel, http://dbpedia.org/resource/Dina_Bonnevie, http://dbpedia.org/resource/Enrico_Villanueva, http://dbpedia.org/resource/Loren_Legarda, http://dbpedia.org/resource/Maria_Lourdes_Sereno]... [INFO] P: 5.173305742369374E-4, R: 1.0, F: 0.0010341261633919337 -[INFO] Cumulative metrics, sample: 84, P: 0.326, R: 0.385, macro F1: 0.324, macro F1*: 0.353 -[INFO] QALD Cumulative metrics, sample: 84, P: 0.540, R: 0.385, macro F1: 0.324, QALD macro F1: 0.450 -[INFO] Current time: 8511 ms, average: 3194.20 ms/ques, entity detection: 673.38 ms/ques (linking tool: 585.60 ms/ques), relation detection: 998.17 ms/ques (similarity: 572.93 ms/ques), EDG generation: 66.45 ms/ques +[INFO] Cumulative metrics, sample: 84, P: 0.326, R: 0.385, macro F1: 0.353 +[INFO] QALD Cumulative metrics, sample: 84, P: 0.540, R: 0.385, average F1: 0.324, QALD macro F1: 0.450 +[INFO] Current question consumed time: 7523 ms, average: 2980.36 ms/ques, entity detection: 613.85 ms/ques (linking tool: 535.70 ms/ques), relation detection: 952.86 ms/ques (similarity: 575.51 ms/ques), EDG generation: 67.12 ms/ques [INFO] Entity Linking P: 0.27380952380952384, R: 0.27380952380952384, micro F1: 0.27380952380952384, macro F1: 0.27380952380952384 [INFO] Relation Linking P: 0.2976190476190476, R: 0.2976190476190476, micro F1: 0.2976190476190476, macro F1: 0.2976190476190476 [INFO] Type Linking P: 0.9166666666666666, R: 0.9166666666666666, micro F1: 0.9166666666666666, macro F1: 0.9166666666666666 @@ -1267,9 +1267,9 @@ Question 84: Who were the parents of Queen Victoria? [INFO] Golden Answer: [http://dbpedia.org/resource/Princess_Victoria_of_Saxe-Coburg-Saalfeld, http://dbpedia.org/resource/Prince_Edward,_Duke_of_Kent_and_Strathearn] [INFO] Predicted Answer: [http://dbpedia.org/resource/Alfred,_Duke_of_Saxe-Coburg_and_Gotha, http://dbpedia.org/resource/Edward_VII, http://dbpedia.org/resource/Princess_Louise,_Duchess_of_Argyll, http://dbpedia.org/resource/Victoria,_Princess_Royal, http://dbpedia.org/resource/Prince_Arthur,_Duke_of_Connaught_and_Strathearn, http://dbpedia.org/resource/Princess_Beatrice_of_the_United_Kingdom, http://dbpedia.org/resource/Princess_Alice_of_the_United_Kingdom, http://dbpedia.org/resource/Princess_Helena_of_the_United_Kingdom, http://dbpedia.org/resource/Prince_Leopold,_Duke_of_Albany, http://dbpedia.org/resource/Princess_Victoria_of_Saxe-Coburg-Saalfeld, http://dbpedia.org/resource/Prince_Edward,_Duke_of_Kent_and_Strathearn] [INFO] P: 0.18181818181818182, R: 1.0, F: 0.3076923076923077 -[INFO] Cumulative metrics, sample: 85, P: 0.324, R: 0.392, macro F1: 0.324, macro F1*: 0.355 -[INFO] QALD Cumulative metrics, sample: 85, P: 0.536, R: 0.392, macro F1: 0.324, QALD macro F1: 0.453 -[INFO] Current time: 2084 ms, average: 3181.15 ms/ques, entity detection: 671.22 ms/ques (linking tool: 583.46 ms/ques), relation detection: 996.07 ms/ques (similarity: 571.13 ms/ques), EDG generation: 65.84 ms/ques +[INFO] Cumulative metrics, sample: 85, P: 0.324, R: 0.392, macro F1: 0.355 +[INFO] QALD Cumulative metrics, sample: 85, P: 0.536, R: 0.392, average F1: 0.324, QALD macro F1: 0.453 +[INFO] Current question consumed time: 1820 ms, average: 2966.71 ms/ques, entity detection: 611.55 ms/ques (linking tool: 533.52 ms/ques), relation detection: 950.66 ms/ques (similarity: 573.87 ms/ques), EDG generation: 66.45 ms/ques [INFO] Entity Linking P: 0.27058823529411763, R: 0.27058823529411763, micro F1: 0.27058823529411763, macro F1: 0.27058823529411763 [INFO] Relation Linking P: 0.29411764705882354, R: 0.29411764705882354, micro F1: 0.29411764705882354, macro F1: 0.29411764705882354 [INFO] Type Linking P: 0.9176470588235294, R: 0.9176470588235294, micro F1: 0.9176470588235294, macro F1: 0.9176470588235294 @@ -1282,9 +1282,9 @@ Question 85: Are there any castles in the United States? [INFO] Golden Answer: [true] [INFO] Predicted Answer: [true] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 86, P: 0.332, R: 0.400, macro F1: 0.332, macro F1*: 0.363 -[INFO] QALD Cumulative metrics, sample: 86, P: 0.541, R: 0.400, macro F1: 0.332, QALD macro F1: 0.460 -[INFO] Current time: 42331 ms, average: 3636.38 ms/ques, entity detection: 890.12 ms/ques (linking tool: 802.20 ms/ques), relation detection: 1140.17 ms/ques (similarity: 591.90 ms/ques), EDG generation: 65.27 ms/ques +[INFO] Cumulative metrics, sample: 86, P: 0.332, R: 0.400, macro F1: 0.363 +[INFO] QALD Cumulative metrics, sample: 86, P: 0.541, R: 0.400, average F1: 0.332, QALD macro F1: 0.460 +[INFO] Current question consumed time: 37742 ms, average: 3371.07 ms/ques, entity detection: 802.71 ms/ques (linking tool: 724.55 ms/ques), relation detection: 1080.71 ms/ques (similarity: 594.20 ms/ques), EDG generation: 65.83 ms/ques [INFO] Entity Linking P: 0.26744186046511625, R: 0.26744186046511625, micro F1: 0.26744186046511625, macro F1: 0.26744186046511625 [INFO] Relation Linking P: 0.29069767441860467, R: 0.29069767441860467, micro F1: 0.29069767441860467, macro F1: 0.29069767441860467 [INFO] Type Linking P: 0.9186046511627907, R: 0.9186046511627907, micro F1: 0.9186046511627907, macro F1: 0.9186046511627907 @@ -1297,9 +1297,9 @@ Question 86: Which languages are spoken in Estonia? [INFO] Golden Answer: [http://dbpedia.org/resource/Estonian_Sign_Language, http://dbpedia.org/resource/Estonian_language, http://dbpedia.org/resource/Modern_Swedish, http://dbpedia.org/resource/Võro_language, http://dbpedia.org/resource/Seto_dialect, http://dbpedia.org/resource/Russian_Sign_Language, http://dbpedia.org/resource/Laiuse_Romani_language, http://dbpedia.org/resource/Baltic_Romani] [INFO] Predicted Answer: [http://dbpedia.org/resource/Estonian_language, http://dbpedia.org/resource/Seto_language, http://dbpedia.org/resource/Võro_language] [INFO] P: 0.6666666666666666, R: 0.25, F: 0.36363636363636365 -[INFO] Cumulative metrics, sample: 87, P: 0.336, R: 0.398, macro F1: 0.332, macro F1*: 0.364 -[INFO] QALD Cumulative metrics, sample: 87, P: 0.543, R: 0.398, macro F1: 0.332, QALD macro F1: 0.459 -[INFO] Current time: 6213 ms, average: 3666.00 ms/ques, entity detection: 893.09 ms/ques (linking tool: 804.76 ms/ques), relation detection: 1152.32 ms/ques (similarity: 609.78 ms/ques), EDG generation: 64.62 ms/ques +[INFO] Cumulative metrics, sample: 87, P: 0.336, R: 0.398, macro F1: 0.364 +[INFO] QALD Cumulative metrics, sample: 87, P: 0.543, R: 0.398, average F1: 0.332, QALD macro F1: 0.459 +[INFO] Current question consumed time: 5609 ms, average: 3396.79 ms/ques, entity detection: 805.16 ms/ques (linking tool: 726.83 ms/ques), relation detection: 1091.10 ms/ques (similarity: 611.92 ms/ques), EDG generation: 65.07 ms/ques [INFO] Entity Linking P: 0.26436781609195403, R: 0.26436781609195403, micro F1: 0.26436781609195403, macro F1: 0.26436781609195403 [INFO] Relation Linking P: 0.28735632183908044, R: 0.28735632183908044, micro F1: 0.28735632183908044, macro F1: 0.28735632183908044 [INFO] Type Linking P: 0.9080459770114943, R: 0.9080459770114943, micro F1: 0.9080459770114943, macro F1: 0.9080459770114943 @@ -1312,9 +1312,9 @@ Question 87: Give me a list of all critically endangered birds. [INFO] Golden Answer: [http://dbpedia.org/resource/Fatu_Hiva_monarch, http://dbpedia.org/resource/Rota_white-eye, http://dbpedia.org/resource/Seychelles_paradise_flycatcher, http://dbpedia.org/resource/White-eyed_river_martin, http://dbpedia.org/resource/Giant_ibis, http://dbpedia.org/resource/Pink-headed_duck, http://dbpedia.org/resource/Okinawa_woodpecker, http://dbpedia.org/resource/Carrizal_seedeater, http://dbpedia.org/resource/Makira_woodhen, http://dbpedia.org/resource/Bahama_oriole, http://dbpedia.org/resource/Amsterdam_albatross, http://dbpedia.org/resource/Mauritius_olive_white-eye, http://dbpedia.org/resource/Bryan's_shearwater, http://dbpedia.org/resource/Tahiti_monarch, http://dbpedia.org/resource/Christmas_frigatebird, http://dbpedia.org/resource/Waved_albatross, http://dbpedia.org/resource/Medium_tree_finch, http://dbpedia.org/resource/White-bellied_heron, http://dbpedia.org/resource/Swift_parrot, http://dbpedia.org/resource/Blue-winged_racket-tail, http://dbpedia.org/resource/Sumatran_ground_cuckoo, http://dbpedia.org/resource/Chinese_crested_tern, http://dbpedia.org/resource/Sangihe_shrikethrush, http://dbpedia.org/resource/Floreana_mockingbird, http://dbpedia.org/resource/ʻAkohekohe, http://dbpedia.org/resource/Hoary-throated_spinetail, http://dbpedia.org/resource/Millerbird, http://dbpedia.org/resource/Blue-throated_macaw, http://dbpedia.org/resource/Uluguru_bushshrike, http://dbpedia.org/resource/Blue-banded_kingfisher, http://dbpedia.org/resource/Zapata_rail, http://dbpedia.org/resource/Banggai_crow, http://dbpedia.org/resource/Fiji_petrel, http://dbpedia.org/resource/Dusky_starfrontlet, http://dbpedia.org/resource/Stresemann's_bristlefront, http://dbpedia.org/resource/Socorro_mockingbird, http://dbpedia.org/resource/Siau_scops_owl, http://dbpedia.org/resource/São_Tomé_grosbeak, http://dbpedia.org/resource/Slender-billed_curlew, http://dbpedia.org/resource/Cherry-throated_tanager, http://dbpedia.org/resource/New_Caledonian_rail, http://dbpedia.org/resource/Brazilian_merganser, http://dbpedia.org/resource/Grenada_dove, http://dbpedia.org/resource/Cone-billed_tanager, http://dbpedia.org/resource/Mangrove_finch, http://dbpedia.org/resource/Taita_thrush, http://dbpedia.org/resource/Blue-fronted_lorikeet, http://dbpedia.org/resource/Ivory-billed_woodpecker, http://dbpedia.org/resource/Red-vented_cockatoo, http://dbpedia.org/resource/Indian_vulture, http://dbpedia.org/resource/Sociable_lapwing, http://dbpedia.org/resource/Red-headed_vulture, http://dbpedia.org/resource/Rüppell's_vulture, http://dbpedia.org/resource/Helmeted_hornbill, http://dbpedia.org/resource/Santa_Marta_wren, http://dbpedia.org/resource/Siberian_crane, http://dbpedia.org/resource/White-collared_kite, http://dbpedia.org/resource/Balearic_shearwater, http://dbpedia.org/resource/São_Tomé_fiscal, http://dbpedia.org/resource/Laysan_duck, http://dbpedia.org/resource/Taita_apalis, http://dbpedia.org/resource/Yellow-crested_cockatoo, http://dbpedia.org/resource/Red-throated_lorikeet, http://dbpedia.org/resource/Marquesan_kingfisher, http://dbpedia.org/resource/Antioquia_brush_finch, http://dbpedia.org/resource/Northern_bald_ibis, http://dbpedia.org/resource/Grey-breasted_parakeet, http://dbpedia.org/resource/Puaiohi, http://dbpedia.org/resource/Black_stilt, http://dbpedia.org/resource/Maui_parrotbill, http://dbpedia.org/resource/Kauaʻi_nukupuʻu, http://dbpedia.org/resource/Pernambuco_pygmy_owl, http://dbpedia.org/resource/Mangareva_kingfisher, http://dbpedia.org/resource/Juan_Fernández_firecrown, http://dbpedia.org/resource/Golden_white-eye, http://dbpedia.org/resource/Long-billed_forest_warbler, http://dbpedia.org/resource/White-winged_guan, http://dbpedia.org/resource/Magenta_petrel, http://dbpedia.org/resource/Coxen's_fig_parrot, http://dbpedia.org/resource/California_condor, http://dbpedia.org/resource/Sulu_hornbill, http://dbpedia.org/resource/Flores_hawk-eagle, http://dbpedia.org/resource/Alagoas_foliage-gleaner, http://dbpedia.org/resource/Nihoa_millerbird, http://dbpedia.org/resource/Spoon-billed_sandpiper, http://dbpedia.org/resource/São_Tomé_ibis, http://dbpedia.org/resource/Jerdon's_courser, http://dbpedia.org/resource/Raso_lark, http://dbpedia.org/resource/Colorful_puffleg, http://dbpedia.org/resource/Hooded_vulture, http://dbpedia.org/resource/Western_ground_parrot, http://dbpedia.org/resource/Réunion_cuckooshrike, http://dbpedia.org/resource/Black-hooded_coucal, http://dbpedia.org/resource/Tooth-billed_pigeon, http://dbpedia.org/resource/Chestnut-capped_piha, http://dbpedia.org/resource/Rio_Branco_antbird, http://dbpedia.org/resource/Sangihe_white-eye, http://dbpedia.org/resource/Jatayu_Conservation_Breeding_Centre,_Pinjore__CR__1, http://dbpedia.org/resource/Gough_finch, http://dbpedia.org/resource/Sulu_bleeding-heart, http://dbpedia.org/resource/Munchique_wood_wren, http://dbpedia.org/resource/New_Caledonian_owlet-nightjar, http://dbpedia.org/resource/‘Akeke‘e, http://dbpedia.org/resource/Black-breasted_puffleg, http://dbpedia.org/resource/Chatham_shag, http://dbpedia.org/resource/Isabela_oriole, http://dbpedia.org/resource/Javan_green_magpie, http://dbpedia.org/resource/Karthala_scops_owl, http://dbpedia.org/resource/Blue-billed_curassow, http://dbpedia.org/resource/Malherbe's_parakeet, http://dbpedia.org/resource/Mindoro_bleeding-heart, http://dbpedia.org/resource/Junin_grebe, http://dbpedia.org/resource/Chilean_woodstar, http://dbpedia.org/resource/Madagascan_fish_eagle, http://dbpedia.org/resource/Horned_curassow, http://dbpedia.org/resource/Cerulean_paradise_flycatcher, http://dbpedia.org/resource/Anjouan_scops_owl, http://dbpedia.org/resource/Great_Indian_bustard, http://dbpedia.org/resource/Masafuera_rayadito, http://dbpedia.org/resource/Nightingale_reed_warbler, http://dbpedia.org/resource/Baer's_pochard, http://dbpedia.org/resource/Edwards's_pheasant, http://dbpedia.org/resource/Gorgeted_puffleg, http://dbpedia.org/resource/Black-winged_starling, http://dbpedia.org/resource/Forest_owlet, http://dbpedia.org/resource/Walden's_hornbill, http://dbpedia.org/resource/Townsend's_shearwater, http://dbpedia.org/resource/Beck's_petrel, http://dbpedia.org/resource/Blue-bearded_helmetcrest, http://dbpedia.org/resource/White-backed_vulture, http://dbpedia.org/resource/White-rumped_vulture, http://dbpedia.org/resource/Sira_curassow, http://dbpedia.org/resource/ʻAkikiki, http://dbpedia.org/resource/Glaucous_macaw, http://dbpedia.org/resource/Bengal_florican, http://dbpedia.org/resource/Philippine_eagle, http://dbpedia.org/resource/Gran_Canaria_blue_chaffinch, http://dbpedia.org/resource/Cebu_flowerpecker, http://dbpedia.org/resource/Pohnpei_starling, http://dbpedia.org/resource/Black-chinned_monarch, http://dbpedia.org/resource/Blue-crowned_laughingthrush, http://dbpedia.org/resource/Djibouti_francolin, http://dbpedia.org/resource/Bali_myna, http://dbpedia.org/resource/Mariana_crow, http://dbpedia.org/resource/Bugun_liocichla, http://dbpedia.org/resource/Negros_fruit_dove, http://dbpedia.org/resource/Alagoas_antwren, http://dbpedia.org/resource/Palila, http://dbpedia.org/resource/Orange-bellied_parrot, http://dbpedia.org/resource/White-shouldered_ibis, http://dbpedia.org/resource/Rio_de_Janeiro_antwren, http://dbpedia.org/resource/Puerto_Rican_amazon, http://dbpedia.org/resource/Cozumel_thrasher, http://dbpedia.org/resource/Royal_cinclodes, http://dbpedia.org/resource/Semper's_warbler, http://dbpedia.org/resource/Silvery_pigeon, http://dbpedia.org/resource/Polynesian_ground_dove, http://dbpedia.org/resource/Nihoa_finch, http://dbpedia.org/resource/Madagascan_pochard, http://dbpedia.org/resource/Sapphire-bellied_hummingbird, http://dbpedia.org/resource/Bogotá_sunangel, http://dbpedia.org/resource/Niceforo's_wren, http://dbpedia.org/resource/Purple-winged_ground_dove, http://dbpedia.org/resource/Mascarene_petrel, http://dbpedia.org/resource/New_Caledonian_lorikeet, http://dbpedia.org/resource/Saipan_reed_warbler, http://dbpedia.org/resource/Ridgway's_hawk, http://dbpedia.org/resource/Slender-billed_vulture, http://dbpedia.org/resource/Maui_nukupuʻu, http://dbpedia.org/resource/Negros_bleeding-heart, http://dbpedia.org/resource/Crow_honeyeater, http://dbpedia.org/resource/Trinidad_piping_guan, http://dbpedia.org/resource/White-bellied_cinclodes, http://dbpedia.org/resource/Regent_honeyeater, http://dbpedia.org/resource/Moheli_scops_owl, http://dbpedia.org/resource/Tristan_albatross, http://dbpedia.org/resource/Saint_Helena_plover, http://dbpedia.org/resource/Montserrat_oriole, http://dbpedia.org/resource/Galápagos_petrel, http://dbpedia.org/resource/Cuban_kite, http://dbpedia.org/resource/Rück's_blue_flycatcher, http://dbpedia.org/resource/Blue-eyed_ground_dove, http://dbpedia.org/resource/Short-crested_coquette, http://dbpedia.org/resource/Iquitos_gnatcatcher, http://dbpedia.org/resource/Araripe_manakin, http://dbpedia.org/resource/White-chested_white-eye, http://dbpedia.org/resource/Chatham_pigeon, http://dbpedia.org/resource/Niau_kingfisher, http://dbpedia.org/resource/Hooded_grebe, http://dbpedia.org/resource/New_Zealand_storm_petrel, http://dbpedia.org/resource/Fuertes's_parrot, http://dbpedia.org/resource/Marañón_spinetail, http://dbpedia.org/resource/Liberian_greenbul, http://dbpedia.org/resource/Urrao_antpitta, http://dbpedia.org/resource/Príncipe_thrush, http://dbpedia.org/resource/Kinglet_calyptura, http://dbpedia.org/resource/White-headed_vulture, http://dbpedia.org/resource/White-winged_flufftail] [INFO] Predicted Answer: [] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 88, P: 0.332, R: 0.393, macro F1: 0.329, macro F1*: 0.360 -[INFO] QALD Cumulative metrics, sample: 88, P: 0.548, R: 0.393, macro F1: 0.329, QALD macro F1: 0.458 -[INFO] Current time: 818 ms, average: 3633.64 ms/ques, entity detection: 884.28 ms/ques (linking tool: 796.38 ms/ques), relation detection: 1139.84 ms/ques (similarity: 603.22 ms/ques), EDG generation: 63.98 ms/ques +[INFO] Cumulative metrics, sample: 88, P: 0.332, R: 0.393, macro F1: 0.360 +[INFO] QALD Cumulative metrics, sample: 88, P: 0.548, R: 0.393, average F1: 0.329, QALD macro F1: 0.458 +[INFO] Current question consumed time: 712 ms, average: 3366.28 ms/ques, entity detection: 797.16 ms/ques (linking tool: 719.24 ms/ques), relation detection: 1079.24 ms/ques (similarity: 605.32 ms/ques), EDG generation: 64.50 ms/ques [INFO] Entity Linking P: 0.2727272727272727, R: 0.2727272727272727, micro F1: 0.2727272727272727, macro F1: 0.2727272727272727 [INFO] Relation Linking P: 0.29545454545454547, R: 0.29545454545454547, micro F1: 0.29545454545454547, macro F1: 0.29545454545454547 [INFO] Type Linking P: 0.9090909090909091, R: 0.9090909090909091, micro F1: 0.9090909090909091, macro F1: 0.9090909090909091 @@ -1327,9 +1327,9 @@ Question 88: Which countries are connected by the Rhine? [INFO] Golden Answer: [http://dbpedia.org/resource/Liechtenstein, http://dbpedia.org/resource/Switzerland, http://dbpedia.org/resource/Kingdom_of_France, http://dbpedia.org/resource/Austria, http://dbpedia.org/resource/Netherlands, Germany] [INFO] Predicted Answer: [http://dbpedia.org/resource/Austria, http://dbpedia.org/resource/Kingdom_of_France, http://dbpedia.org/resource/Liechtenstein, http://dbpedia.org/resource/Netherlands, http://dbpedia.org/resource/Switzerland] [INFO] P: 1.0, R: 0.8333333333333334, F: 0.9090909090909091 -[INFO] Cumulative metrics, sample: 89, P: 0.339, R: 0.398, macro F1: 0.335, macro F1*: 0.366 -[INFO] QALD Cumulative metrics, sample: 89, P: 0.553, R: 0.398, macro F1: 0.335, QALD macro F1: 0.463 -[INFO] Current time: 3087 ms, average: 3627.49 ms/ques, entity detection: 878.21 ms/ques (linking tool: 789.97 ms/ques), relation detection: 1139.92 ms/ques (similarity: 610.11 ms/ques), EDG generation: 63.38 ms/ques +[INFO] Cumulative metrics, sample: 89, P: 0.339, R: 0.398, macro F1: 0.366 +[INFO] QALD Cumulative metrics, sample: 89, P: 0.553, R: 0.398, average F1: 0.335, QALD macro F1: 0.463 +[INFO] Current question consumed time: 2658 ms, average: 3358.33 ms/ques, entity detection: 791.35 ms/ques (linking tool: 713.43 ms/ques), relation detection: 1078.72 ms/ques (similarity: 612.40 ms/ques), EDG generation: 63.90 ms/ques [INFO] Entity Linking P: 0.2696629213483146, R: 0.2696629213483146, micro F1: 0.2696629213483146, macro F1: 0.2696629213483146 [INFO] Relation Linking P: 0.29213483146067415, R: 0.29213483146067415, micro F1: 0.29213483146067415, macro F1: 0.29213483146067415 [INFO] Type Linking P: 0.898876404494382, R: 0.898876404494382, micro F1: 0.898876404494382, macro F1: 0.898876404494382 @@ -1342,9 +1342,9 @@ Question 89: Who was the father of Queen Elizabeth II? [INFO] Golden Answer: [http://dbpedia.org/resource/George_VI] [INFO] Predicted Answer: [] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 90, P: 0.336, R: 0.394, macro F1: 0.331, macro F1*: 0.362 -[INFO] QALD Cumulative metrics, sample: 90, P: 0.558, R: 0.394, macro F1: 0.331, QALD macro F1: 0.462 -[INFO] Current time: 1048 ms, average: 3598.83 ms/ques, entity detection: 872.78 ms/ques (linking tool: 784.91 ms/ques), relation detection: 1128.16 ms/ques (similarity: 603.84 ms/ques), EDG generation: 62.78 ms/ques +[INFO] Cumulative metrics, sample: 90, P: 0.336, R: 0.394, macro F1: 0.362 +[INFO] QALD Cumulative metrics, sample: 90, P: 0.558, R: 0.394, average F1: 0.331, QALD macro F1: 0.462 +[INFO] Current question consumed time: 895 ms, average: 3330.96 ms/ques, entity detection: 786.21 ms/ques (linking tool: 708.62 ms/ques), relation detection: 1067.59 ms/ques (similarity: 606.00 ms/ques), EDG generation: 63.34 ms/ques [INFO] Entity Linking P: 0.2777777777777778, R: 0.2777777777777778, micro F1: 0.2777777777777778, macro F1: 0.2777777777777778 [INFO] Relation Linking P: 0.3, R: 0.3, micro F1: 0.3, macro F1: 0.3 [INFO] Type Linking P: 0.9, R: 0.9, micro F1: 0.9, macro F1: 0.9 @@ -1357,9 +1357,9 @@ Question 90: Give me all chemical elements. [INFO] Golden Answer: [http://dbpedia.org/resource/Hassium, http://dbpedia.org/resource/Osmium, http://dbpedia.org/resource/Rhenium, http://dbpedia.org/resource/Actinium, http://dbpedia.org/resource/Aluminium, http://dbpedia.org/resource/Americium, http://dbpedia.org/resource/Antimony, http://dbpedia.org/resource/Argon, http://dbpedia.org/resource/Arsenic, http://dbpedia.org/resource/Astatine, http://dbpedia.org/resource/Barium, http://dbpedia.org/resource/Berkelium, http://dbpedia.org/resource/Beryllium, http://dbpedia.org/resource/Bismuth, http://dbpedia.org/resource/Boron, http://dbpedia.org/resource/Bromine, http://dbpedia.org/resource/Cadmium, http://dbpedia.org/resource/Caesium, http://dbpedia.org/resource/Calcium, http://dbpedia.org/resource/Carbon, http://dbpedia.org/resource/Cerium, http://dbpedia.org/resource/Chemical_element, http://dbpedia.org/resource/Chlorine, http://dbpedia.org/resource/Chromium, http://dbpedia.org/resource/Cobalt, http://dbpedia.org/resource/Copper, http://dbpedia.org/resource/Curium, http://dbpedia.org/resource/Dubnium, http://dbpedia.org/resource/Einsteinium, http://dbpedia.org/resource/Europium, http://dbpedia.org/resource/Fermium, http://dbpedia.org/resource/Fluorine, http://dbpedia.org/resource/Francium, http://dbpedia.org/resource/Gallium, http://dbpedia.org/resource/Germanium, http://dbpedia.org/resource/Gold, http://dbpedia.org/resource/Hafnium, http://dbpedia.org/resource/Helium, http://dbpedia.org/resource/Hydrogen, http://dbpedia.org/resource/Indium, http://dbpedia.org/resource/Iodine, http://dbpedia.org/resource/Iridium, http://dbpedia.org/resource/Iron, http://dbpedia.org/resource/Krypton, http://dbpedia.org/resource/Lanthanum, http://dbpedia.org/resource/Lawrencium, http://dbpedia.org/resource/Lead, http://dbpedia.org/resource/Lithium, http://dbpedia.org/resource/Lutetium, http://dbpedia.org/resource/Magnesium, http://dbpedia.org/resource/Manganese, http://dbpedia.org/resource/Mercury_(element), http://dbpedia.org/resource/Molybdenum, http://dbpedia.org/resource/Neodymium, http://dbpedia.org/resource/Neon, http://dbpedia.org/resource/Neptunium, http://dbpedia.org/resource/Nickel, http://dbpedia.org/resource/Niobium, http://dbpedia.org/resource/Nitrogen, http://dbpedia.org/resource/Nobelium, http://dbpedia.org/resource/Oxygen, http://dbpedia.org/resource/Palladium, http://dbpedia.org/resource/Phosphorus, http://dbpedia.org/resource/Platinum, http://dbpedia.org/resource/Plutonium, http://dbpedia.org/resource/Polonium, http://dbpedia.org/resource/Potassium, http://dbpedia.org/resource/Praseodymium, http://dbpedia.org/resource/Protactinium, http://dbpedia.org/resource/Radium, http://dbpedia.org/resource/Radon, http://dbpedia.org/resource/Rhodium, http://dbpedia.org/resource/Rubidium, http://dbpedia.org/resource/Selenium, http://dbpedia.org/resource/Silicon, http://dbpedia.org/resource/Silver, http://dbpedia.org/resource/Sodium, http://dbpedia.org/resource/Strontium, http://dbpedia.org/resource/Sulfur, http://dbpedia.org/resource/Tantalum, http://dbpedia.org/resource/Technetium, http://dbpedia.org/resource/Tellurium, http://dbpedia.org/resource/Thallium, http://dbpedia.org/resource/Thorium, http://dbpedia.org/resource/Tin, http://dbpedia.org/resource/Titanium, http://dbpedia.org/resource/Tungsten, http://dbpedia.org/resource/Ununoctium, http://dbpedia.org/resource/Uranium, http://dbpedia.org/resource/Vanadium, http://dbpedia.org/resource/Xenon, http://dbpedia.org/resource/Ytterbium, http://dbpedia.org/resource/Zinc, http://dbpedia.org/resource/Zirconium, http://dbpedia.org/resource/Erbium, http://dbpedia.org/resource/Holmium, http://dbpedia.org/resource/Mendelevium, http://dbpedia.org/resource/Rutherfordium, http://dbpedia.org/resource/Thulium, http://dbpedia.org/resource/Californium, http://dbpedia.org/resource/Dysprosium, http://dbpedia.org/resource/Gadolinium, http://dbpedia.org/resource/Meitnerium, http://dbpedia.org/resource/Samarium, http://dbpedia.org/resource/Terbium, http://dbpedia.org/resource/Ruthenium, http://dbpedia.org/resource/Bohrium, http://dbpedia.org/resource/Ununseptium, http://dbpedia.org/resource/Copernicium, http://dbpedia.org/resource/Darmstadtium, http://dbpedia.org/resource/Flerovium, http://dbpedia.org/resource/Roentgenium, http://dbpedia.org/resource/Scandium, http://dbpedia.org/resource/Seaborgium, http://dbpedia.org/resource/The_Mystery_of_Matter_(film), http://dbpedia.org/resource/Ununpentium, http://dbpedia.org/resource/Ununtrium, http://dbpedia.org/resource/Yttrium, http://dbpedia.org/resource/Symbol_(chemistry)] [INFO] Predicted Answer: [http://dbpedia.org/resource/Lost_Gravity_(roller_coaster)__*Dive_loop_*Zero-G_Roll__1] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 91, P: 0.332, R: 0.389, macro F1: 0.328, macro F1*: 0.358 -[INFO] QALD Cumulative metrics, sample: 91, P: 0.552, R: 0.389, macro F1: 0.328, QALD macro F1: 0.457 -[INFO] Current time: 387 ms, average: 3563.54 ms/ques, entity detection: 863.19 ms/ques (linking tool: 776.29 ms/ques), relation detection: 1116.87 ms/ques (similarity: 597.92 ms/ques), EDG generation: 62.16 ms/ques +[INFO] Cumulative metrics, sample: 91, P: 0.332, R: 0.389, macro F1: 0.358 +[INFO] QALD Cumulative metrics, sample: 91, P: 0.552, R: 0.389, average F1: 0.328, QALD macro F1: 0.457 +[INFO] Current question consumed time: 323 ms, average: 3297.90 ms/ques, entity detection: 777.57 ms/ques (linking tool: 700.84 ms/ques), relation detection: 1056.89 ms/ques (similarity: 600.04 ms/ques), EDG generation: 62.84 ms/ques [INFO] Entity Linking P: 0.2857142857142857, R: 0.2857142857142857, micro F1: 0.2857142857142857, macro F1: 0.2857142857142857 [INFO] Relation Linking P: 0.3076923076923077, R: 0.3076923076923077, micro F1: 0.3076923076923077, macro F1: 0.3076923076923077 [INFO] Type Linking P: 0.8901098901098901, R: 0.8901098901098901, micro F1: 0.8901098901098901, macro F1: 0.8901098901098901 @@ -1372,9 +1372,9 @@ Question 91: Give me all American presidents of the last 20 years. [INFO] Golden Answer: [http://dbpedia.org/resource/Barack_Obama, http://dbpedia.org/resource/Bill_Clinton, http://dbpedia.org/resource/George_W._Bush] [INFO] Predicted Answer: [] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 92, P: 0.328, R: 0.385, macro F1: 0.324, macro F1*: 0.354 -[INFO] QALD Cumulative metrics, sample: 92, P: 0.557, R: 0.385, macro F1: 0.324, QALD macro F1: 0.455 -[INFO] Current time: 1084 ms, average: 3536.60 ms/ques, entity detection: 854.75 ms/ques (linking tool: 768.24 ms/ques), relation detection: 1108.50 ms/ques (similarity: 593.89 ms/ques), EDG generation: 61.71 ms/ques +[INFO] Cumulative metrics, sample: 92, P: 0.328, R: 0.385, macro F1: 0.354 +[INFO] QALD Cumulative metrics, sample: 92, P: 0.557, R: 0.385, average F1: 0.324, QALD macro F1: 0.455 +[INFO] Current question consumed time: 948 ms, average: 3272.36 ms/ques, entity detection: 769.98 ms/ques (linking tool: 693.55 ms/ques), relation detection: 1048.80 ms/ques (similarity: 595.89 ms/ques), EDG generation: 62.38 ms/ques [INFO] Entity Linking P: 0.29347826086956524, R: 0.29347826086956524, micro F1: 0.29347826086956524, macro F1: 0.29347826086956524 [INFO] Relation Linking P: 0.31521739130434784, R: 0.31521739130434784, micro F1: 0.31521739130434784, macro F1: 0.31521739130434784 [INFO] Type Linking P: 0.8913043478260869, R: 0.8913043478260869, micro F1: 0.8913043478260869, macro F1: 0.8913043478260869 @@ -1387,9 +1387,9 @@ Question 92: When is the movie Worst Case Scenario going to be in cinemas in the [INFO] Golden Answer: [2008-01-30] [INFO] Predicted Answer: [2015] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 93, P: 0.325, R: 0.381, macro F1: 0.321, macro F1*: 0.351 -[INFO] QALD Cumulative metrics, sample: 93, P: 0.551, R: 0.381, macro F1: 0.321, QALD macro F1: 0.450 -[INFO] Current time: 13296 ms, average: 3641.54 ms/ques, entity detection: 880.90 ms/ques (linking tool: 793.51 ms/ques), relation detection: 1149.02 ms/ques (similarity: 613.03 ms/ques), EDG generation: 61.33 ms/ques +[INFO] Cumulative metrics, sample: 93, P: 0.325, R: 0.381, macro F1: 0.351 +[INFO] QALD Cumulative metrics, sample: 93, P: 0.551, R: 0.381, average F1: 0.321, QALD macro F1: 0.450 +[INFO] Current question consumed time: 11459 ms, average: 3360.39 ms/ques, entity detection: 791.80 ms/ques (linking tool: 714.82 ms/ques), relation detection: 1085.44 ms/ques (similarity: 615.42 ms/ques), EDG generation: 61.97 ms/ques [INFO] Entity Linking P: 0.2903225806451613, R: 0.2903225806451613, micro F1: 0.2903225806451613, macro F1: 0.2903225806451613 [INFO] Relation Linking P: 0.3118279569892473, R: 0.3118279569892473, micro F1: 0.3118279569892473, macro F1: 0.3118279569892473 [INFO] Type Linking P: 0.8924731182795699, R: 0.8924731182795699, micro F1: 0.8924731182795699, macro F1: 0.8924731182795699 @@ -1402,9 +1402,9 @@ Question 93: Whom did Lance Bass marry? [INFO] Golden Answer: [http://dbpedia.org/resource/Michael_Turchin] [INFO] Predicted Answer: [http://dbpedia.org/resource/Michael_Turchin, 2014, Michael Turchin] [INFO] P: 0.5, R: 1.0, F: 0.6666666666666666 -[INFO] Cumulative metrics, sample: 94, P: 0.327, R: 0.388, macro F1: 0.324, macro F1*: 0.355 -[INFO] QALD Cumulative metrics, sample: 94, P: 0.550, R: 0.388, macro F1: 0.324, QALD macro F1: 0.455 -[INFO] Current time: 1014 ms, average: 3613.59 ms/ques, entity detection: 873.43 ms/ques (linking tool: 786.38 ms/ques), relation detection: 1140.38 ms/ques (similarity: 608.61 ms/ques), EDG generation: 60.76 ms/ques +[INFO] Cumulative metrics, sample: 94, P: 0.327, R: 0.388, macro F1: 0.355 +[INFO] QALD Cumulative metrics, sample: 94, P: 0.550, R: 0.388, average F1: 0.324, QALD macro F1: 0.455 +[INFO] Current question consumed time: 801 ms, average: 3333.16 ms/ques, entity detection: 784.88 ms/ques (linking tool: 708.37 ms/ques), relation detection: 1076.71 ms/ques (similarity: 610.87 ms/ques), EDG generation: 61.38 ms/ques [INFO] Entity Linking P: 0.2872340425531915, R: 0.2872340425531915, micro F1: 0.2872340425531915, macro F1: 0.2872340425531915 [INFO] Relation Linking P: 0.30851063829787234, R: 0.30851063829787234, micro F1: 0.30851063829787234, macro F1: 0.30851063829787234 [INFO] Type Linking P: 0.8936170212765957, R: 0.8936170212765957, micro F1: 0.8936170212765957, macro F1: 0.8936170212765957 @@ -1417,9 +1417,9 @@ Question 94: Which subsidiary of TUI Travel serves both Glasgow and Dublin? [INFO] Golden Answer: [http://dbpedia.org/resource/Thomson_Airways] [INFO] Predicted Answer: [http://dbpedia.org/resource/Thomson_Holidays, http://dbpedia.org/resource/Thomson_Airways] [INFO] P: 0.5, R: 1.0, F: 0.6666666666666666 -[INFO] Cumulative metrics, sample: 95, P: 0.328, R: 0.394, macro F1: 0.328, macro F1*: 0.358 -[INFO] QALD Cumulative metrics, sample: 95, P: 0.549, R: 0.394, macro F1: 0.328, QALD macro F1: 0.459 -[INFO] Current time: 17890 ms, average: 3763.86 ms/ques, entity detection: 893.41 ms/ques (linking tool: 804.05 ms/ques), relation detection: 1216.52 ms/ques (similarity: 648.86 ms/ques), EDG generation: 60.47 ms/ques +[INFO] Cumulative metrics, sample: 95, P: 0.328, R: 0.394, macro F1: 0.358 +[INFO] QALD Cumulative metrics, sample: 95, P: 0.549, R: 0.394, average F1: 0.328, QALD macro F1: 0.459 +[INFO] Current question consumed time: 15110 ms, average: 3457.13 ms/ques, entity detection: 800.79 ms/ques (linking tool: 722.69 ms/ques), relation detection: 1144.65 ms/ques (similarity: 651.24 ms/ques), EDG generation: 61.06 ms/ques [INFO] Entity Linking P: 0.28421052631578947, R: 0.28421052631578947, micro F1: 0.28421052631578947, macro F1: 0.28421052631578947 [INFO] Relation Linking P: 0.30526315789473685, R: 0.30526315789473685, micro F1: 0.30526315789473685, macro F1: 0.30526315789473685 [INFO] Type Linking P: 0.8947368421052632, R: 0.8947368421052632, micro F1: 0.8947368421052632, macro F1: 0.8947368421052632 @@ -1432,9 +1432,9 @@ Question 95: What languages are spoken in Pakistan? [INFO] Golden Answer: [http://dbpedia.org/resource/Khowar_language, http://dbpedia.org/resource/Shina_language, http://dbpedia.org/resource/Burushaski, http://dbpedia.org/resource/Balochi_language, http://dbpedia.org/resource/Brahui_language, http://dbpedia.org/resource/English_language, http://dbpedia.org/resource/Kashmiri_language, http://dbpedia.org/resource/Pashto_language, http://dbpedia.org/resource/Punjabi_language, http://dbpedia.org/resource/Sindhi_language, http://dbpedia.org/resource/Urdu, http://dbpedia.org/resource/Kalash_language, http://dbpedia.org/resource/Balti_language, http://dbpedia.org/resource/Domaaki_language, http://dbpedia.org/resource/Dameli_language, http://dbpedia.org/resource/Gawar-Bati_language] [INFO] Predicted Answer: [http://dbpedia.org/resource/Khowar_language, http://dbpedia.org/resource/Shina_language, http://dbpedia.org/resource/Burushaski, http://dbpedia.org/resource/Balochi_language, http://dbpedia.org/resource/Brahui_language, http://dbpedia.org/resource/English_language, http://dbpedia.org/resource/Kashmiri_language, http://dbpedia.org/resource/Pashto_language, http://dbpedia.org/resource/Punjabi_language, http://dbpedia.org/resource/Sindhi_language, http://dbpedia.org/resource/Urdu, http://dbpedia.org/resource/Kalash_language, http://dbpedia.org/resource/Balti_language, http://dbpedia.org/resource/Domaaki_language, http://dbpedia.org/resource/Dameli_language, http://dbpedia.org/resource/Gawar-Bati_language] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 96, P: 0.335, R: 0.400, macro F1: 0.335, macro F1*: 0.365 -[INFO] QALD Cumulative metrics, sample: 96, P: 0.554, R: 0.400, macro F1: 0.335, QALD macro F1: 0.465 -[INFO] Current time: 8718 ms, average: 3815.47 ms/ques, entity detection: 903.46 ms/ques (linking tool: 813.79 ms/ques), relation detection: 1234.27 ms/ques (similarity: 670.84 ms/ques), EDG generation: 59.94 ms/ques +[INFO] Cumulative metrics, sample: 96, P: 0.335, R: 0.400, macro F1: 0.365 +[INFO] QALD Cumulative metrics, sample: 96, P: 0.554, R: 0.400, average F1: 0.335, QALD macro F1: 0.465 +[INFO] Current question consumed time: 7399 ms, average: 3498.19 ms/ques, entity detection: 809.04 ms/ques (linking tool: 730.79 ms/ques), relation detection: 1158.71 ms/ques (similarity: 672.85 ms/ques), EDG generation: 60.43 ms/ques [INFO] Entity Linking P: 0.28125, R: 0.28125, micro F1: 0.28125, macro F1: 0.28125 [INFO] Relation Linking P: 0.3020833333333333, R: 0.3020833333333333, micro F1: 0.3020833333333333, macro F1: 0.3020833333333333 [INFO] Type Linking P: 0.8854166666666666, R: 0.8854166666666666, micro F1: 0.8854166666666666, macro F1: 0.8854166666666666 @@ -1447,9 +1447,9 @@ Question 96: Who is the daughter of Bill Clinton married to? [INFO] Golden Answer: [http://dbpedia.org/resource/Marc_Mezvinsky] [INFO] Predicted Answer: [] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 97, P: 0.332, R: 0.396, macro F1: 0.331, macro F1*: 0.361 -[INFO] QALD Cumulative metrics, sample: 97, P: 0.559, R: 0.396, macro F1: 0.331, QALD macro F1: 0.464 -[INFO] Current time: 1457 ms, average: 3791.15 ms/ques, entity detection: 900.33 ms/ques (linking tool: 811.03 ms/ques), relation detection: 1222.68 ms/ques (similarity: 664.76 ms/ques), EDG generation: 59.48 ms/ques +[INFO] Cumulative metrics, sample: 97, P: 0.332, R: 0.396, macro F1: 0.361 +[INFO] QALD Cumulative metrics, sample: 97, P: 0.559, R: 0.396, average F1: 0.331, QALD macro F1: 0.464 +[INFO] Current question consumed time: 1264 ms, average: 3475.15 ms/ques, entity detection: 806.11 ms/ques (linking tool: 728.25 ms/ques), relation detection: 1147.77 ms/ques (similarity: 666.66 ms/ques), EDG generation: 59.97 ms/ques [INFO] Entity Linking P: 0.28865979381443296, R: 0.28865979381443296, micro F1: 0.28865979381443296, macro F1: 0.28865979381443296 [INFO] Relation Linking P: 0.30927835051546393, R: 0.30927835051546393, micro F1: 0.30927835051546393, macro F1: 0.30927835051546393 [INFO] Type Linking P: 0.8865979381443299, R: 0.8865979381443299, micro F1: 0.8865979381443299, macro F1: 0.8865979381443299 @@ -1462,9 +1462,9 @@ Question 97: Who was on the Apollo 11 mission? [INFO] Golden Answer: [http://dbpedia.org/resource/Buzz_Aldrin, http://dbpedia.org/resource/Michael_Collins_(astronaut), http://dbpedia.org/resource/Neil_Armstrong] [INFO] Predicted Answer: [http://dbpedia.org/resource/Buzz_Aldrin, http://dbpedia.org/resource/Michael_Collins_(astronaut), http://dbpedia.org/resource/Neil_Armstrong] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 98, P: 0.339, R: 0.402, macro F1: 0.338, macro F1*: 0.368 -[INFO] QALD Cumulative metrics, sample: 98, P: 0.563, R: 0.402, macro F1: 0.338, QALD macro F1: 0.469 -[INFO] Current time: 1392 ms, average: 3766.67 ms/ques, entity detection: 892.72 ms/ques (linking tool: 804.13 ms/ques), relation detection: 1216.70 ms/ques (similarity: 661.93 ms/ques), EDG generation: 58.97 ms/ques +[INFO] Cumulative metrics, sample: 98, P: 0.339, R: 0.402, macro F1: 0.368 +[INFO] QALD Cumulative metrics, sample: 98, P: 0.563, R: 0.402, average F1: 0.338, QALD macro F1: 0.469 +[INFO] Current question consumed time: 1209 ms, average: 3452.03 ms/ques, entity detection: 799.17 ms/ques (linking tool: 721.94 ms/ques), relation detection: 1142.00 ms/ques (similarity: 663.77 ms/ques), EDG generation: 59.47 ms/ques [INFO] Entity Linking P: 0.2857142857142857, R: 0.2857142857142857, micro F1: 0.2857142857142857, macro F1: 0.2857142857142857 [INFO] Relation Linking P: 0.30612244897959184, R: 0.30612244897959184, micro F1: 0.30612244897959184, macro F1: 0.30612244897959184 [INFO] Type Linking P: 0.8877551020408163, R: 0.8877551020408163, micro F1: 0.8877551020408163, macro F1: 0.8877551020408163 @@ -1477,9 +1477,9 @@ Question 98: What is the wavelength of Indigo? [INFO] Golden Answer: [4.5E-7] [INFO] Predicted Answer: [450, 4.5E-7] [INFO] P: 0.5, R: 1.0, F: 0.6666666666666666 -[INFO] Cumulative metrics, sample: 99, P: 0.340, R: 0.408, macro F1: 0.342, macro F1*: 0.371 -[INFO] QALD Cumulative metrics, sample: 99, P: 0.563, R: 0.408, macro F1: 0.342, QALD macro F1: 0.473 -[INFO] Current time: 1620 ms, average: 3744.99 ms/ques, entity detection: 886.84 ms/ques (linking tool: 797.64 ms/ques), relation detection: 1209.60 ms/ques (similarity: 658.34 ms/ques), EDG generation: 58.47 ms/ques +[INFO] Cumulative metrics, sample: 99, P: 0.340, R: 0.408, macro F1: 0.371 +[INFO] QALD Cumulative metrics, sample: 99, P: 0.563, R: 0.408, average F1: 0.342, QALD macro F1: 0.473 +[INFO] Current question consumed time: 1389 ms, average: 3431.19 ms/ques, entity detection: 793.75 ms/ques (linking tool: 716.10 ms/ques), relation detection: 1135.24 ms/ques (similarity: 660.18 ms/ques), EDG generation: 58.98 ms/ques [INFO] Entity Linking P: 0.2828282828282828, R: 0.2828282828282828, micro F1: 0.2828282828282828, macro F1: 0.2828282828282828 [INFO] Relation Linking P: 0.30303030303030304, R: 0.30303030303030304, micro F1: 0.30303030303030304, macro F1: 0.30303030303030304 [INFO] Type Linking P: 0.8888888888888888, R: 0.8888888888888888, micro F1: 0.8888888888888888, macro F1: 0.8888888888888888 @@ -1492,9 +1492,9 @@ Question 99: Give me all B-sides of the Ramones. [INFO] Golden Answer: [\"Babysitter\", \"I Don't Care\", \"Daytime Dilemma (Dangers of Love)\", \"Go Home Ann\" (12\" only), \"Gee Girl\", \"Miss Joan And Mr. Sam\", \"California Sun\" / \"I Dont Wanna Walk Around With You\" (live), \"Do You Wanna Dance?\", \"No-One Could Ever Love You More\" (Norway), \"Saturday Night Out\", Censorshit] [INFO] Predicted Answer: [http://dbpedia.org/resource/Joey_Ramone, http://dbpedia.org/resource/Richie_Ramone, http://dbpedia.org/resource/Richard_Hell_and_the_Voidoids, http://dbpedia.org/resource/Misfits_(band), http://dbpedia.org/resource/Ed_Stasium, http://dbpedia.org/resource/Clem_Burke, http://dbpedia.org/resource/The_Heartbreakers, http://dbpedia.org/resource/The_Paley_Brothers, http://dbpedia.org/resource/Dust_(band), http://dbpedia.org/resource/White_Trash_Debutantes, http://dbpedia.org/resource/Garrett_Uhlenbrock, http://dbpedia.org/resource/C._J._Ramone, http://dbpedia.org/resource/Dee_Dee_Ramone, http://dbpedia.org/resource/Jed_Davis, http://dbpedia.org/resource/Johnny_Ramone, http://dbpedia.org/resource/Marky_Ramone, http://dbpedia.org/resource/Sniper_(American_band), http://dbpedia.org/resource/Tommy_Ramone, http://dbpedia.org/resource/Osaka_Popstar, http://dbpedia.org/resource/D-A-D, http://dbpedia.org/resource/Dee_Dee_Ramone_and_the_Chinese_Dragons] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 100, P: 0.337, R: 0.404, macro F1: 0.338, macro F1*: 0.368 -[INFO] QALD Cumulative metrics, sample: 100, P: 0.557, R: 0.404, macro F1: 0.338, QALD macro F1: 0.469 -[INFO] Current time: 1827 ms, average: 3725.81 ms/ques, entity detection: 881.01 ms/ques (linking tool: 791.78 ms/ques), relation detection: 1201.90 ms/ques (similarity: 654.29 ms/ques), EDG generation: 58.05 ms/ques +[INFO] Cumulative metrics, sample: 100, P: 0.337, R: 0.404, macro F1: 0.368 +[INFO] QALD Cumulative metrics, sample: 100, P: 0.557, R: 0.404, average F1: 0.338, QALD macro F1: 0.469 +[INFO] Current question consumed time: 1528 ms, average: 3412.16 ms/ques, entity detection: 788.13 ms/ques (linking tool: 710.51 ms/ques), relation detection: 1127.91 ms/ques (similarity: 656.15 ms/ques), EDG generation: 58.58 ms/ques [INFO] Entity Linking P: 0.28, R: 0.28, micro F1: 0.28, macro F1: 0.28 [INFO] Relation Linking P: 0.3, R: 0.3, micro F1: 0.3, macro F1: 0.3 [INFO] Type Linking P: 0.89, R: 0.89, micro F1: 0.89, macro F1: 0.89 @@ -1507,9 +1507,9 @@ Question 100: Who was called Scarface? [INFO] Golden Answer: [http://dbpedia.org/resource/Al_Capone, http://dbpedia.org/resource/Scarface_(rapper)] [INFO] Predicted Answer: [http://dbpedia.org/resource/Napoleon_(rapper), http://dbpedia.org/resource/Willie_D, http://dbpedia.org/resource/Beanie_Sigel, http://dbpedia.org/resource/DMG_(rapper), http://dbpedia.org/resource/Snypaz, http://dbpedia.org/resource/Ghetto_Twiinz, http://dbpedia.org/resource/Menace_Clan, http://dbpedia.org/resource/Devin_the_Dude, http://dbpedia.org/resource/Nottz, http://dbpedia.org/resource/Vinnie_Paz, http://dbpedia.org/resource/Seagram_(rapper), http://dbpedia.org/resource/Skull_Duggery_(rapper), http://dbpedia.org/resource/B-Legit, http://dbpedia.org/resource/Facemob, http://dbpedia.org/resource/Z-Ro, http://dbpedia.org/resource/Hussein_Fatal, http://dbpedia.org/resource/Do_or_Die_(group), http://dbpedia.org/resource/Trick_Daddy, http://dbpedia.org/resource/Criminal_Manne, http://dbpedia.org/resource/N.O.R.E., http://dbpedia.org/resource/Papa_Reu, http://dbpedia.org/resource/Ray_Cash, http://dbpedia.org/resource/Trae_tha_Truth, http://dbpedia.org/resource/Yukmouth, http://dbpedia.org/resource/Outlawz, http://dbpedia.org/resource/Ice_Cube, http://dbpedia.org/resource/Master_P, http://dbpedia.org/resource/Nas, http://dbpedia.org/resource/DJ_Muggs, http://dbpedia.org/resource/Geto_Boys, http://dbpedia.org/resource/Gang_starr, http://dbpedia.org/resource/2Pac, http://dbpedia.org/resource/Too_Short, http://dbpedia.org/resource/UGK, http://dbpedia.org/resource/UTP_(group)] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 101, P: 0.334, R: 0.400, macro F1: 0.335, macro F1*: 0.364 -[INFO] QALD Cumulative metrics, sample: 101, P: 0.551, R: 0.400, macro F1: 0.335, QALD macro F1: 0.464 -[INFO] Current time: 1329 ms, average: 3702.08 ms/ques, entity detection: 874.77 ms/ques (linking tool: 785.44 ms/ques), relation detection: 1193.29 ms/ques (similarity: 649.60 ms/ques), EDG generation: 57.53 ms/ques +[INFO] Cumulative metrics, sample: 101, P: 0.334, R: 0.400, macro F1: 0.364 +[INFO] QALD Cumulative metrics, sample: 101, P: 0.551, R: 0.400, average F1: 0.335, QALD macro F1: 0.464 +[INFO] Current question consumed time: 1141 ms, average: 3389.67 ms/ques, entity detection: 782.40 ms/ques (linking tool: 704.77 ms/ques), relation detection: 1119.72 ms/ques (similarity: 651.45 ms/ques), EDG generation: 58.06 ms/ques [INFO] Entity Linking P: 0.27722772277227725, R: 0.27722772277227725, micro F1: 0.27722772277227725, macro F1: 0.27722772277227725 [INFO] Relation Linking P: 0.297029702970297, R: 0.297029702970297, micro F1: 0.297029702970297, macro F1: 0.297029702970297 [INFO] Type Linking P: 0.8910891089108911, R: 0.8910891089108911, micro F1: 0.8910891089108911, macro F1: 0.8910891089108911 @@ -1522,9 +1522,9 @@ Question 101: Which rivers flow into the North Sea? [INFO] Golden Answer: [http://dbpedia.org/resource/Oude_Rijn_(Utrecht_and_South_Holland), http://dbpedia.org/resource/River_Tay, http://dbpedia.org/resource/Elliot_Water, http://dbpedia.org/resource/Elbe, http://dbpedia.org/resource/North_Sea_Canal, http://dbpedia.org/resource/Rhine, http://dbpedia.org/resource/Ems_(river), http://dbpedia.org/resource/Weser, http://dbpedia.org/resource/River_Forth, http://dbpedia.org/resource/River_Stour,_Suffolk, http://dbpedia.org/resource/River_Thames, http://dbpedia.org/resource/River_Tweed, http://dbpedia.org/resource/Asheldham_Brook, http://dbpedia.org/resource/Eider_(river), http://dbpedia.org/resource/River_South_Esk, http://dbpedia.org/resource/River_Crouch, http://dbpedia.org/resource/Barry_Burn] [INFO] Predicted Answer: [http://dbpedia.org/resource/Oude_Rijn_(Utrecht_and_South_Holland), http://dbpedia.org/resource/River_Tay, http://dbpedia.org/resource/Elliot_Water, http://dbpedia.org/resource/Elbe, http://dbpedia.org/resource/North_Sea_Canal, http://dbpedia.org/resource/Rhine, http://dbpedia.org/resource/Ems_(river), http://dbpedia.org/resource/Weser, http://dbpedia.org/resource/River_Forth, http://dbpedia.org/resource/River_Stour,_Suffolk, http://dbpedia.org/resource/River_Thames, http://dbpedia.org/resource/River_Tweed, http://dbpedia.org/resource/Asheldham_Brook, http://dbpedia.org/resource/Eider_(river), http://dbpedia.org/resource/River_South_Esk, http://dbpedia.org/resource/River_Crouch, http://dbpedia.org/resource/Barry_Burn] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 102, P: 0.340, R: 0.406, macro F1: 0.341, macro F1*: 0.370 -[INFO] QALD Cumulative metrics, sample: 102, P: 0.556, R: 0.406, macro F1: 0.341, QALD macro F1: 0.469 -[INFO] Current time: 2437 ms, average: 3689.68 ms/ques, entity detection: 870.33 ms/ques (linking tool: 781.13 ms/ques), relation detection: 1189.81 ms/ques (similarity: 651.34 ms/ques), EDG generation: 57.07 ms/ques +[INFO] Cumulative metrics, sample: 102, P: 0.340, R: 0.406, macro F1: 0.370 +[INFO] QALD Cumulative metrics, sample: 102, P: 0.556, R: 0.406, average F1: 0.341, QALD macro F1: 0.469 +[INFO] Current question consumed time: 2146 ms, average: 3377.48 ms/ques, entity detection: 778.02 ms/ques (linking tool: 700.62 ms/ques), relation detection: 1115.83 ms/ques (similarity: 653.14 ms/ques), EDG generation: 58.22 ms/ques [INFO] Entity Linking P: 0.27450980392156865, R: 0.27450980392156865, micro F1: 0.27450980392156865, macro F1: 0.27450980392156865 [INFO] Relation Linking P: 0.29411764705882354, R: 0.29411764705882354, micro F1: 0.29411764705882354, macro F1: 0.29411764705882354 [INFO] Type Linking P: 0.8823529411764706, R: 0.8823529411764706, micro F1: 0.8823529411764706, macro F1: 0.8823529411764706 @@ -1537,9 +1537,9 @@ Question 102: Where is Fort Knox located? [INFO] Golden Answer: [http://dbpedia.org/resource/Kentucky] [INFO] Predicted Answer: [http://dbpedia.org/resource/United_States_Bullion_Depository, http://dbpedia.org/resource/Kentucky, http://dbpedia.org/resource/Fort_Knox__location__1] [INFO] P: 0.3333333333333333, R: 1.0, F: 0.5 -[INFO] Cumulative metrics, sample: 103, P: 0.340, R: 0.412, macro F1: 0.343, macro F1*: 0.373 -[INFO] QALD Cumulative metrics, sample: 103, P: 0.554, R: 0.412, macro F1: 0.343, QALD macro F1: 0.472 -[INFO] Current time: 1186 ms, average: 3665.37 ms/ques, entity detection: 863.79 ms/ques (linking tool: 774.79 ms/ques), relation detection: 1182.42 ms/ques (similarity: 647.44 ms/ques), EDG generation: 56.59 ms/ques +[INFO] Cumulative metrics, sample: 103, P: 0.340, R: 0.412, macro F1: 0.373 +[INFO] QALD Cumulative metrics, sample: 103, P: 0.554, R: 0.412, average F1: 0.343, QALD macro F1: 0.472 +[INFO] Current question consumed time: 977 ms, average: 3354.17 ms/ques, entity detection: 771.94 ms/ques (linking tool: 694.75 ms/ques), relation detection: 1108.77 ms/ques (similarity: 649.16 ms/ques), EDG generation: 57.72 ms/ques [INFO] Entity Linking P: 0.27184466019417475, R: 0.27184466019417475, micro F1: 0.27184466019417475, macro F1: 0.27184466019417475 [INFO] Relation Linking P: 0.2912621359223301, R: 0.2912621359223301, micro F1: 0.2912621359223301, macro F1: 0.2912621359223301 [INFO] Type Linking P: 0.883495145631068, R: 0.883495145631068, micro F1: 0.883495145631068, macro F1: 0.883495145631068 @@ -1552,9 +1552,9 @@ Question 103: Which daughters of British earls died at the same place they were [INFO] Golden Answer: [http://dbpedia.org/resource/Lady_Augusta_Murray, http://dbpedia.org/resource/Joan_of_Kent, http://dbpedia.org/resource/Katherine_Neville,_Baroness_Hastings, http://dbpedia.org/resource/Lady_Brigid_Guinness, http://dbpedia.org/resource/Eleanor_Holland,_Countess_of_Salisbury, http://dbpedia.org/resource/Eleanor_Beauchamp,_Duchess_of_Somerset, http://dbpedia.org/resource/Joan_FitzGerald,_Countess_of_Carrick, http://dbpedia.org/resource/Anne_Hyde, http://dbpedia.org/resource/Diana_Russell,_Duchess_of_Bedford, http://dbpedia.org/resource/Mary_Elphinstone,_Lady_Elphinstone, http://dbpedia.org/resource/Agnes_Keith,_Countess_of_Moray, http://dbpedia.org/resource/Isabel_de_Clare,_4th_Countess_of_Pembroke, http://dbpedia.org/resource/Ela_of_Salisbury,_3rd_Countess_of_Salisbury] [INFO] Predicted Answer: [] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 104, P: 0.337, R: 0.408, macro F1: 0.340, macro F1*: 0.369 -[INFO] QALD Cumulative metrics, sample: 104, P: 0.558, R: 0.408, macro F1: 0.340, QALD macro F1: 0.471 -[INFO] Current time: 726 ms, average: 3637.11 ms/ques, entity detection: 856.32 ms/ques (linking tool: 767.72 ms/ques), relation detection: 1171.92 ms/ques (similarity: 641.80 ms/ques), EDG generation: 56.41 ms/ques +[INFO] Cumulative metrics, sample: 104, P: 0.337, R: 0.408, macro F1: 0.369 +[INFO] QALD Cumulative metrics, sample: 104, P: 0.558, R: 0.408, average F1: 0.340, QALD macro F1: 0.471 +[INFO] Current question consumed time: 596 ms, average: 3327.65 ms/ques, entity detection: 765.17 ms/ques (linking tool: 688.37 ms/ques), relation detection: 1098.89 ms/ques (similarity: 643.49 ms/ques), EDG generation: 57.59 ms/ques [INFO] Entity Linking P: 0.27884615384615385, R: 0.27884615384615385, micro F1: 0.27884615384615385, macro F1: 0.27884615384615385 [INFO] Relation Linking P: 0.2980769230769231, R: 0.2980769230769231, micro F1: 0.2980769230769231, macro F1: 0.2980769230769231 [INFO] Type Linking P: 0.8846153846153846, R: 0.8846153846153846, micro F1: 0.8846153846153846, macro F1: 0.8846153846153846 @@ -1567,9 +1567,9 @@ Question 104: How many emperors did China have? [INFO] Golden Answer: [245] [INFO] Predicted Answer: [1, 3] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 105, P: 0.334, R: 0.404, macro F1: 0.336, macro F1*: 0.366 -[INFO] QALD Cumulative metrics, sample: 105, P: 0.553, R: 0.404, macro F1: 0.336, QALD macro F1: 0.467 -[INFO] Current time: 11192 ms, average: 3709.06 ms/ques, entity detection: 867.90 ms/ques (linking tool: 779.14 ms/ques), relation detection: 1198.05 ms/ques (similarity: 670.22 ms/ques), EDG generation: 55.96 ms/ques +[INFO] Cumulative metrics, sample: 105, P: 0.334, R: 0.404, macro F1: 0.366 +[INFO] QALD Cumulative metrics, sample: 105, P: 0.553, R: 0.404, average F1: 0.336, QALD macro F1: 0.467 +[INFO] Current question consumed time: 9710 ms, average: 3388.44 ms/ques, entity detection: 772.99 ms/ques (linking tool: 696.33 ms/ques), relation detection: 1121.52 ms/ques (similarity: 670.16 ms/ques), EDG generation: 57.14 ms/ques [INFO] Entity Linking P: 0.2761904761904762, R: 0.2761904761904762, micro F1: 0.2761904761904762, macro F1: 0.2761904761904762 [INFO] Relation Linking P: 0.29523809523809524, R: 0.29523809523809524, micro F1: 0.29523809523809524, macro F1: 0.29523809523809524 [INFO] Type Linking P: 0.8857142857142857, R: 0.8857142857142857, micro F1: 0.8857142857142857, macro F1: 0.8857142857142857 @@ -1582,9 +1582,9 @@ Question 105: What are the names of the Teenage Mutant Ninja Turtles? [INFO] Golden Answer: [Donatello@en, Leonardo@en, Michelangelo@en, Raphael@en] [INFO] Predicted Answer: [] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 106, P: 0.331, R: 0.400, macro F1: 0.333, macro F1*: 0.362 -[INFO] QALD Cumulative metrics, sample: 106, P: 0.557, R: 0.400, macro F1: 0.333, QALD macro F1: 0.466 -[INFO] Current time: 948 ms, average: 3683.01 ms/ques, entity detection: 861.75 ms/ques (linking tool: 772.91 ms/ques), relation detection: 1189.30 ms/ques (similarity: 665.36 ms/ques), EDG generation: 55.55 ms/ques +[INFO] Cumulative metrics, sample: 106, P: 0.331, R: 0.400, macro F1: 0.362 +[INFO] QALD Cumulative metrics, sample: 106, P: 0.557, R: 0.400, average F1: 0.333, QALD macro F1: 0.466 +[INFO] Current question consumed time: 839 ms, average: 3364.39 ms/ques, entity detection: 767.29 ms/ques (linking tool: 690.65 ms/ques), relation detection: 1113.30 ms/ques (similarity: 665.31 ms/ques), EDG generation: 56.76 ms/ques [INFO] Entity Linking P: 0.2830188679245283, R: 0.2830188679245283, micro F1: 0.2830188679245283, macro F1: 0.2830188679245283 [INFO] Relation Linking P: 0.3018867924528302, R: 0.3018867924528302, micro F1: 0.3018867924528302, macro F1: 0.3018867924528302 [INFO] Type Linking P: 0.8867924528301887, R: 0.8867924528301887, micro F1: 0.8867924528301887, macro F1: 0.8867924528301887 @@ -1597,9 +1597,9 @@ Question 106: Where does Piccadilly start? [INFO] Golden Answer: [http://dbpedia.org/resource/Dover_Street] [INFO] Predicted Answer: [http://dbpedia.org/resource/Dover_Street] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 107, P: 0.337, R: 0.406, macro F1: 0.339, macro F1*: 0.368 -[INFO] QALD Cumulative metrics, sample: 107, P: 0.561, R: 0.406, macro F1: 0.339, QALD macro F1: 0.471 -[INFO] Current time: 947 ms, average: 3657.44 ms/ques, entity detection: 855.32 ms/ques (linking tool: 766.58 ms/ques), relation detection: 1181.25 ms/ques (similarity: 660.87 ms/ques), EDG generation: 55.07 ms/ques +[INFO] Cumulative metrics, sample: 107, P: 0.337, R: 0.406, macro F1: 0.368 +[INFO] QALD Cumulative metrics, sample: 107, P: 0.561, R: 0.406, average F1: 0.339, QALD macro F1: 0.471 +[INFO] Current question consumed time: 813 ms, average: 3340.55 ms/ques, entity detection: 761.44 ms/ques (linking tool: 684.96 ms/ques), relation detection: 1105.63 ms/ques (similarity: 660.81 ms/ques), EDG generation: 56.29 ms/ques [INFO] Entity Linking P: 0.2803738317757009, R: 0.2803738317757009, micro F1: 0.2803738317757009, macro F1: 0.2803738317757009 [INFO] Relation Linking P: 0.29906542056074764, R: 0.29906542056074764, micro F1: 0.29906542056074764, macro F1: 0.29906542056074764 [INFO] Type Linking P: 0.8878504672897196, R: 0.8878504672897196, micro F1: 0.8878504672897196, macro F1: 0.8878504672897196 @@ -1612,9 +1612,9 @@ Question 107: What is the name of the university where Obama's wife studied? [INFO] Golden Answer: [http://dbpedia.org/resource/Princeton_University, http://dbpedia.org/resource/Harvard_Law_School] [INFO] Predicted Answer: [http://dbpedia.org/resource/Category:Geoplanidae] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 108, P: 0.334, R: 0.402, macro F1: 0.336, macro F1*: 0.365 -[INFO] QALD Cumulative metrics, sample: 108, P: 0.556, R: 0.402, macro F1: 0.336, QALD macro F1: 0.467 -[INFO] Current time: 1046 ms, average: 3633.26 ms/ques, entity detection: 848.20 ms/ques (linking tool: 759.76 ms/ques), relation detection: 1172.99 ms/ques (similarity: 657.58 ms/ques), EDG generation: 54.80 ms/ques +[INFO] Cumulative metrics, sample: 108, P: 0.334, R: 0.402, macro F1: 0.365 +[INFO] QALD Cumulative metrics, sample: 108, P: 0.556, R: 0.402, average F1: 0.336, QALD macro F1: 0.467 +[INFO] Current question consumed time: 926 ms, average: 3318.19 ms/ques, entity detection: 754.96 ms/ques (linking tool: 678.76 ms/ques), relation detection: 1097.91 ms/ques (similarity: 657.39 ms/ques), EDG generation: 55.99 ms/ques [INFO] Entity Linking P: 0.2777777777777778, R: 0.2777777777777778, micro F1: 0.2777777777777778, macro F1: 0.2777777777777778 [INFO] Relation Linking P: 0.2962962962962963, R: 0.2962962962962963, micro F1: 0.2962962962962963, macro F1: 0.2962962962962963 [INFO] Type Linking P: 0.8888888888888888, R: 0.8888888888888888, micro F1: 0.8888888888888888, macro F1: 0.8888888888888888 @@ -1627,9 +1627,9 @@ Question 108: When did Paraguay proclaim its independence? [INFO] Golden Answer: [1811-05-14, 1811-05-15] [INFO] Predicted Answer: [1811-05-14, 1811-05-15] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 109, P: 0.340, R: 0.408, macro F1: 0.342, macro F1*: 0.371 -[INFO] QALD Cumulative metrics, sample: 109, P: 0.560, R: 0.408, macro F1: 0.342, QALD macro F1: 0.472 -[INFO] Current time: 4098 ms, average: 3637.52 ms/ques, entity detection: 846.96 ms/ques (linking tool: 757.77 ms/ques), relation detection: 1178.82 ms/ques (similarity: 660.83 ms/ques), EDG generation: 54.37 ms/ques +[INFO] Cumulative metrics, sample: 109, P: 0.340, R: 0.408, macro F1: 0.371 +[INFO] QALD Cumulative metrics, sample: 109, P: 0.560, R: 0.408, average F1: 0.342, QALD macro F1: 0.472 +[INFO] Current question consumed time: 3624 ms, average: 3321.00 ms/ques, entity detection: 753.77 ms/ques (linking tool: 676.97 ms/ques), relation detection: 1103.02 ms/ques (similarity: 660.67 ms/ques), EDG generation: 55.62 ms/ques [INFO] Entity Linking P: 0.27522935779816515, R: 0.27522935779816515, micro F1: 0.27522935779816515, macro F1: 0.27522935779816515 [INFO] Relation Linking P: 0.29357798165137616, R: 0.29357798165137616, micro F1: 0.29357798165137616, macro F1: 0.29357798165137616 [INFO] Type Linking P: 0.8899082568807339, R: 0.8899082568807339, micro F1: 0.8899082568807339, macro F1: 0.8899082568807339 @@ -1642,9 +1642,9 @@ Question 109: How short is the shortest active NBA player? [INFO] Golden Answer: [1.7526] [INFO] Predicted Answer: [] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 110, P: 0.337, R: 0.404, macro F1: 0.339, macro F1*: 0.367 -[INFO] QALD Cumulative metrics, sample: 110, P: 0.564, R: 0.404, macro F1: 0.339, QALD macro F1: 0.471 -[INFO] Current time: 452 ms, average: 3608.56 ms/ques, entity detection: 840.93 ms/ques (linking tool: 751.71 ms/ques), relation detection: 1168.93 ms/ques (similarity: 655.31 ms/ques), EDG generation: 53.98 ms/ques +[INFO] Cumulative metrics, sample: 110, P: 0.337, R: 0.404, macro F1: 0.367 +[INFO] QALD Cumulative metrics, sample: 110, P: 0.564, R: 0.404, average F1: 0.339, QALD macro F1: 0.471 +[INFO] Current question consumed time: 438 ms, average: 3294.79 ms/ques, entity detection: 748.56 ms/ques (linking tool: 671.91 ms/ques), relation detection: 1093.85 ms/ques (similarity: 655.23 ms/ques), EDG generation: 55.12 ms/ques [INFO] Entity Linking P: 0.2818181818181818, R: 0.2818181818181818, micro F1: 0.2818181818181818, macro F1: 0.2818181818181818 [INFO] Relation Linking P: 0.3, R: 0.3, micro F1: 0.3, macro F1: 0.3 [INFO] Type Linking P: 0.8909090909090909, R: 0.8909090909090909, micro F1: 0.8909090909090909, macro F1: 0.8909090909090909 @@ -1657,9 +1657,9 @@ Question 110: Where did Abraham Lincoln die? [INFO] Golden Answer: [http://dbpedia.org/resource/Petersen_House_(Washington,_D.C.), http://dbpedia.org/resource/Washington,_D.C.] [INFO] Predicted Answer: [http://dbpedia.org/resource/Petersen_House_(Washington,_D.C.), http://dbpedia.org/resource/Washington,_D.C., Washington, D.C., U.S.] [INFO] P: 0.6666666666666666, R: 1.0, F: 0.8 -[INFO] Cumulative metrics, sample: 111, P: 0.340, R: 0.409, macro F1: 0.343, macro F1*: 0.371 -[INFO] QALD Cumulative metrics, sample: 111, P: 0.565, R: 0.409, macro F1: 0.343, QALD macro F1: 0.475 -[INFO] Current time: 2880 ms, average: 3602.01 ms/ques, entity detection: 837.67 ms/ques (linking tool: 748.28 ms/ques), relation detection: 1169.88 ms/ques (similarity: 655.46 ms/ques), EDG generation: 53.56 ms/ques +[INFO] Cumulative metrics, sample: 111, P: 0.340, R: 0.409, macro F1: 0.371 +[INFO] QALD Cumulative metrics, sample: 111, P: 0.565, R: 0.409, average F1: 0.343, QALD macro F1: 0.475 +[INFO] Current question consumed time: 2371 ms, average: 3286.47 ms/ques, entity detection: 745.21 ms/ques (linking tool: 668.52 ms/ques), relation detection: 1094.02 ms/ques (similarity: 655.34 ms/ques), EDG generation: 54.76 ms/ques [INFO] Entity Linking P: 0.27927927927927926, R: 0.27927927927927926, micro F1: 0.27927927927927926, macro F1: 0.27927927927927926 [INFO] Relation Linking P: 0.2972972972972973, R: 0.2972972972972973, micro F1: 0.2972972972972973, macro F1: 0.2972972972972973 [INFO] Type Linking P: 0.8918918918918919, R: 0.8918918918918919, micro F1: 0.8918918918918919, macro F1: 0.8918918918918919 @@ -1672,9 +1672,9 @@ Question 111: When was Jack Wolfskin founded? [INFO] Golden Answer: [1981-01-01] [INFO] Predicted Answer: [1981] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 112, P: 0.346, R: 0.415, macro F1: 0.349, macro F1*: 0.377 -[INFO] QALD Cumulative metrics, sample: 112, P: 0.569, R: 0.415, macro F1: 0.349, QALD macro F1: 0.480 -[INFO] Current time: 781 ms, average: 3576.82 ms/ques, entity detection: 831.92 ms/ques (linking tool: 742.39 ms/ques), relation detection: 1161.46 ms/ques (similarity: 650.79 ms/ques), EDG generation: 53.14 ms/ques +[INFO] Cumulative metrics, sample: 112, P: 0.346, R: 0.415, macro F1: 0.377 +[INFO] QALD Cumulative metrics, sample: 112, P: 0.569, R: 0.415, average F1: 0.349, QALD macro F1: 0.480 +[INFO] Current question consumed time: 645 ms, average: 3262.88 ms/ques, entity detection: 739.87 ms/ques (linking tool: 663.16 ms/ques), relation detection: 1086.07 ms/ques (similarity: 650.71 ms/ques), EDG generation: 54.33 ms/ques [INFO] Entity Linking P: 0.2767857142857143, R: 0.2767857142857143, micro F1: 0.2767857142857143, macro F1: 0.2767857142857143 [INFO] Relation Linking P: 0.29464285714285715, R: 0.29464285714285715, micro F1: 0.29464285714285715, macro F1: 0.29464285714285715 [INFO] Type Linking P: 0.8928571428571429, R: 0.8928571428571429, micro F1: 0.8928571428571429, macro F1: 0.8928571428571429 @@ -1687,9 +1687,9 @@ Question 112: In which city is the headquarter of Air China? [INFO] Golden Answer: [http://dbpedia.org/resource/Beijing] [INFO] Predicted Answer: [http://dbpedia.org/resource/Beijing] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 113, P: 0.351, R: 0.420, macro F1: 0.355, macro F1*: 0.383 -[INFO] QALD Cumulative metrics, sample: 113, P: 0.573, R: 0.420, macro F1: 0.355, QALD macro F1: 0.484 -[INFO] Current time: 3311 ms, average: 3574.47 ms/ques, entity detection: 835.00 ms/ques (linking tool: 745.51 ms/ques), relation detection: 1154.99 ms/ques (similarity: 649.54 ms/ques), EDG generation: 52.80 ms/ques +[INFO] Cumulative metrics, sample: 113, P: 0.351, R: 0.420, macro F1: 0.383 +[INFO] QALD Cumulative metrics, sample: 113, P: 0.573, R: 0.420, average F1: 0.355, QALD macro F1: 0.484 +[INFO] Current question consumed time: 2765 ms, average: 3258.48 ms/ques, entity detection: 742.01 ms/ques (linking tool: 665.30 ms/ques), relation detection: 1079.80 ms/ques (similarity: 649.29 ms/ques), EDG generation: 53.98 ms/ques [INFO] Entity Linking P: 0.2743362831858407, R: 0.2743362831858407, micro F1: 0.2743362831858407, macro F1: 0.2743362831858407 [INFO] Relation Linking P: 0.2920353982300885, R: 0.2920353982300885, micro F1: 0.2920353982300885, macro F1: 0.2920353982300885 [INFO] Type Linking P: 0.8849557522123894, R: 0.8849557522123894, micro F1: 0.8849557522123894, macro F1: 0.8849557522123894 @@ -1702,9 +1702,9 @@ Question 113: What is the founding year of the brewery that produces Pilsner Urq [INFO] Golden Answer: [1842-01-01] [INFO] Predicted Answer: [1842] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 114, P: 0.357, R: 0.425, macro F1: 0.361, macro F1*: 0.388 -[INFO] QALD Cumulative metrics, sample: 114, P: 0.576, R: 0.425, macro F1: 0.361, QALD macro F1: 0.489 -[INFO] Current time: 1540 ms, average: 3556.62 ms/ques, entity detection: 828.85 ms/ques (linking tool: 739.67 ms/ques), relation detection: 1150.81 ms/ques (similarity: 648.46 ms/ques), EDG generation: 52.61 ms/ques +[INFO] Cumulative metrics, sample: 114, P: 0.357, R: 0.425, macro F1: 0.388 +[INFO] QALD Cumulative metrics, sample: 114, P: 0.576, R: 0.425, average F1: 0.361, QALD macro F1: 0.489 +[INFO] Current question consumed time: 1422 ms, average: 3242.37 ms/ques, entity detection: 736.60 ms/ques (linking tool: 660.15 ms/ques), relation detection: 1075.91 ms/ques (similarity: 648.24 ms/ques), EDG generation: 53.78 ms/ques [INFO] Entity Linking P: 0.2719298245614035, R: 0.2719298245614035, micro F1: 0.2719298245614035, macro F1: 0.2719298245614035 [INFO] Relation Linking P: 0.2894736842105263, R: 0.2894736842105263, micro F1: 0.2894736842105263, macro F1: 0.2894736842105263 [INFO] Type Linking P: 0.8859649122807017, R: 0.8859649122807017, micro F1: 0.8859649122807017, macro F1: 0.8859649122807017 @@ -1717,9 +1717,9 @@ Question 114: When did the Boston Tea Party take place? [INFO] Golden Answer: [1773-12-16] [INFO] Predicted Answer: [1773-12-16] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 115, P: 0.363, R: 0.430, macro F1: 0.366, macro F1*: 0.393 -[INFO] QALD Cumulative metrics, sample: 115, P: 0.580, R: 0.430, macro F1: 0.366, QALD macro F1: 0.494 -[INFO] Current time: 777 ms, average: 3532.45 ms/ques, entity detection: 822.83 ms/ques (linking tool: 733.97 ms/ques), relation detection: 1142.75 ms/ques (similarity: 644.01 ms/ques), EDG generation: 52.23 ms/ques +[INFO] Cumulative metrics, sample: 115, P: 0.363, R: 0.430, macro F1: 0.393 +[INFO] QALD Cumulative metrics, sample: 115, P: 0.580, R: 0.430, average F1: 0.366, QALD macro F1: 0.494 +[INFO] Current question consumed time: 656 ms, average: 3219.88 ms/ques, entity detection: 731.28 ms/ques (linking tool: 655.09 ms/ques), relation detection: 1068.32 ms/ques (similarity: 643.64 ms/ques), EDG generation: 53.44 ms/ques [INFO] Entity Linking P: 0.26956521739130435, R: 0.26956521739130435, micro F1: 0.26956521739130435, macro F1: 0.26956521739130435 [INFO] Relation Linking P: 0.28695652173913044, R: 0.28695652173913044, micro F1: 0.28695652173913044, macro F1: 0.28695652173913044 [INFO] Type Linking P: 0.8869565217391304, R: 0.8869565217391304, micro F1: 0.8869565217391304, macro F1: 0.8869565217391304 @@ -1732,9 +1732,9 @@ Question 115: Which animals are critically endangered? [INFO] Golden Answer: [http://dbpedia.org/resource/Bali_myna, http://dbpedia.org/resource/Himalayan_brown_bear, http://dbpedia.org/resource/Marsican_brown_bear, http://dbpedia.org/resource/Siberian_crane, http://dbpedia.org/resource/Sunda_pangolin, http://dbpedia.org/resource/Yellow-crested_cockatoo, http://dbpedia.org/resource/Baiji, http://dbpedia.org/resource/Chinese_sturgeon, http://dbpedia.org/resource/Giant_barb, http://dbpedia.org/resource/Pan's_box_turtle, http://dbpedia.org/resource/Yangtze_giant_softshell_turtle, http://dbpedia.org/resource/Yellow-headed_box_turtle, http://dbpedia.org/resource/Yunnan_box_turtle, http://dbpedia.org/resource/Antioquia_brush_finch, http://dbpedia.org/resource/Blue-banded_kingfisher, http://dbpedia.org/resource/Cozumel_thrasher, http://dbpedia.org/resource/Glaucous_macaw, http://dbpedia.org/resource/Ivory-billed_woodpecker, http://dbpedia.org/resource/Juan_Fernández_firecrown, http://dbpedia.org/resource/Kauaʻi_nukupuʻu, http://dbpedia.org/resource/Makira_woodhen, http://dbpedia.org/resource/Maui_nukupuʻu, http://dbpedia.org/resource/Millerbird, http://dbpedia.org/resource/Negros_fruit_dove, http://dbpedia.org/resource/New_Caledonian_lorikeet, http://dbpedia.org/resource/New_Caledonian_rail, http://dbpedia.org/resource/Nightingale_reed_warbler, http://dbpedia.org/resource/Pink-headed_duck, http://dbpedia.org/resource/Pohnpei_starling, http://dbpedia.org/resource/Polynesian_ground_dove, http://dbpedia.org/resource/Rück's_blue_flycatcher, http://dbpedia.org/resource/Semper's_warbler, http://dbpedia.org/resource/Siau_scops_owl, http://dbpedia.org/resource/Slender-billed_curlew, http://dbpedia.org/resource/Sulu_bleeding-heart, http://dbpedia.org/resource/Sumatran_elephant, http://dbpedia.org/resource/White-chested_white-eye, http://dbpedia.org/resource/Northwest_African_cheetah, http://dbpedia.org/resource/Bocaccio_rockfish, http://dbpedia.org/resource/Coregonus_bavaricus, http://dbpedia.org/resource/Coregonus_trybomi, http://dbpedia.org/resource/Northern_hairy-nosed_wombat, http://dbpedia.org/resource/Cuban_crocodile, http://dbpedia.org/resource/Golden_mantella, http://dbpedia.org/resource/Orinoco_crocodile, http://dbpedia.org/resource/Philippine_crocodile, http://dbpedia.org/resource/Phyllomedusa_ayeaye, http://dbpedia.org/resource/Siamese_crocodile, http://dbpedia.org/resource/Slender-snouted_crocodile, http://dbpedia.org/resource/Smalltooth_sawfish, http://dbpedia.org/resource/Elkhorn_coral, http://dbpedia.org/resource/Sakhalin_taimen, http://dbpedia.org/resource/Totoaba, http://dbpedia.org/resource/Bastard_sturgeon, http://dbpedia.org/resource/Black_dorcopsis, http://dbpedia.org/resource/Black_crested_gibbon, http://dbpedia.org/resource/Eastern_black_crested_gibbon, http://dbpedia.org/resource/Hainan_black_crested_gibbon, http://dbpedia.org/resource/Arakan_forest_turtle, http://dbpedia.org/resource/Jamaican_iguana, http://dbpedia.org/resource/McCord's_box_turtle, http://dbpedia.org/resource/Bengal_florican, http://dbpedia.org/resource/Riverine_rabbit, http://dbpedia.org/resource/Edwards's_pheasant, http://dbpedia.org/resource/Red-vented_cockatoo, http://dbpedia.org/resource/Tristan_albatross, http://dbpedia.org/resource/Beck's_petrel, http://dbpedia.org/resource/Black-breasted_puffleg, http://dbpedia.org/resource/Black-hooded_coucal, http://dbpedia.org/resource/Blue-fronted_lorikeet, http://dbpedia.org/resource/Brazilian_merganser, http://dbpedia.org/resource/Bryan's_shearwater, http://dbpedia.org/resource/Chilean_woodstar, http://dbpedia.org/resource/Colorful_puffleg, http://dbpedia.org/resource/Dusky_starfrontlet, http://dbpedia.org/resource/Horned_curassow, http://dbpedia.org/resource/Kinglet_calyptura, http://dbpedia.org/resource/Mascarene_petrel, http://dbpedia.org/resource/Red-throated_lorikeet, http://dbpedia.org/resource/Sapphire-bellied_hummingbird, http://dbpedia.org/resource/Short-crested_coquette, http://dbpedia.org/resource/Townsend's_shearwater, http://dbpedia.org/resource/White-winged_guan, http://dbpedia.org/resource/Leiolopisma_alazon, http://dbpedia.org/resource/Colombian_spider_monkey, http://dbpedia.org/resource/Abe's_salamander, http://dbpedia.org/resource/Gorgan_salamander, http://dbpedia.org/resource/Oki_salamander, http://dbpedia.org/resource/Paghman_stream_salamander, http://dbpedia.org/resource/Squatina_squatina, http://dbpedia.org/resource/Northern_muriqui, http://dbpedia.org/resource/Prodontria_lewisi, http://dbpedia.org/resource/Chapultepec_splitfin, http://dbpedia.org/resource/Rosy_bitterling, http://dbpedia.org/resource/Ecuadorian_capuchin, http://dbpedia.org/resource/Calico_grouper, http://dbpedia.org/resource/Moapa_dace, http://dbpedia.org/resource/Clarence_galaxias, http://dbpedia.org/resource/Neopasiphae_simplicior, http://dbpedia.org/resource/Oreochromis_chungruruensis, http://dbpedia.org/resource/Oreochromis_jipe, http://dbpedia.org/resource/Oreochromis_variabilis, http://dbpedia.org/resource/Sarotherodon_linnellii, http://dbpedia.org/resource/Microhyla_karunaratnei, http://dbpedia.org/resource/Eungella_torrent_frog, http://dbpedia.org/resource/Cachorrito_cabezon, http://dbpedia.org/resource/Saint_Croix_ground_lizard, http://dbpedia.org/resource/Smoothback_angelshark, http://dbpedia.org/resource/Lycodon_chrysoprateros, http://dbpedia.org/resource/Dikume, http://dbpedia.org/resource/Konye, http://dbpedia.org/resource/Tropical_pocket_gopher, http://dbpedia.org/resource/Mount_Oku_hylomyscus, http://dbpedia.org/resource/Palaemonetes_cummingi, http://dbpedia.org/resource/Synodontis_dekimpei, http://dbpedia.org/resource/Colostethus_dunni, http://dbpedia.org/resource/Chalchalero_vizcacha_rat, http://dbpedia.org/resource/Chiapan_climbing_rat, http://dbpedia.org/resource/Margarita_Island_kangaroo_rat, http://dbpedia.org/resource/Tumbala_climbing_rat, http://dbpedia.org/resource/Barbodes_amarus, http://dbpedia.org/resource/Barbodes_baoulan, http://dbpedia.org/resource/Barbodes_clemensi, http://dbpedia.org/resource/Barbodes_flavifuscus, http://dbpedia.org/resource/Barbodes_herrei, http://dbpedia.org/resource/Barbodes_katolo, http://dbpedia.org/resource/Barbodes_lanaoensis, http://dbpedia.org/resource/Barbodes_manalak, http://dbpedia.org/resource/Barbodes_pachycheilus, http://dbpedia.org/resource/Barbodes_palata, http://dbpedia.org/resource/Barbodes_resimus, http://dbpedia.org/resource/Barbodes_tras, http://dbpedia.org/resource/Barbodes_truncatulus, http://dbpedia.org/resource/Aaadonta_constricta_babelthuapi, http://dbpedia.org/resource/Aaadonta_constricta_constricta, http://dbpedia.org/resource/Aaadonta_fuscozonata_depressa, http://dbpedia.org/resource/Aaadonta_fuscozonata_fuscozonata, http://dbpedia.org/resource/Chrysoritis_cotrelli, http://dbpedia.org/resource/David's_tiger, http://dbpedia.org/resource/Lepidochrysops_lotana, http://dbpedia.org/resource/Mortlock_flying_fox, http://dbpedia.org/resource/Piedmont_anomalous_blue, http://dbpedia.org/resource/Prairie_sphinx_moth, http://dbpedia.org/resource/San_José_brush_rabbit, http://dbpedia.org/resource/Allotoca_maculata, http://dbpedia.org/resource/Andean_catfish, http://dbpedia.org/resource/Aplocheilichthys_sp._nov._'Baringo', http://dbpedia.org/resource/Black_mogurnda, http://dbpedia.org/resource/Cachorrito_de_mezquital, http://dbpedia.org/resource/Charco_Palma_pupfish, http://dbpedia.org/resource/Chinese_bahaba, http://dbpedia.org/resource/Cuatrocienegas_killifish, http://dbpedia.org/resource/Encheloclarias_curtisoma, http://dbpedia.org/resource/Encheloclarias_kelioides, http://dbpedia.org/resource/Glass_blue-eye, http://dbpedia.org/resource/Gymnocephalus_ambriaelacus, http://dbpedia.org/resource/Hong_Kong_grouper, http://dbpedia.org/resource/Hubbsina_turneri, http://dbpedia.org/resource/Hyporthodus_nigritus, http://dbpedia.org/resource/Ilyodon_whitei, http://dbpedia.org/resource/Japanese_sturgeon, http://dbpedia.org/resource/Karomo, http://dbpedia.org/resource/Killarney_shad, http://dbpedia.org/resource/Knipowitschia_ephesi, http://dbpedia.org/resource/Knipowitschia_mermere, http://dbpedia.org/resource/Knipowitschia_mrakovcici, http://dbpedia.org/resource/Lake_Chala_tilapia, http://dbpedia.org/resource/Lake_Wanam_rainbowfish, http://dbpedia.org/resource/Largefin_pupfish, http://dbpedia.org/resource/Monopterus_desilvai, http://dbpedia.org/resource/Myaka_(fish), http://dbpedia.org/resource/Orthochromis_uvinzae, http://dbpedia.org/resource/Pachypanchax_sakaramyi, http://dbpedia.org/resource/Pygmy_sculpin, http://dbpedia.org/resource/Sentani_rainbowfish, http://dbpedia.org/resource/Variegated_mogurnda, http://dbpedia.org/resource/Appalachian_monkey-face_pearly_mussel, http://dbpedia.org/resource/Atlas_pebblesnail, http://dbpedia.org/resource/Bruneau_hot_springsnail, http://dbpedia.org/resource/Catalina_mountain_snail, http://dbpedia.org/resource/Coosa_pebblesnail, http://dbpedia.org/resource/Cumberland_bean_pearly_mussel, http://dbpedia.org/resource/Dlinza_Forest_pinwheel, http://dbpedia.org/resource/Dwarf_pebblesnail, http://dbpedia.org/resource/Fluted_pebblesnail, http://dbpedia.org/resource/Marstonia_castor, http://dbpedia.org/resource/Marstonia_ozarkensis, http://dbpedia.org/resource/Opaque_pebblesnail, http://dbpedia.org/resource/Pygmy_pebblesnail, http://dbpedia.org/resource/Quadrate_pebblesnail, http://dbpedia.org/resource/Quadrula_mitchelli, http://dbpedia.org/resource/Radioconus_goeldi, http://dbpedia.org/resource/Rhysoconcha_variumbilicata, http://dbpedia.org/resource/Rio_Grande_monkeyface, http://dbpedia.org/resource/Rolling_pebblesnail, http://dbpedia.org/resource/Ruatara_koarana, http://dbpedia.org/resource/Samoana_bellula, http://dbpedia.org/resource/Samoana_burchi, http://dbpedia.org/resource/Samoana_cramptoni, http://dbpedia.org/resource/Samoana_decussatula, http://dbpedia.org/resource/Samoana_dryas, http://dbpedia.org/resource/Samoana_fragilis, http://dbpedia.org/resource/Samoana_hamadryas, http://dbpedia.org/resource/Short_Samoan_tree_snail, http://dbpedia.org/resource/Southern_kidneyshell, http://dbpedia.org/resource/Stocky_pebblesnail, http://dbpedia.org/resource/Tennessee_pebblesnail, http://dbpedia.org/resource/Theba_arinagae, http://dbpedia.org/resource/Theodoxus_altenai, http://dbpedia.org/resource/Trachycystis_placenta, http://dbpedia.org/resource/Trochochlamys_ogasawarana, http://dbpedia.org/resource/Trochoidea_pseudojacosta, http://dbpedia.org/resource/Tropidoptera_heliciformis, http://dbpedia.org/resource/Wicker_ancylid, http://dbpedia.org/resource/Winged_mapleleaf, http://dbpedia.org/resource/Xerosecta_giustii, http://dbpedia.org/resource/Zilchogyra_paulistana, http://dbpedia.org/resource/Zingis_radiolata, http://dbpedia.org/resource/Cocoharpinia, http://dbpedia.org/resource/Holozoster_ovalis, http://dbpedia.org/resource/Sitalcicus_incertus, http://dbpedia.org/resource/Anilany_helenae, http://dbpedia.org/resource/Dianchi_bullhead, http://dbpedia.org/resource/Iglica_gratulabunda, http://dbpedia.org/resource/Tyrrhenaria_ceratina, http://dbpedia.org/resource/Pita_skate, http://dbpedia.org/resource/Belalanda_chameleon, http://dbpedia.org/resource/Balkan_lynx, http://dbpedia.org/resource/Beluga_(sturgeon), http://dbpedia.org/resource/Chinese_giant_salamander, http://dbpedia.org/resource/Crotalus_durissus_unicolor, http://dbpedia.org/resource/European_eel, http://dbpedia.org/resource/European_sea_sturgeon, http://dbpedia.org/resource/Adetomyrma_venatrix, http://dbpedia.org/resource/Leadbeater's_possum, http://dbpedia.org/resource/Sri_Lankan_relict_ant, http://dbpedia.org/resource/White-rumped_vulture, http://dbpedia.org/resource/Saint_Helena_plover, http://dbpedia.org/resource/Staghorn_coral, http://dbpedia.org/resource/Mauritius_olive_white-eye, http://dbpedia.org/resource/Rota_white-eye, http://dbpedia.org/resource/Sangihe_white-eye, http://dbpedia.org/resource/Blue-throated_macaw, http://dbpedia.org/resource/Djibouti_francolin, http://dbpedia.org/resource/Grenada_dove, http://dbpedia.org/resource/Siau_Island_tarsier, http://dbpedia.org/resource/Helan_Shan_pika, http://dbpedia.org/resource/Golden_coquí, http://dbpedia.org/resource/Dryococelus_australis, http://dbpedia.org/resource/Golden_white-eye, http://dbpedia.org/resource/White-collared_kite, http://dbpedia.org/resource/Araripe_manakin, http://dbpedia.org/resource/Sira_curassow, http://dbpedia.org/resource/Painted_terrapin, http://dbpedia.org/resource/Oreochromis_esculentus, http://dbpedia.org/resource/Dahl's_toad-headed_turtle, http://dbpedia.org/resource/Magdalena_River_turtle, http://dbpedia.org/resource/Crotalus_catalinensis, http://dbpedia.org/resource/Nicobar_shrew, http://dbpedia.org/resource/Partula_otaheitana, http://dbpedia.org/resource/Cyclura_rileyi, http://dbpedia.org/resource/Fatu_Hiva_monarch, http://dbpedia.org/resource/Liophis_cursor, http://dbpedia.org/resource/ʻAkikiki, http://dbpedia.org/resource/ʻAkohekohe, http://dbpedia.org/resource/‘Akeke‘e, http://dbpedia.org/resource/Belgrandia_varica, http://dbpedia.org/resource/Bythiospeum_pfeifferi, http://dbpedia.org/resource/Eleutherodactylus_rufescens, http://dbpedia.org/resource/Edwardsina_tasmaniensis, http://dbpedia.org/resource/Hasarius_mahensis, http://dbpedia.org/resource/Typhlogarra_widdowsoni, http://dbpedia.org/resource/Cerion_nanus, http://dbpedia.org/resource/Azuero_spider_monkey, http://dbpedia.org/resource/Pene_galilaea, http://dbpedia.org/resource/Oreocnemis_phoenix, http://dbpedia.org/resource/Kunming_catfish, http://dbpedia.org/resource/Frost's_arboreal_alligator_lizard, http://dbpedia.org/resource/Tylognathus_festai, http://dbpedia.org/resource/Andaman_shrew, http://dbpedia.org/resource/Jenkins'_shrew, http://dbpedia.org/resource/Royal_cinclodes, http://dbpedia.org/resource/Yellow-tailed_woolly_monkey, http://dbpedia.org/resource/Bermuda_rock_skink, http://dbpedia.org/resource/California_condor, http://dbpedia.org/resource/Murray_cod, http://dbpedia.org/resource/Vancouver_Island_marmot, http://dbpedia.org/resource/West_African_lion, http://dbpedia.org/resource/El_Hierro_giant_lizard, http://dbpedia.org/resource/Hawksbill_sea_turtle, http://dbpedia.org/resource/Kemp's_ridley_sea_turtle, http://dbpedia.org/resource/La_Gomera_giant_lizard, http://dbpedia.org/resource/La_Palma_giant_lizard, http://dbpedia.org/resource/Asiatic_cheetah, http://dbpedia.org/resource/Tooth-billed_pigeon, http://dbpedia.org/resource/Enggano_rat, http://dbpedia.org/resource/Hooded_grebe, http://dbpedia.org/resource/Junin_grebe, http://dbpedia.org/resource/Regent_honeyeater, http://dbpedia.org/resource/Harenna_shrew, http://dbpedia.org/resource/Alagoas_antwren, http://dbpedia.org/resource/Alagoas_foliage-gleaner, http://dbpedia.org/resource/Black-chinned_monarch, http://dbpedia.org/resource/Black-winged_starling, http://dbpedia.org/resource/Blue-eyed_ground_dove, http://dbpedia.org/resource/Cebu_flowerpecker, http://dbpedia.org/resource/Cerulean_paradise_flycatcher, http://dbpedia.org/resource/Cherry-throated_tanager, http://dbpedia.org/resource/Crow_honeyeater, http://dbpedia.org/resource/Forest_owlet, http://dbpedia.org/resource/Galápagos_petrel, http://dbpedia.org/resource/Gough_finch, http://dbpedia.org/resource/Hoary-throated_spinetail, http://dbpedia.org/resource/Jerdon's_courser, http://dbpedia.org/resource/Liberian_greenbul, http://dbpedia.org/resource/Mangrove_finch, http://dbpedia.org/resource/Medium_tree_finch, http://dbpedia.org/resource/Mindoro_bleeding-heart, http://dbpedia.org/resource/Negros_bleeding-heart, http://dbpedia.org/resource/Purple-winged_ground_dove, http://dbpedia.org/resource/Rio_Branco_antbird, http://dbpedia.org/resource/Rio_de_Janeiro_antwren, http://dbpedia.org/resource/Socorro_mockingbird, http://dbpedia.org/resource/Stresemann's_bristlefront, http://dbpedia.org/resource/Tahiti_monarch, http://dbpedia.org/resource/White-bellied_cinclodes, http://dbpedia.org/resource/White-winged_flufftail, http://dbpedia.org/resource/Alabama_sturgeon, http://dbpedia.org/resource/Amu_Darya_sturgeon, http://dbpedia.org/resource/Dwarf_sturgeon, http://dbpedia.org/resource/Persian_sturgeon, http://dbpedia.org/resource/Russian_sturgeon, http://dbpedia.org/resource/Sakhalin_sturgeon, http://dbpedia.org/resource/Starry_sturgeon, http://dbpedia.org/resource/Syr_Darya_sturgeon, http://dbpedia.org/resource/Conolophus_marthae, http://dbpedia.org/resource/Ctenosaura_bakeri, http://dbpedia.org/resource/Ctenosaura_oaxacana, http://dbpedia.org/resource/Bay_checkerspot_butterfly, http://dbpedia.org/resource/White_River_spinedace, http://dbpedia.org/resource/Bermudamysis, http://dbpedia.org/resource/Platyops, http://dbpedia.org/resource/Takifugu_chinensis, http://dbpedia.org/resource/Monterrey_platyfish, http://dbpedia.org/resource/Kululu, http://dbpedia.org/resource/Western_purple-faced_langur, http://dbpedia.org/resource/Bouvier's_red_colobus, http://dbpedia.org/resource/Emperor_rat, http://dbpedia.org/resource/Ethiopian_amphibious_rat, http://dbpedia.org/resource/Guadalcanal_rat, http://dbpedia.org/resource/Wimmer's_shrew, http://dbpedia.org/resource/Caquetá_titi, http://dbpedia.org/resource/Aipysurus_apraefrontalis, http://dbpedia.org/resource/Seychelles_sheath-tailed_bat, http://dbpedia.org/resource/Bolivian_chinchilla_rat, http://dbpedia.org/resource/Clarias_cavernicola, http://dbpedia.org/resource/Clarias_maclareni, http://dbpedia.org/resource/Niger_Delta_red_colobus, http://dbpedia.org/resource/Cuban_greater_funnel-eared_bat, http://dbpedia.org/resource/Gastrotheca_lauzuricae, http://dbpedia.org/resource/Jamaican_greater_funnel-eared_bat, http://dbpedia.org/resource/Mantiqueira_Atlantic_tree-rat, http://dbpedia.org/resource/Short-furred_Atlantic_tree-rat, http://dbpedia.org/resource/Romanichthys_valsanicola, http://dbpedia.org/resource/Chapa_pygmy_dormouse, http://dbpedia.org/resource/Albericus_siegfriedi, http://dbpedia.org/resource/Callulina_dawida, http://dbpedia.org/resource/Centrolene_ballux, http://dbpedia.org/resource/Centrolene_gemmatum, http://dbpedia.org/resource/Colostethus_jacobuspetersi, http://dbpedia.org/resource/Oxylapia_polli, http://dbpedia.org/resource/Taita_apalis, http://dbpedia.org/resource/Amphilophus_zaliosus, http://dbpedia.org/resource/Mangareva_kingfisher, http://dbpedia.org/resource/Niau_kingfisher, http://dbpedia.org/resource/Elegant_frog, http://dbpedia.org/resource/Caseolus_subcalliferus, http://dbpedia.org/resource/Discula_testudinalis, http://dbpedia.org/resource/Pachysaga_strobila, http://dbpedia.org/resource/Paracyclopia, http://dbpedia.org/resource/Parhippolyte_sterreri, http://dbpedia.org/resource/Perbrinckia_gabadagei, http://dbpedia.org/resource/Perbrinckia_glabra, http://dbpedia.org/resource/Perbrinckia_scitula, http://dbpedia.org/resource/Platycypha_amboniensis, http://dbpedia.org/resource/Procambarus_attiguus, http://dbpedia.org/resource/Procambarus_delicatus, http://dbpedia.org/resource/Procambarus_morrisi, http://dbpedia.org/resource/Procaris_chacei, http://dbpedia.org/resource/Proischnura_polychromatica, http://dbpedia.org/resource/Pseudoniphargus_grandimanus, http://dbpedia.org/resource/Rhinocypha_ogasawarensis, http://dbpedia.org/resource/Schayera_baiulus, http://dbpedia.org/resource/Sinhalestes_orientalis, http://dbpedia.org/resource/Speleoithona_bermudensis, http://dbpedia.org/resource/Speleophria_bivexilla, http://dbpedia.org/resource/Speleophria_scottodicarloi, http://dbpedia.org/resource/Streptocephalus_gracilis, http://dbpedia.org/resource/Streptocephalus_moorei, http://dbpedia.org/resource/Tetrathemis_denticauda, http://dbpedia.org/resource/Tetrathemis_ruwensoriensis, http://dbpedia.org/resource/Tettigidea_empedonepia, http://dbpedia.org/resource/Trithemis_nigra, http://dbpedia.org/resource/Bombus_suckleyi, http://dbpedia.org/resource/Partula_affinis, http://dbpedia.org/resource/Partula_clara, http://dbpedia.org/resource/Partula_emersoni, http://dbpedia.org/resource/Partula_guamensis, http://dbpedia.org/resource/Partula_langfordi, http://dbpedia.org/resource/Partula_martensiana, http://dbpedia.org/resource/Chornaya_tubenose_goby, http://dbpedia.org/resource/Abrau_sprat, http://dbpedia.org/resource/Trichonis_blenny, http://dbpedia.org/resource/Baja_Verapaz_salamander, http://dbpedia.org/resource/Monzon's_hidden_salamander, http://dbpedia.org/resource/Campo_Grande_tree_frog, http://dbpedia.org/resource/Continental_Divide_tree_frog, http://dbpedia.org/resource/Isla_Bonita_tree_frog, http://dbpedia.org/resource/Madascincus_arenicola, http://dbpedia.org/resource/Blond_capuchin, http://dbpedia.org/resource/Christmas_Island_pipistrelle, http://dbpedia.org/resource/Christmas_frigatebird, http://dbpedia.org/resource/Cone-billed_tanager, http://dbpedia.org/resource/Cotton-top_tamarin, http://dbpedia.org/resource/Golden-bellied_capuchin, http://dbpedia.org/resource/Kaapori_capuchin, http://dbpedia.org/resource/Raso_lark, http://dbpedia.org/resource/Rüppell's_vulture, http://dbpedia.org/resource/Superagui_lion_tamarin, http://dbpedia.org/resource/Black_rhinoceros, http://dbpedia.org/resource/Maui's_dolphin, http://dbpedia.org/resource/Northern_white_rhinoceros, http://dbpedia.org/resource/Puerto_Rican_amazon, http://dbpedia.org/resource/Vaquita, http://dbpedia.org/resource/Western_chimpanzee, http://dbpedia.org/resource/Saimaa_ringed_seal, http://dbpedia.org/resource/Masafuera_rayadito, http://dbpedia.org/resource/Common_sawfish, http://dbpedia.org/resource/European_mink, http://dbpedia.org/resource/Northern_white-cheeked_gibbon, http://dbpedia.org/resource/Arabian_leopard, http://dbpedia.org/resource/Dwarf_pygmy_goby, http://dbpedia.org/resource/Okinawa_woodpecker, http://dbpedia.org/resource/Amur_leopard, http://dbpedia.org/resource/Pseudoeurycea_brunnata, http://dbpedia.org/resource/São_Tomé_fiscal, http://dbpedia.org/resource/Hooded_vulture, http://dbpedia.org/resource/Indian_vulture, http://dbpedia.org/resource/Slender-billed_vulture, http://dbpedia.org/resource/White-backed_vulture, http://dbpedia.org/resource/White-headed_vulture, http://dbpedia.org/resource/Mountain_galaxias, http://dbpedia.org/resource/June_sucker, http://dbpedia.org/resource/Bog_turtle, http://dbpedia.org/resource/Delta_green_ground_beetle, http://dbpedia.org/resource/Baer's_pochard, http://dbpedia.org/resource/Blue-billed_curassow, http://dbpedia.org/resource/Golden_coin_turtle, http://dbpedia.org/resource/Radiated_tortoise, http://dbpedia.org/resource/Isabela_oriole, http://dbpedia.org/resource/Siamese_tigerfish, http://dbpedia.org/resource/Cyprinus_barbatus, http://dbpedia.org/resource/Cyprinus_fuxianensis, http://dbpedia.org/resource/Cyprinus_micristius, http://dbpedia.org/resource/Brookesia_bonsi, http://dbpedia.org/resource/Brookesia_desperata, http://dbpedia.org/resource/Calumma_tarzan, http://dbpedia.org/resource/Cnemaspis_anaikattiensis, http://dbpedia.org/resource/Flattened_musk_turtle, http://dbpedia.org/resource/Geoscincus, http://dbpedia.org/resource/Lygodactylus_williamsi, http://dbpedia.org/resource/Madagascan_big-headed_turtle, http://dbpedia.org/resource/Paroedura_lohatsara, http://dbpedia.org/resource/Abronia_campbelli, http://dbpedia.org/resource/Boninthemis, http://dbpedia.org/resource/Celestus_warreni, http://dbpedia.org/resource/Libellula_angelina, http://dbpedia.org/resource/Maltese_skate, http://dbpedia.org/resource/Saipan_reed_warbler, http://dbpedia.org/resource/Lake_Pedder_planarian, http://dbpedia.org/resource/Armoured_frog, http://dbpedia.org/resource/Atelopus_ebenoides, http://dbpedia.org/resource/Bavarian_pine_vole, http://dbpedia.org/resource/Discus_guerinianus, http://dbpedia.org/resource/Hadramphus_tuberculatus, http://dbpedia.org/resource/Isthmohyla_rivularis, http://dbpedia.org/resource/Ixalotriton_niger, http://dbpedia.org/resource/Northern_tinker_frog, http://dbpedia.org/resource/Philippine_naked-backed_fruit_bat, http://dbpedia.org/resource/Thorius_minutissimus, http://dbpedia.org/resource/Pale_lilliput_naiad, http://dbpedia.org/resource/Longcomb_sawfish, http://dbpedia.org/resource/Alosa_vistonica, http://dbpedia.org/resource/Flores_shrew, http://dbpedia.org/resource/Gansu_shrew, http://dbpedia.org/resource/Nelson's_small-eared_shrew, http://dbpedia.org/resource/Salenski's_shrew, http://dbpedia.org/resource/San_Cristobal_shrew, http://dbpedia.org/resource/Taita_shrew, http://dbpedia.org/resource/Turks_and_Caicos_rock_iguana, http://dbpedia.org/resource/Hoge's_side-necked_turtle, http://dbpedia.org/resource/Adclarkia_dawsonensis, http://dbpedia.org/resource/Allan's_lerista, http://dbpedia.org/resource/Austrogammarus_australis, http://dbpedia.org/resource/Beddomeia_tumida, http://dbpedia.org/resource/Edgbaston_goby, http://dbpedia.org/resource/Elizabeth_Springs_goby, http://dbpedia.org/resource/Galaxias_fontanus, http://dbpedia.org/resource/Galaxias_fuscus, http://dbpedia.org/resource/Geocrinia_alba, http://dbpedia.org/resource/Litoria_castanea, http://dbpedia.org/resource/Peppered_tree_frog, http://dbpedia.org/resource/Spotted_handfish, http://dbpedia.org/resource/Pethia_pookodensis, http://dbpedia.org/resource/Pyrrhosoma_elisabethae, http://dbpedia.org/resource/Diamond_darter, http://dbpedia.org/resource/Reig's_tuco-tuco, http://dbpedia.org/resource/Roig's_tuco-tuco, http://dbpedia.org/resource/Social_tuco-tuco, http://dbpedia.org/resource/Euchondrus_ramonensis, http://dbpedia.org/resource/Mexican_howler, http://dbpedia.org/resource/Rhizosomichthys_totae, http://dbpedia.org/resource/Mantidactylus_pauliani, http://dbpedia.org/resource/Philautus_jacobsoni, http://dbpedia.org/resource/Cardioglossa_alsco, http://dbpedia.org/resource/Cardioglossa_trifasciata, http://dbpedia.org/resource/Nectophrynoides_wendyae, http://dbpedia.org/resource/Golden_vizcacha_rat, http://dbpedia.org/resource/Chile_Darwin's_frog, http://dbpedia.org/resource/Cottus_rondeleti, http://dbpedia.org/resource/Exerodonta_perkinsi, http://dbpedia.org/resource/Barbodes_bovanicus, http://dbpedia.org/resource/Glyptothorax_kashmirensis, http://dbpedia.org/resource/Horalabiosa_arunachalami, http://dbpedia.org/resource/Hypselobarbus_pulchellus, http://dbpedia.org/resource/Hypselobarbus_thomassi, http://dbpedia.org/resource/Labeo_potail, http://dbpedia.org/resource/Mesonoemacheilus_herrei, http://dbpedia.org/resource/Parapsilorhynchus_prateri, http://dbpedia.org/resource/Lucifuga_simile, http://dbpedia.org/resource/Rhytida_oconnori, http://dbpedia.org/resource/Bulimulus_achatellinus, http://dbpedia.org/resource/Bulimulus_adelphus, http://dbpedia.org/resource/Bulimulus_adserseni, http://dbpedia.org/resource/Bulimulus_chemitzioides, http://dbpedia.org/resource/Bulimulus_curtus, http://dbpedia.org/resource/Bulimulus_deridderi, http://dbpedia.org/resource/Bulimulus_duncanus, http://dbpedia.org/resource/Bulimulus_eos, http://dbpedia.org/resource/Bulimulus_eschariferus, http://dbpedia.org/resource/Bulimulus_galapaganus, http://dbpedia.org/resource/Bulimulus_habeli, http://dbpedia.org/resource/Bulimulus_indefatigabilis, http://dbpedia.org/resource/Bulimulus_jacobi, http://dbpedia.org/resource/Bulimulus_lycodus, http://dbpedia.org/resource/Bulimulus_ochsneri, http://dbpedia.org/resource/Bulimulus_reibischi, http://dbpedia.org/resource/Bulimulus_saeronius, http://dbpedia.org/resource/Bulimulus_sculpturatus, http://dbpedia.org/resource/Bulimulus_sp._nov._'josevillani', http://dbpedia.org/resource/Bulimulus_sp._nov._'krameri', http://dbpedia.org/resource/Bulimulus_sp._nov._'tuideroyi', http://dbpedia.org/resource/Bulimulus_sp._nov._'vanmoli', http://dbpedia.org/resource/Bulimulus_tanneri, http://dbpedia.org/resource/Bulimulus_wolfi, http://dbpedia.org/resource/Haplochromis_annectidens, http://dbpedia.org/resource/Haplochromis_argenteus, http://dbpedia.org/resource/Haplochromis_barbarae, http://dbpedia.org/resource/Haplochromis_beadlei, http://dbpedia.org/resource/Haplochromis_brownae, http://dbpedia.org/resource/Haplochromis_cassius, http://dbpedia.org/resource/Haplochromis_crassilabris, http://dbpedia.org/resource/Haplochromis_flavipinnis, http://dbpedia.org/resource/Haplochromis_granti, http://dbpedia.org/resource/Haplochromis_guiarti, http://dbpedia.org/resource/Haplochromis_heusinkveldi, http://dbpedia.org/resource/Haplochromis_ishmaeli, http://dbpedia.org/resource/Haplochromis_latifasciatus, http://dbpedia.org/resource/Haplochromis_longirostris, http://dbpedia.org/resource/Haplochromis_macrognathus, http://dbpedia.org/resource/Haplochromis_martini, http://dbpedia.org/resource/Haplochromis_michaeli, http://dbpedia.org/resource/Haplochromis_microdon, http://dbpedia.org/resource/Haplochromis_mylergates, http://dbpedia.org/resource/Haplochromis_nanoserranus, http://dbpedia.org/resource/Haplochromis_percoides, http://dbpedia.org/resource/Haplochromis_perrieri, http://dbpedia.org/resource/Haplochromis_sp._'Kyoga_flameback', http://dbpedia.org/resource/Haplochromis_sp._'Rusinga_oral_sheller', http://dbpedia.org/resource/Haplochromis_sp._'backflash_cryptodon', http://dbpedia.org/resource/Haplochromis_sp._'black_cryptodon', http://dbpedia.org/resource/Haplochromis_sp._'parvidens-like', http://dbpedia.org/resource/Haplochromis_sp._'rainbow_sheller', http://dbpedia.org/resource/Haplochromis_sp._'ruby', http://dbpedia.org/resource/Haplochromis_sp._'small_obesoid', http://dbpedia.org/resource/Haplochromis_teegelaari, http://dbpedia.org/resource/Haplochromis_victorianus, http://dbpedia.org/resource/Haplochromis_vonlinnei, http://dbpedia.org/resource/Haplochromis_xenostoma, http://dbpedia.org/resource/Acanthobrama_hadiyahensis, http://dbpedia.org/resource/Alburnus_mandrensis, http://dbpedia.org/resource/Alburnus_nasreddini, http://dbpedia.org/resource/Alburnus_timarensis, http://dbpedia.org/resource/Alburnus_vistonicus, http://dbpedia.org/resource/Anabarilius_andersoni, http://dbpedia.org/resource/Anabarilius_qiluensis, http://dbpedia.org/resource/Anabarilius_yangzonensis, http://dbpedia.org/resource/Asian_narrow-headed_softshell_turtle, http://dbpedia.org/resource/Bedotia_albomarginata, http://dbpedia.org/resource/Bedotia_sp._nov._'Manombo', http://dbpedia.org/resource/Bedotia_sp._nov._'Sambava', http://dbpedia.org/resource/Bedotia_tricolor, http://dbpedia.org/resource/Cookeconcha_contorta, http://dbpedia.org/resource/Endodonta_apiculata, http://dbpedia.org/resource/Giant_salmon_carp, http://dbpedia.org/resource/Melanomys_zunigae, http://dbpedia.org/resource/Pseudophoxinus_maeandricus, http://dbpedia.org/resource/Twee_River_redfin, http://dbpedia.org/resource/Brazilian_guitarfish, http://dbpedia.org/resource/Chiloglanis_polyodon, http://dbpedia.org/resource/Chiloglanis_ruziziensis, http://dbpedia.org/resource/Greek_lamprey, http://dbpedia.org/resource/Louisiana_pearlshell, http://dbpedia.org/resource/Partula_taeniata, http://dbpedia.org/resource/Graecoanatolica_brevis, http://dbpedia.org/resource/Graecoanatolica_conica, http://dbpedia.org/resource/Bellamya_liberiana, http://dbpedia.org/resource/Alycaeus_balingensis, http://dbpedia.org/resource/Aphanius_richardsoni, http://dbpedia.org/resource/Barbatula_eregliensis, http://dbpedia.org/resource/Barbatula_simavica, http://dbpedia.org/resource/Belgrandiella_parreyssii, http://dbpedia.org/resource/Bermudagidiella, http://dbpedia.org/resource/Bulimulus_hirsutus, http://dbpedia.org/resource/Cryptobatrachus_nicefori, http://dbpedia.org/resource/Delminichthys_jadovensis, http://dbpedia.org/resource/Delminichthys_krbavensis, http://dbpedia.org/resource/Fluvidona_petterdi, http://dbpedia.org/resource/Glabrennea_silhouettensis, http://dbpedia.org/resource/Gyraulus_cockburni, http://dbpedia.org/resource/Haplochromis_sp._nov._'Amboseli', http://dbpedia.org/resource/Iberochondrostoma_almacai, http://dbpedia.org/resource/Iberochondrostoma_oretanum, http://dbpedia.org/resource/Labeo_lankae, http://dbpedia.org/resource/Pantanodon_sp._nov._'Manombo', http://dbpedia.org/resource/Parachondrostoma_arrigonis, http://dbpedia.org/resource/Paratilapia_sp._nov._'Vevembe', http://dbpedia.org/resource/Partula_calypso, http://dbpedia.org/resource/Partula_leucothoe, http://dbpedia.org/resource/Pelasgus_laconicus, http://dbpedia.org/resource/Phrynocephalus_horvathi, http://dbpedia.org/resource/Rhytida_clarki, http://dbpedia.org/resource/Samoana_oreas, http://dbpedia.org/resource/Samoana_strigata, http://dbpedia.org/resource/Tomichia_tristis, http://dbpedia.org/resource/Xenocypris_yunnanensis, http://dbpedia.org/resource/Zoogoneticus_tequila, http://dbpedia.org/resource/Zygonychidium_gracile, http://dbpedia.org/resource/Krabi_mouth-brooding_betta, http://dbpedia.org/resource/Gran_Canaria_blue_chaffinch, http://dbpedia.org/resource/Iriomote_cat, http://dbpedia.org/resource/Maui_parrotbill, http://dbpedia.org/resource/Nihoa_finch, http://dbpedia.org/resource/Palila, http://dbpedia.org/resource/Bonytail_chub, http://dbpedia.org/resource/Kanab_ambersnail, http://dbpedia.org/resource/Razorback_sucker, http://dbpedia.org/resource/Sumatran_rhinoceros, http://dbpedia.org/resource/Sylvilagus_palustris_hefneri, http://dbpedia.org/resource/Javan_slow_loris, http://dbpedia.org/resource/Marquesan_kingfisher, http://dbpedia.org/resource/Red-headed_vulture, http://dbpedia.org/resource/Delacour's_langur, http://dbpedia.org/resource/Eastern_black_rhinoceros, http://dbpedia.org/resource/Gray-shanked_douc, http://dbpedia.org/resource/Northern_Sumatran_rhinoceros, http://dbpedia.org/resource/Philippine_eagle, http://dbpedia.org/resource/Selous'_zebra, http://dbpedia.org/resource/Somali_wild_ass, http://dbpedia.org/resource/South-central_black_rhinoceros, http://dbpedia.org/resource/Tenkile, http://dbpedia.org/resource/Tonkin_snub-nosed_monkey, http://dbpedia.org/resource/White-headed_langur, http://dbpedia.org/resource/Balearic_shearwater, http://dbpedia.org/resource/Celebes_crested_macaque, http://dbpedia.org/resource/Corroboree_frog, http://dbpedia.org/resource/Delta_smelt, http://dbpedia.org/resource/Sumatran_ground_cuckoo, http://dbpedia.org/resource/Trinidad_piping_guan, http://dbpedia.org/resource/Apache_trout, http://dbpedia.org/resource/Black-spotted_cuscus, http://dbpedia.org/resource/Blue-eyed_spotted_cuscus, http://dbpedia.org/resource/Giant_sea_bass, http://dbpedia.org/resource/Mekong_giant_catfish, http://dbpedia.org/resource/Northern_glider, http://dbpedia.org/resource/Talaud_bear_cuscus, http://dbpedia.org/resource/Haplochromis_aelocephalus, http://dbpedia.org/resource/Haplochromis_apogonoides, http://dbpedia.org/resource/Haplochromis_dentex, http://dbpedia.org/resource/Haplochromis_dichrourus, http://dbpedia.org/resource/Seychelles_paradise_flycatcher, http://dbpedia.org/resource/Eastern_long-beaked_echidna, http://dbpedia.org/resource/Sir_David's_long-beaked_echidna, http://dbpedia.org/resource/Western_long-beaked_echidna, http://dbpedia.org/resource/Waved_albatross, http://dbpedia.org/resource/Black_stilt, http://dbpedia.org/resource/Chinese_crested_tern, http://dbpedia.org/resource/Zapata_rail, http://dbpedia.org/resource/Anderson's_salamander, http://dbpedia.org/resource/Blue-crowned_laughingthrush, http://dbpedia.org/resource/Delicate-skinned_salamander, http://dbpedia.org/resource/Granular_salamander, http://dbpedia.org/resource/Lake_Lerma_salamander, http://dbpedia.org/resource/Lake_Patzcuaro_salamander, http://dbpedia.org/resource/Taylor's_salamander, http://dbpedia.org/resource/Carrizal_seedeater, http://dbpedia.org/resource/Iquitos_gnatcatcher, http://dbpedia.org/resource/Fiji_crested_iguana, http://dbpedia.org/resource/Cophotis_dumbara, http://dbpedia.org/resource/Barkudia_insularis, http://dbpedia.org/resource/Chalcides_ebneri, http://dbpedia.org/resource/Brown_spider_monkey, http://dbpedia.org/resource/Kipunji, http://dbpedia.org/resource/Namdapha_flying_squirrel, http://dbpedia.org/resource/Pennant's_colobus, http://dbpedia.org/resource/Preuss's_red_colobus, http://dbpedia.org/resource/Sarawak_surili, http://dbpedia.org/resource/Bahama_oriole, http://dbpedia.org/resource/Socorro_springsnail, http://dbpedia.org/resource/Manusela_mosaic-tailed_rat, http://dbpedia.org/resource/Woundfin, http://dbpedia.org/resource/Gulickia_alexandri, http://dbpedia.org/resource/São_Tomé_grosbeak, http://dbpedia.org/resource/Puntius_deccanensis, http://dbpedia.org/resource/Barbara_Brown's_titi, http://dbpedia.org/resource/Black_bearded_saki, http://dbpedia.org/resource/Rio_Mayo_titi, http://dbpedia.org/resource/Hylomantis_lemur, http://dbpedia.org/resource/Armenian_tortoise, http://dbpedia.org/resource/Angel_Island_mouse, http://dbpedia.org/resource/Burt's_deer_mouse, http://dbpedia.org/resource/Dickey's_deer_mouse, http://dbpedia.org/resource/False_canyon_mouse, http://dbpedia.org/resource/San_Esteban_Island_mouse, http://dbpedia.org/resource/San_Lorenzo_mouse, http://dbpedia.org/resource/Neritina_tiassalensis, http://dbpedia.org/resource/Mount_Kahuzi_climbing_mouse, http://dbpedia.org/resource/Myanmar_snub-nosed_monkey, http://dbpedia.org/resource/Opisthostoma_mirabile, http://dbpedia.org/resource/Zingel_asper, http://dbpedia.org/resource/Alcorn's_pocket_gopher, http://dbpedia.org/resource/Big_pocket_gopher, http://dbpedia.org/resource/Biomphalaria_barthi, http://dbpedia.org/resource/Leora's_stream_salamander, http://dbpedia.org/resource/Iberolacerta_martinezricai, http://dbpedia.org/resource/Hucho_bleekeri, http://dbpedia.org/resource/Barbodes_disa, http://dbpedia.org/resource/Sangihe_shrikethrush, http://dbpedia.org/resource/Lamotte's_roundleaf_bat, http://dbpedia.org/resource/Anodonthyla_vallani, http://dbpedia.org/resource/Cophyla_berara, http://dbpedia.org/resource/Striped_smooth-hound, http://dbpedia.org/resource/Pungu, http://dbpedia.org/resource/Ancylus_ashangiensis, http://dbpedia.org/resource/Heliconius_nattereri, http://dbpedia.org/resource/Angonoka_tortoise, http://dbpedia.org/resource/Be'er_Sheva_fringe-fingered_lizard, http://dbpedia.org/resource/Cyclura_cychlura_figginsi, http://dbpedia.org/resource/Cyclura_rileyi_cristata, http://dbpedia.org/resource/Doumergue's_fringe-fingered_lizard, http://dbpedia.org/resource/Kikuzato's_brook_snake, http://dbpedia.org/resource/Saint_Vincent_blacksnake, http://dbpedia.org/resource/Tylomelania_kruimeli, http://dbpedia.org/resource/Lampedusa_melitensis, http://dbpedia.org/resource/Roatan_coral_snake, http://dbpedia.org/resource/Cerro_Pital_salamander, http://dbpedia.org/resource/Austrodontura_castletoni, http://dbpedia.org/resource/Trioza_barrettae, http://dbpedia.org/resource/Ganges_shark, http://dbpedia.org/resource/Gharial, http://dbpedia.org/resource/Great_Indian_bustard, http://dbpedia.org/resource/Mariana_crow, http://dbpedia.org/resource/Northern_bald_ibis, http://dbpedia.org/resource/Pygmy_three-toed_sloth, http://dbpedia.org/resource/Flores_hawk-eagle, http://dbpedia.org/resource/Oncorhynchus_masou_formosanus, http://dbpedia.org/resource/Bugun_liocichla, http://dbpedia.org/resource/Cherax_tenuimanus, http://dbpedia.org/resource/Javan_leopard, http://dbpedia.org/resource/Opisthopatus_roseus, http://dbpedia.org/resource/Speleoperipatus, http://dbpedia.org/resource/Swift_parrot, http://dbpedia.org/resource/Javan_green_magpie, http://dbpedia.org/resource/Negev_Tortoise, http://dbpedia.org/resource/Puaiohi, http://dbpedia.org/resource/Christmas_Island_forest_skink, http://dbpedia.org/resource/Kangaroo_Island_dunnart, http://dbpedia.org/resource/Chatham_shag, http://dbpedia.org/resource/Miami_blue, http://dbpedia.org/resource/Culebra_Island_giant_anole, http://dbpedia.org/resource/Dexteria, http://dbpedia.org/resource/Epioblasma_torulosa, http://dbpedia.org/resource/Puebla_deer_mouse, http://dbpedia.org/resource/Ridgway's_hawk, http://dbpedia.org/resource/San_Quintin_kangaroo_rat, http://dbpedia.org/resource/Sclater's_shrew, http://dbpedia.org/resource/Shortnose_cisco, http://dbpedia.org/resource/Sociable_lapwing, http://dbpedia.org/resource/Spoon-billed_sandpiper, http://dbpedia.org/resource/Web-footed_coquí, http://dbpedia.org/resource/Daggernose_shark, http://dbpedia.org/resource/Irrawaddy_river_shark, http://dbpedia.org/resource/Northern_river_shark, http://dbpedia.org/resource/Pagai_Island_macaque, http://dbpedia.org/resource/Pondicherry_shark, http://dbpedia.org/resource/Devils_Hole_pupfish, http://dbpedia.org/resource/Madagascan_pochard, http://dbpedia.org/resource/Salmo_carpio, http://dbpedia.org/resource/Long-billed_forest_warbler, http://dbpedia.org/resource/Aprasia_aurita, http://dbpedia.org/resource/Black-headed_spider_monkey, http://dbpedia.org/resource/Bay_Lycian_salamander, http://dbpedia.org/resource/Chinhai_spiny_newt, http://dbpedia.org/resource/Kaiser's_spotted_newt, http://dbpedia.org/resource/Kurdistan_spotted_newt, http://dbpedia.org/resource/Atelopus_varius, http://dbpedia.org/resource/Lesser_electric_ray, http://dbpedia.org/resource/Schizothorax_grahami, http://dbpedia.org/resource/Sinocyclocheilus_grahami, http://dbpedia.org/resource/Azuero_howler, http://dbpedia.org/resource/Northern_brown_howler, http://dbpedia.org/resource/Geckoella_jeyporensis, http://dbpedia.org/resource/Cyclura_carinata_bartschi, http://dbpedia.org/resource/Madagascan_fish_eagle, http://dbpedia.org/resource/Santamartamys, http://dbpedia.org/resource/Clanwilliam_sandfish, http://dbpedia.org/resource/Chevron-spotted_brown_frog, http://dbpedia.org/resource/Copan_brook_frog, http://dbpedia.org/resource/Honduran_brook_frog, http://dbpedia.org/resource/Eisentraut's_mouse_shrew, http://dbpedia.org/resource/São_Tomé_shrew, http://dbpedia.org/resource/Indochinese_box_turtle, http://dbpedia.org/resource/Vietnamese_pond_turtle, http://dbpedia.org/resource/Leptodactylus_magistris, http://dbpedia.org/resource/Leptodactylus_silvanimbus, http://dbpedia.org/resource/Vietnamese_three-striped_box_turtle, http://dbpedia.org/resource/Alabama_cavefish, http://dbpedia.org/resource/Pachliopta_jophon, http://dbpedia.org/resource/Vipera_darevskii, http://dbpedia.org/resource/Vipera_wagneri, http://dbpedia.org/resource/Bleeding_toad, http://dbpedia.org/resource/Nimbaphrynoides_liberiensis, http://dbpedia.org/resource/Western_Nimba_toad, http://dbpedia.org/resource/Telestes_turskyi, http://dbpedia.org/resource/Western_swamp_turtle, http://dbpedia.org/resource/Aaadonta_irregularis, http://dbpedia.org/resource/Telestes_fontinalis, http://dbpedia.org/resource/Telestes_polylepis, http://dbpedia.org/resource/Cracking_pearlymussel, http://dbpedia.org/resource/Pondoland_cannibal_snail, http://dbpedia.org/resource/Alexteroon_jynx, http://dbpedia.org/resource/Amji's_salamander, http://dbpedia.org/resource/Pickersgill's_reed_frog, http://dbpedia.org/resource/Spencer's_river_tree_frog, http://dbpedia.org/resource/Bokermannohyla_izecksohni, http://dbpedia.org/resource/Pethia_bandula, http://dbpedia.org/resource/Elongate_bitterling, http://dbpedia.org/resource/Amanipodagrion, http://dbpedia.org/resource/Blind_cave_loach, http://dbpedia.org/resource/Pseudochazara_cingovskii, http://dbpedia.org/resource/Vermilion_darter, http://dbpedia.org/resource/Raorchestes_chlorosomma, http://dbpedia.org/resource/Franklin's_bumblebee, http://dbpedia.org/resource/Cloud_forest_salamander, http://dbpedia.org/resource/El_Cusuco_salamander, http://dbpedia.org/resource/Longest_climbing_salamander, http://dbpedia.org/resource/Monte_Escondido_salamander, http://dbpedia.org/resource/Orphan_salamander, http://dbpedia.org/resource/Zarciadero_web-footed_salamander, http://dbpedia.org/resource/Pilalo_tree_frog, http://dbpedia.org/resource/Acıgöl_toothcarp, http://dbpedia.org/resource/Arytropteris_pondo, http://dbpedia.org/resource/Black-spotted_false_shieldback, http://dbpedia.org/resource/Kızılırmak_toothcarp, http://dbpedia.org/resource/Nike's_squeaker, http://dbpedia.org/resource/Eupropacris_abbreviata, http://dbpedia.org/resource/Laysan_duck, http://dbpedia.org/resource/Leptodactylus_fallax, http://dbpedia.org/resource/Montserrat_galliwasp, http://dbpedia.org/resource/Montserrat_oriole, http://dbpedia.org/resource/Dryas_monkey, http://dbpedia.org/resource/Antiguan_racer, http://dbpedia.org/resource/Chinese_pangolin, http://dbpedia.org/resource/De_Winton's_golden_mole, http://dbpedia.org/resource/Silvery_pigeon, http://dbpedia.org/resource/Amsterdam_albatross, http://dbpedia.org/resource/Fiji_petrel, http://dbpedia.org/resource/Magenta_petrel, http://dbpedia.org/resource/New_Zealand_storm_petrel, http://dbpedia.org/resource/Kaluga_(fish), http://dbpedia.org/resource/Lake_Oku_clawed_frog, http://dbpedia.org/resource/Giant_pangasius, http://dbpedia.org/resource/Amastra_rubens, http://dbpedia.org/resource/Baw_Baw_frog, http://dbpedia.org/resource/Coxen's_fig_parrot, http://dbpedia.org/resource/Kroombit_tinker_frog, http://dbpedia.org/resource/Munchique_wood_wren, http://dbpedia.org/resource/Niceforo's_wren, http://dbpedia.org/resource/Nothomyrmecia, http://dbpedia.org/resource/Orange-bellied_parrot, http://dbpedia.org/resource/Santa_Marta_wren, http://dbpedia.org/resource/Western_ground_parrot, http://dbpedia.org/resource/Woylie, http://dbpedia.org/resource/Burmese_star_tortoise, http://dbpedia.org/resource/Flat-backed_spider_tortoise, http://dbpedia.org/resource/Geometric_tortoise, http://dbpedia.org/resource/Kleinmann's_tortoise, http://dbpedia.org/resource/Spider_tortoise, http://dbpedia.org/resource/Cozumel_raccoon, http://dbpedia.org/resource/Banggai_crow, http://dbpedia.org/resource/Thickshell_pondsnail, http://dbpedia.org/resource/Anjouan_scops_owl, http://dbpedia.org/resource/Giant_ibis, http://dbpedia.org/resource/Malabar_large-spotted_civet, http://dbpedia.org/resource/São_Tomé_ibis, http://dbpedia.org/resource/White-shouldered_ibis, http://dbpedia.org/resource/White-bellied_heron, http://dbpedia.org/resource/Gilbert's_potoroo, http://dbpedia.org/resource/Príncipe_thrush, http://dbpedia.org/resource/Taita_thrush, http://dbpedia.org/resource/Karthala_scops_owl, http://dbpedia.org/resource/Narrow-ridged_finless_porpoise, http://dbpedia.org/resource/Cuban_kite, http://dbpedia.org/resource/Central_American_river_turtle, http://dbpedia.org/resource/Cyclura_nubila_caymanensis, http://dbpedia.org/resource/Rhodeus_smithii, http://dbpedia.org/resource/Convex_horseshoe_bat, http://dbpedia.org/resource/Jamaican_flower_bat, http://dbpedia.org/resource/Lord_Howe_flax_snail, http://dbpedia.org/resource/Rhinolophus_hilli, http://dbpedia.org/resource/Solitary_fruit-eating_bat, http://dbpedia.org/resource/Archey's_frog, http://dbpedia.org/resource/Moheli_scops_owl, http://dbpedia.org/resource/Nihoa_millerbird, http://dbpedia.org/resource/Northern_river_terrapin, http://dbpedia.org/resource/Conasauga_logperch, http://dbpedia.org/resource/Ingerana_charlesdarwini, http://dbpedia.org/resource/Chinanteco_deer_mouse, http://dbpedia.org/resource/Cozumel_harvest_mouse, http://dbpedia.org/resource/Delicate_deer_mouse, http://dbpedia.org/resource/Ixtlán_deer_mouse, http://dbpedia.org/resource/Maya_mouse, http://dbpedia.org/resource/Nelson's_woodrat, http://dbpedia.org/resource/Perote_mouse, http://dbpedia.org/resource/Slevin's_mouse, http://dbpedia.org/resource/Zempoaltepec_deer_mouse, http://dbpedia.org/resource/Booroolong_frog, http://dbpedia.org/resource/Atelopus_andinus, http://dbpedia.org/resource/Atelopus_angelito, http://dbpedia.org/resource/Atelopus_arthuri, http://dbpedia.org/resource/Atelopus_balios, http://dbpedia.org/resource/Atelopus_boulengeri, http://dbpedia.org/resource/Atelopus_chocoensis, http://dbpedia.org/resource/Atelopus_chrysocorallus, http://dbpedia.org/resource/Atelopus_coynei, http://dbpedia.org/resource/Atelopus_erythropus, http://dbpedia.org/resource/Atelopus_eusebianus, http://dbpedia.org/resource/Atelopus_eusebiodiazi, http://dbpedia.org/resource/Atelopus_exiguus, http://dbpedia.org/resource/Atelopus_famelicus, http://dbpedia.org/resource/Atelopus_galactogaster, http://dbpedia.org/resource/Atelopus_glyphus, http://dbpedia.org/resource/Atelopus_guitarraensis, http://dbpedia.org/resource/Atelopus_halihelos, http://dbpedia.org/resource/Atelopus_lozanoi, http://dbpedia.org/resource/Atelopus_lynchi, http://dbpedia.org/resource/Atelopus_mandingues, http://dbpedia.org/resource/Atelopus_mindoensis, http://dbpedia.org/resource/Atelopus_minutulus, http://dbpedia.org/resource/Atelopus_monohernandezii, http://dbpedia.org/resource/Atelopus_mucubajiensis, http://dbpedia.org/resource/Atelopus_muisca, http://dbpedia.org/resource/Atelopus_nanay, http://dbpedia.org/resource/Atelopus_nepiozomus, http://dbpedia.org/resource/Atelopus_nicefori, http://dbpedia.org/resource/Atelopus_onorei, http://dbpedia.org/resource/Atelopus_oxyrhynchus, http://dbpedia.org/resource/Atelopus_pachydermus, http://dbpedia.org/resource/Atelopus_patazensis, http://dbpedia.org/resource/Atelopus_pedimarmoratus, http://dbpedia.org/resource/Atelopus_peruensis, http://dbpedia.org/resource/Atelopus_petersi, http://dbpedia.org/resource/Atelopus_petriruizi, http://dbpedia.org/resource/Atelopus_pictiventris, http://dbpedia.org/resource/Atelopus_planispina, http://dbpedia.org/resource/Atelopus_reticulatus, http://dbpedia.org/resource/Atelopus_seminiferus, http://dbpedia.org/resource/Atelopus_sernai, http://dbpedia.org/resource/Atelopus_subornatus, http://dbpedia.org/resource/Atelopus_walkeri]... [INFO] Predicted Answer: [http://dbpedia.org/resource/Acarapis_woodi, http://dbpedia.org/resource/Acari, http://dbpedia.org/resource/African_fish_eagle, http://dbpedia.org/resource/Africanized_bee, http://dbpedia.org/resource/Agriotherium, http://dbpedia.org/resource/Ailurarctos, http://dbpedia.org/resource/Ailuridae, http://dbpedia.org/resource/Ailuropoda_baconi, http://dbpedia.org/resource/Ailuropoda_microta, http://dbpedia.org/resource/Alaska_Peninsula_brown_bear, http://dbpedia.org/resource/Aleutian_tern, http://dbpedia.org/resource/Alkali_bee, http://dbpedia.org/resource/Allodapini, http://dbpedia.org/resource/Amegilla_dawsoni, http://dbpedia.org/resource/American_coot, http://dbpedia.org/resource/American_gizzard_shad, http://dbpedia.org/resource/American_herring_gull, http://dbpedia.org/resource/American_red_squirrel, http://dbpedia.org/resource/Amphicynodon, http://dbpedia.org/resource/Ampulicidae, http://dbpedia.org/resource/Andrenidae, http://dbpedia.org/resource/Anthophorini, http://dbpedia.org/resource/Arctic_ground_squirrel, http://dbpedia.org/resource/Arctotherium, http://dbpedia.org/resource/Ardea_(genus), http://dbpedia.org/resource/Ardipithecus, http://dbpedia.org/resource/Army_cutworm, http://dbpedia.org/resource/Ascidiacea, http://dbpedia.org/resource/Asian_palm_civet, http://dbpedia.org/resource/Astatinae, http://dbpedia.org/resource/Australopithecus_afarensis, http://dbpedia.org/resource/Bali_myna, http://dbpedia.org/resource/Baltimore_oriole, http://dbpedia.org/resource/Barracuda, http://dbpedia.org/resource/Basiliscus_(genus), http://dbpedia.org/resource/Bearded_seal, http://dbpedia.org/resource/Beetle, http://dbpedia.org/resource/Beewolf, http://dbpedia.org/resource/Bembicini, http://dbpedia.org/resource/Black-naped_oriole, http://dbpedia.org/resource/Black_grouse, http://dbpedia.org/resource/Black_rat, http://dbpedia.org/resource/Blue_jay, http://dbpedia.org/resource/Blue_quail, http://dbpedia.org/resource/Bombus_hyperboreus, http://dbpedia.org/resource/Borneo_elephant, http://dbpedia.org/resource/Broad_whitefish, http://dbpedia.org/resource/Brown_bullhead, http://dbpedia.org/resource/Brown_pelican, http://dbpedia.org/resource/Bushbuck, http://dbpedia.org/resource/Buteo, http://dbpedia.org/resource/Caddisfly, http://dbpedia.org/resource/California_grizzly_bear, http://dbpedia.org/resource/California_quail, http://dbpedia.org/resource/Carpenter_bee, http://dbpedia.org/resource/Castor_californicus, http://dbpedia.org/resource/Castoridae, http://dbpedia.org/resource/Castorimorpha, http://dbpedia.org/resource/Castoroides, http://dbpedia.org/resource/Catostomus, http://dbpedia.org/resource/Cave_bear, http://dbpedia.org/resource/Cave_hyena, http://dbpedia.org/resource/Cephalogale, http://dbpedia.org/resource/Ceratina, http://dbpedia.org/resource/Chalicothere, http://dbpedia.org/resource/Chum_salmon, http://dbpedia.org/resource/Ciona_intestinalis, http://dbpedia.org/resource/Coccinellidae, http://dbpedia.org/resource/Coho_salmon, http://dbpedia.org/resource/Colinus, http://dbpedia.org/resource/Colletes_inaequalis, http://dbpedia.org/resource/Colletidae, http://dbpedia.org/resource/Common_bottlenose_dolphin, http://dbpedia.org/resource/Common_frog, http://dbpedia.org/resource/Common_galaxias, http://dbpedia.org/resource/Common_loon, http://dbpedia.org/resource/Common_snapping_turtle, http://dbpedia.org/resource/Coot, http://dbpedia.org/resource/Copenhagen_(horse), http://dbpedia.org/resource/Cottontail_rabbit, http://dbpedia.org/resource/Crab-eating_macaque, http://dbpedia.org/resource/Crabronidae, http://dbpedia.org/resource/Crabroninae, http://dbpedia.org/resource/Crested_treeswift, http://dbpedia.org/resource/Ctenosaura, http://dbpedia.org/resource/Culicoides, http://dbpedia.org/resource/Culicoides_imicola, http://dbpedia.org/resource/Culicoides_obsoletus, http://dbpedia.org/resource/Culicoides_variipennis, http://dbpedia.org/resource/Cutthroat_trout, http://dbpedia.org/resource/Dayak_fruit_bat, http://dbpedia.org/resource/Diabrotica, http://dbpedia.org/resource/Dichistius, http://dbpedia.org/resource/Dipodomyinae, http://dbpedia.org/resource/Dollarbird, http://dbpedia.org/resource/Drosophilidae, http://dbpedia.org/resource/East_Siberian_brown_bear, http://dbpedia.org/resource/Eastern_kingbird, http://dbpedia.org/resource/Egret, http://dbpedia.org/resource/Emperor_goose, http://dbpedia.org/resource/Eulachon, http://dbpedia.org/resource/Eurasian_beaver, http://dbpedia.org/resource/Eurasian_brown_bear, http://dbpedia.org/resource/European_corn_borer, http://dbpedia.org/resource/European_honey_buzzard, http://dbpedia.org/resource/European_robin, http://dbpedia.org/resource/Even-toed_ungulate, http://dbpedia.org/resource/Fall_armyworm, http://dbpedia.org/resource/Fathead_minnow, http://dbpedia.org/resource/Ferruginous_hawk, http://dbpedia.org/resource/Firmicutes, http://dbpedia.org/resource/Flea, http://dbpedia.org/resource/Fossa_(animal), http://dbpedia.org/resource/Gall_wasp, http://dbpedia.org/resource/Gambel's_quail, http://dbpedia.org/resource/Gelada, http://dbpedia.org/resource/Giant_moray, http://dbpedia.org/resource/Giant_pangolin, http://dbpedia.org/resource/Gigantopithecus, http://dbpedia.org/resource/Gobi_bear, http://dbpedia.org/resource/Grasshopper, http://dbpedia.org/resource/Grayling_(species), http://dbpedia.org/resource/Great_black-backed_gull, http://dbpedia.org/resource/Great_blue_heron, http://dbpedia.org/resource/Great_egret, http://dbpedia.org/resource/Greater_honeyguide, http://dbpedia.org/resource/Grey_reef_shark, http://dbpedia.org/resource/Ground_pangolin, http://dbpedia.org/resource/Grus_(genus), http://dbpedia.org/resource/Gyrfalcon, http://dbpedia.org/resource/Halictidae, http://dbpedia.org/resource/Helicoverpa_zea, http://dbpedia.org/resource/Hemicyoninae, http://dbpedia.org/resource/Hermann's_tortoise, http://dbpedia.org/resource/Heteromyidae, http://dbpedia.org/resource/Himalayan_brown_bear, http://dbpedia.org/resource/Himalayan_marmot, http://dbpedia.org/resource/Homotherium, http://dbpedia.org/resource/Hornet, http://dbpedia.org/resource/Horseshoe_crab, http://dbpedia.org/resource/Hose's_palm_civet, http://dbpedia.org/resource/Ictalurus, http://dbpedia.org/resource/Indarctos, http://dbpedia.org/resource/Japanese_quail, http://dbpedia.org/resource/Japanese_rat_snake, http://dbpedia.org/resource/Java_sparrow, http://dbpedia.org/resource/Kamchatka_brown_bear, http://dbpedia.org/resource/Kangaroo, http://dbpedia.org/resource/Kangaroo_rat, http://dbpedia.org/resource/Kodiak_bear, http://dbpedia.org/resource/Kolponomos, http://dbpedia.org/resource/Largescale_sucker, http://dbpedia.org/resource/Lasioglossum_leucozonium, http://dbpedia.org/resource/Lasioglossum_vierecki, http://dbpedia.org/resource/Leopard_cat, http://dbpedia.org/resource/Leopard_seal, http://dbpedia.org/resource/Lesser_adjutant, http://dbpedia.org/resource/Linckia, http://dbpedia.org/resource/Long-tailed_shrike, http://dbpedia.org/resource/Longnose_sucker, http://dbpedia.org/resource/Louse, http://dbpedia.org/resource/Lycidae, http://dbpedia.org/resource/Lycopodiophyta, http://dbpedia.org/resource/MacQueen's_bustard, http://dbpedia.org/resource/Malayan_tapir, http://dbpedia.org/resource/Mamba, http://dbpedia.org/resource/Man_o'_War, http://dbpedia.org/resource/Mandarin_duck, http://dbpedia.org/resource/Marsican_brown_bear, http://dbpedia.org/resource/Marsupial_lion, http://dbpedia.org/resource/Marten, http://dbpedia.org/resource/Meadow_Court, http://dbpedia.org/resource/Meerkat, http://dbpedia.org/resource/Megachile_pluto, http://dbpedia.org/resource/Meganthropus, http://dbpedia.org/resource/Melittidae, http://dbpedia.org/resource/Melittosphex_burmensis, http://dbpedia.org/resource/Merginae, http://dbpedia.org/resource/Mexican_grizzly_bear, http://dbpedia.org/resource/Midge, http://dbpedia.org/resource/Milky_stork, http://dbpedia.org/resource/Mink, http://dbpedia.org/resource/Morpho_peleides, http://dbpedia.org/resource/Mountain_quail, http://dbpedia.org/resource/Mycetozoa, http://dbpedia.org/resource/Needlefish, http://dbpedia.org/resource/North_American_beaver, http://dbpedia.org/resource/North_American_porcupine, http://dbpedia.org/resource/Northern_gannet, http://dbpedia.org/resource/Northern_pocket_gopher, http://dbpedia.org/resource/Nototherium, http://dbpedia.org/resource/Nyssonini, http://dbpedia.org/resource/Old_World_flycatcher, http://dbpedia.org/resource/Oncorhynchus_masou, http://dbpedia.org/resource/Oriental_rat_flea, http://dbpedia.org/resource/Pacific_herring, http://dbpedia.org/resource/Pacific_razor_clam, http://dbpedia.org/resource/Pacific_swallow, http://dbpedia.org/resource/Pallas's_reed_bunting, http://dbpedia.org/resource/Palorchestes, http://dbpedia.org/resource/Panthera_leo_spelaea, http://dbpedia.org/resource/Paper_wasp, http://dbpedia.org/resource/Paranthropus, http://dbpedia.org/resource/Paranthropus_robustus, http://dbpedia.org/resource/Parictis, http://dbpedia.org/resource/Pemphredoninae, http://dbpedia.org/resource/Pernis_(bird), http://dbpedia.org/resource/Philanthinae, http://dbpedia.org/resource/Phoberocyon, http://dbpedia.org/resource/Pika, http://dbpedia.org/resource/Pink_bollworm, http://dbpedia.org/resource/Pink_salmon, http://dbpedia.org/resource/Plateau_pika, http://dbpedia.org/resource/Plionarctos, http://dbpedia.org/resource/Plithocyon, http://dbpedia.org/resource/Plover, http://dbpedia.org/resource/Pollen_wasp, http://dbpedia.org/resource/Prasinohaema, http://dbpedia.org/resource/Proteobacteria, http://dbpedia.org/resource/Rain_quail, http://dbpedia.org/resource/Red-rumped_swallow, http://dbpedia.org/resource/Red-tailed_hawk, http://dbpedia.org/resource/Reticulated_python, http://dbpedia.org/resource/Rough-legged_buzzard, http://dbpedia.org/resource/Sacred_kingfisher, http://dbpedia.org/resource/Sahelanthropus, http://dbpedia.org/resource/Sand_lance, http://dbpedia.org/resource/Sandhill_crane, http://dbpedia.org/resource/Savanna_nightjar, http://dbpedia.org/resource/Scaled_quail, http://dbpedia.org/resource/Seabiscuit, http://dbpedia.org/resource/Short-beaked_echidna, http://dbpedia.org/resource/Short-faced_bear, http://dbpedia.org/resource/Siberian_chipmunk, http://dbpedia.org/resource/Siberian_crane, http://dbpedia.org/resource/Skink, http://dbpedia.org/resource/Slipper_lobster, http://dbpedia.org/resource/Sloth_bear, http://dbpedia.org/resource/Snow_goose, http://dbpedia.org/resource/Snowshoe_hare, http://dbpedia.org/resource/Sockeye_salmon, http://dbpedia.org/resource/Southern_elephant_seal, http://dbpedia.org/resource/Sphecidae, http://dbpedia.org/resource/Spot-bellied_bobwhite, http://dbpedia.org/resource/Spotted_hyena, http://dbpedia.org/resource/Spotted_seal, http://dbpedia.org/resource/Spotted_skunk, http://dbpedia.org/resource/Squash_bee, http://dbpedia.org/resource/Stenotritidae, http://dbpedia.org/resource/Sternotherus_odoratus, http://dbpedia.org/resource/Stingless_bee, http://dbpedia.org/resource/Stork-billed_kingfisher, http://dbpedia.org/resource/Storm_petrel, http://dbpedia.org/resource/Sun_bear, http://dbpedia.org/resource/Sunda_pangolin, http://dbpedia.org/resource/Syrian_brown_bear, http://dbpedia.org/resource/Takydromus, http://dbpedia.org/resource/Tetragonula_carbonaria, http://dbpedia.org/resource/Threadfin_shad, http://dbpedia.org/resource/Tick, http://dbpedia.org/resource/Tremarctinae, http://dbpedia.org/resource/Tremarctos, http://dbpedia.org/resource/Tremarctos_floridanus, http://dbpedia.org/resource/Trichinella_spiralis, http://dbpedia.org/resource/Trogontherium, http://dbpedia.org/resource/Trumpeter_swan, http://dbpedia.org/resource/Ursavini, http://dbpedia.org/resource/Ursavus, http://dbpedia.org/resource/Ursinae, http://dbpedia.org/resource/Ursus_(genus), http://dbpedia.org/resource/Ursus_americanus_carlottae, http://dbpedia.org/resource/Ursus_deningeri, http://dbpedia.org/resource/Ursus_etruscus, http://dbpedia.org/resource/Ursus_maritimus_tyrannus, http://dbpedia.org/resource/Ursus_minimus, http://dbpedia.org/resource/Ursus_rossicus, http://dbpedia.org/resource/Ussuri_brown_bear, http://dbpedia.org/resource/Vampire_bat, http://dbpedia.org/resource/Varroa, http://dbpedia.org/resource/Vertebrata, http://dbpedia.org/resource/Vespoidea, http://dbpedia.org/resource/Western_corn_rootworm, http://dbpedia.org/resource/Western_grebe, http://dbpedia.org/resource/White-tailed_eagle, http://dbpedia.org/resource/White_bass, http://dbpedia.org/resource/Whooper_swan, http://dbpedia.org/resource/Whooping_crane, http://dbpedia.org/resource/Xenopus, http://dbpedia.org/resource/Yellow-crested_cockatoo, http://dbpedia.org/resource/Yellow-vented_bulbul, http://dbpedia.org/resource/Zarigani, http://dbpedia.org/resource/Zebrafish, http://dbpedia.org/resource/Zygomaturus, http://dbpedia.org/resource/Abbottina, http://dbpedia.org/resource/Abronia_lythrochila, http://dbpedia.org/resource/Acanthobrama, http://dbpedia.org/resource/Acheilognathinae, http://dbpedia.org/resource/Acheilognathus, http://dbpedia.org/resource/Achondrostoma, http://dbpedia.org/resource/Acrossocheilus, http://dbpedia.org/resource/Adephaga, http://dbpedia.org/resource/Aelurodon, http://dbpedia.org/resource/African_wildcat, http://dbpedia.org/resource/Alburnoides, http://dbpedia.org/resource/Alburnus, http://dbpedia.org/resource/Aleochara, http://dbpedia.org/resource/Alewife, http://dbpedia.org/resource/Algansea, http://dbpedia.org/resource/Allen's_chipmunk, http://dbpedia.org/resource/Alpine_chipmunk, http://dbpedia.org/resource/Amblypharyngodon, http://dbpedia.org/resource/Amblyrhynchichthys, http://dbpedia.org/resource/Amblytelus, http://dbpedia.org/resource/American_badger, http://dbpedia.org/resource/Anabarilius, http://dbpedia.org/resource/Anabarilius_liui_liui, http://dbpedia.org/resource/Anabarilius_polylepis, http://dbpedia.org/resource/Ancherythroculter, http://dbpedia.org/resource/Ancylostoma, http://dbpedia.org/resource/Ancylostoma_caninum, http://dbpedia.org/resource/Anguillidae, http://dbpedia.org/resource/Anobiidae, http://dbpedia.org/resource/Anthia, http://dbpedia.org/resource/Aphyocypris, http://dbpedia.org/resource/Arabian_wolf, http://dbpedia.org/resource/Arabibarbus, http://dbpedia.org/resource/Archaeocyon, http://dbpedia.org/resource/Archostemata, http://dbpedia.org/resource/Arctic_wolf, http://dbpedia.org/resource/Arctocyonidae, http://dbpedia.org/resource/Asp_(fish), http://dbpedia.org/resource/Aspidoparia, http://dbpedia.org/resource/Aspius, http://dbpedia.org/resource/Atrilinea, http://dbpedia.org/resource/Attelabidae, http://dbpedia.org/resource/Australian_feral_camel, http://dbpedia.org/resource/Austroplatypus_incompertus, http://dbpedia.org/resource/Baffin_Island_wolf, http://dbpedia.org/resource/Baiji, http://dbpedia.org/resource/Balantiocheilos, http://dbpedia.org/resource/Balkan_green_lizard, http://dbpedia.org/resource/Balkan_whip_snake, http://dbpedia.org/resource/Ballerus, http://dbpedia.org/resource/Banded_cat-eyed_snake, http://dbpedia.org/resource/Bangana, http://dbpedia.org/resource/Barbodes, http://dbpedia.org/resource/Barboides, http://dbpedia.org/resource/Barbonymus, http://dbpedia.org/resource/Barbus, http://dbpedia.org/resource/Barilius, http://dbpedia.org/resource/Bark_beetle, http://dbpedia.org/resource/Barred_frogs, http://dbpedia.org/resource/Barycnemis_blediator, http://dbpedia.org/resource/Belligobio, http://dbpedia.org/resource/Bernard's_wolf, http://dbpedia.org/resource/Bettong, http://dbpedia.org/resource/Bighead_carp, http://dbpedia.org/resource/Biwia, http://dbpedia.org/resource/Black-backed_jackal, http://dbpedia.org/resource/Black_carp, http://dbpedia.org/resource/Blasius's_horseshoe_bat, http://dbpedia.org/resource/Bledius_spectabilis, http://dbpedia.org/resource/Blister_beetle, http://dbpedia.org/resource/Bombardier_beetle, http://dbpedia.org/resource/Bombus_hortorum, http://dbpedia.org/resource/Bombyx_mori, http://dbpedia.org/resource/Bonelli's_eagle, http://dbpedia.org/resource/Boraras, http://dbpedia.org/resource/Borophaginae, http://dbpedia.org/resource/Borophagus, http://dbpedia.org/resource/Borophagus_diversidens, http://dbpedia.org/resource/Borophagus_dudleyi, http://dbpedia.org/resource/Borophagus_hilli, http://dbpedia.org/resource/Borophagus_littoralis, http://dbpedia.org/resource/Borophagus_orc, http://dbpedia.org/resource/Borophagus_parvus, http://dbpedia.org/resource/Borophagus_pugnator, http://dbpedia.org/resource/Borophagus_secundus, http://dbpedia.org/resource/Brevibora, http://dbpedia.org/resource/Brine_shrimp, http://dbpedia.org/resource/British_Columbia_wolf, http://dbpedia.org/resource/Brontispa_longissima, http://dbpedia.org/resource/Bufo, http://dbpedia.org/resource/Buller's_chipmunk, http://dbpedia.org/resource/Buprestidae, http://dbpedia.org/resource/Burying_beetle, http://dbpedia.org/resource/Caecocypris, http://dbpedia.org/resource/Caedocyon, http://dbpedia.org/resource/California_chipmunk, http://dbpedia.org/resource/Callosobruchus_maculatus, http://dbpedia.org/resource/Cama_(animal), http://dbpedia.org/resource/Campostoma, http://dbpedia.org/resource/Candidia, http://dbpedia.org/resource/Cane_beetle, http://dbpedia.org/resource/Canis_apolloniensis, http://dbpedia.org/resource/Canis_cedazoensis, http://dbpedia.org/resource/Canis_edwardii, http://dbpedia.org/resource/Canis_ferox, http://dbpedia.org/resource/Canis_lepophagus, http://dbpedia.org/resource/Canis_nehringi, http://dbpedia.org/resource/Capoeta, http://dbpedia.org/resource/Carasobarbus, http://dbpedia.org/resource/Carassioides, http://dbpedia.org/resource/Carassius, http://dbpedia.org/resource/Carpocyon, http://dbpedia.org/resource/Catla, http://dbpedia.org/resource/Cephalota_circumdata, http://dbpedia.org/resource/Cerdocyon_avius, http://dbpedia.org/resource/Chagunius, http://dbpedia.org/resource/Chalcides_ocellatus, http://dbpedia.org/resource/Chanodichthys, http://dbpedia.org/resource/Chela_(fish), http://dbpedia.org/resource/Chelaethiops, http://dbpedia.org/resource/Chelicerata, http://dbpedia.org/resource/Chimaera_(genus), http://dbpedia.org/resource/Chinese_box_turtle, http://dbpedia.org/resource/Chinese_fire_belly_newt, http://dbpedia.org/resource/Chinese_high_fin_banded_shark, http://dbpedia.org/resource/Chinese_mitten_crab, http://dbpedia.org/resource/Chinese_mountain_cat, http://dbpedia.org/resource/Chinese_paddlefish, http://dbpedia.org/resource/Chinese_pond_turtle, http://dbpedia.org/resource/Chinese_softshell_turtle, http://dbpedia.org/resource/Chinese_stripe-necked_turtle, http://dbpedia.org/resource/Chinese_sturgeon, http://dbpedia.org/resource/Chironex, http://dbpedia.org/resource/Chironex_fleckeri, http://dbpedia.org/resource/Chondrophore, http://dbpedia.org/resource/Chondropyga_dorsalis, http://dbpedia.org/resource/Chondrostoma, http://dbpedia.org/resource/Chrosomus, http://dbpedia.org/resource/Chrysomeloidea, http://dbpedia.org/resource/Cicindela, http://dbpedia.org/resource/Cirrhinus, http://dbpedia.org/resource/Cleopatra_(gastropod), http://dbpedia.org/resource/Cleopatra_(horse), http://dbpedia.org/resource/Cleroidea, http://dbpedia.org/resource/Click_beetle, http://dbpedia.org/resource/Cliff_chipmunk, http://dbpedia.org/resource/Clinostomus, http://dbpedia.org/resource/Clypeobarbus, http://dbpedia.org/resource/Cockchafer, http://dbpedia.org/resource/Coelenterata, http://dbpedia.org/resource/Colorado_chipmunk, http://dbpedia.org/resource/Colorado_pikeminnow, http://dbpedia.org/resource/Colorado_potato_beetle, http://dbpedia.org/resource/Common_nase, http://dbpedia.org/resource/Common_pipistrelle, http://dbpedia.org/resource/Common_rudd, http://dbpedia.org/resource/Common_stingray, http://dbpedia.org/resource/Common_tree_frog, http://dbpedia.org/resource/Cophecheilus, http://dbpedia.org/resource/Coptoclavidae, http://dbpedia.org/resource/Coptostomabarbus, http://dbpedia.org/resource/Coreius, http://dbpedia.org/resource/Coreoleuciscus, http://dbpedia.org/resource/Cormocyon, http://dbpedia.org/resource/Cosmochilus, http://dbpedia.org/resource/Coywolf, http://dbpedia.org/resource/Cozumel_fox, http://dbpedia.org/resource/Crab-eating_fox, http://dbpedia.org/resource/Craspedacusta_sowerbii, http://dbpedia.org/resource/Cretan_owl, http://dbpedia.org/resource/Cretan_shrew, http://dbpedia.org/resource/Cretan_wildcat, http://dbpedia.org/resource/Ctenophora, http://dbpedia.org/resource/Cucujoidea, http://dbpedia.org/resource/Culpeo, http://dbpedia.org/resource/Culter_(genus), http://dbpedia.org/resource/Cultrinae, http://dbpedia.org/resource/Cupedidae, http://dbpedia.org/resource/Curculionidae, http://dbpedia.org/resource/Cyclocheilichthys, http://dbpedia.org/resource/Cyclorana, http://dbpedia.org/resource/Cynarctoides, http://dbpedia.org/resource/Cynarctoides_acridens, http://dbpedia.org/resource/Cynarctoides_emryi, http://dbpedia.org/resource/Cynarctoides_gawnae, http://dbpedia.org/resource/Cynarctoides_harlowi, http://dbpedia.org/resource/Cynarctoides_lemur, http://dbpedia.org/resource/Cynarctoides_luskensis, http://dbpedia.org/resource/Cynarctoides_roii, http://dbpedia.org/resource/Cynarctus, http://dbpedia.org/resource/Cynodesmus, http://dbpedia.org/resource/Cyprinella, http://dbpedia.org/resource/Cyprinion, http://dbpedia.org/resource/Cyprinus, http://dbpedia.org/resource/Danionella, http://dbpedia.org/resource/Darkling_beetle, http://dbpedia.org/resource/Darwin's_fox, http://dbpedia.org/resource/Dawkinsia, http://dbpedia.org/resource/Delminichthys, http://dbpedia.org/resource/Deltatheridium, http://dbpedia.org/resource/Dermestidae, http://dbpedia.org/resource/Desmocyon, http://dbpedia.org/resource/Desmopuntius, http://dbpedia.org/resource/Dhole, http://dbpedia.org/resource/Dice_snake, http://dbpedia.org/resource/Dicheirotrichus_gustavi, http://dbpedia.org/resource/Didelphis, http://dbpedia.org/resource/Dionda, http://dbpedia.org/resource/Diplocheilichthys, http://dbpedia.org/resource/Diptychus, http://dbpedia.org/resource/Discherodontus, http://dbpedia.org/resource/Discocheilus, http://dbpedia.org/resource/Discogobio, http://dbpedia.org/resource/Distoechodon, http://dbpedia.org/resource/Domestic_pigeon, http://dbpedia.org/resource/Dung_beetle, http://dbpedia.org/resource/Durango_chipmunk, http://dbpedia.org/resource/Dusicyon, http://dbpedia.org/resource/Dusky_dolphin, http://dbpedia.org/resource/Dynastinae, http://dbpedia.org/resource/Eastern_chipmunk, http://dbpedia.org/resource/Eastern_gray_squirrel, http://dbpedia.org/resource/Eastern_oyster, http://dbpedia.org/resource/Echinococcus_granulosus, http://dbpedia.org/resource/Ectopocynus, http://dbpedia.org/resource/Edible_dormouse, http://dbpedia.org/resource/Eirmotus, http://dbpedia.org/resource/Elateroidea, http://dbpedia.org/resource/Elm_leaf_beetle, http://dbpedia.org/resource/Enhydrocyon, http://dbpedia.org/resource/Enhydrocyon_basilatus, http://dbpedia.org/resource/Epalzeorhynchos, http://dbpedia.org/resource/Epicyon, http://dbpedia.org/resource/Ericymba, http://dbpedia.org/resource/Erimystax, http://dbpedia.org/resource/Esomus, http://dbpedia.org/resource/Euchiloglanis_kishinouyei, http://dbpedia.org/resource/Eucyon, http://dbpedia.org/resource/Eurasian_otter, http://dbpedia.org/resource/European_cat_snake, http://dbpedia.org/resource/European_free-tailed_bat, http://dbpedia.org/resource/European_mantis, http://dbpedia.org/resource/European_wildcat, http://dbpedia.org/resource/Euscorpius, http://dbpedia.org/resource/Eutamias, http://dbpedia.org/resource/Evarra, http://dbpedia.org/resource/Exoglossum, http://dbpedia.org/resource/Finless_porpoise, http://dbpedia.org/resource/Firefly, http://dbpedia.org/resource/Flea_beetle, http://dbpedia.org/resource/Florida_black_wolf, http://dbpedia.org/resource/Flying_gurnard, http://dbpedia.org/resource/Freshwater_crocodile, http://dbpedia.org/resource/Galago, http://dbpedia.org/resource/Galjoen, http://dbpedia.org/resource/Garra, http://dbpedia.org/resource/Geoffroy's_bat, http://dbpedia.org/resource/Geometer_moth, http://dbpedia.org/resource/Giant_barb, http://dbpedia.org/resource/Giant_burrowing_frog, http://dbpedia.org/resource/Gila_(genus), http://dbpedia.org/resource/Gnathopogon, http://dbpedia.org/resource/Gobio, http://dbpedia.org/resource/Gobiobotia, http://dbpedia.org/resource/Gobioninae, http://dbpedia.org/resource/Golden-mantled_ground_squirrel, http://dbpedia.org/resource/Gonepteryx_cleopatra, http://dbpedia.org/resource/Gonorhynchus, http://dbpedia.org/resource/Gorgonia, http://dbpedia.org/resource/Grass_carp, http://dbpedia.org/resource/Gray-collared_chipmunk, http://dbpedia.org/resource/Gray-footed_chipmunk, http://dbpedia.org/resource/Great_Lakes_boreal_wolf, http://dbpedia.org/resource/Great_Plains_wolf, http://dbpedia.org/resource/Greater_horseshoe_bat, http://dbpedia.org/resource/Greenland_wolf, http://dbpedia.org/resource/Gregory's_wolf, http://dbpedia.org/resource/Grey_wagtail, http://dbpedia.org/resource/Ground_beetle, http://dbpedia.org/resource/Ground_sloth, http://dbpedia.org/resource/Guppy, http://dbpedia.org/resource/Gymnocypris, http://dbpedia.org/resource/Gymnodiptychus, http://dbpedia.org/resource/Haliplidae, http://dbpedia.org/resource/Haludaria, http://dbpedia.org/resource/Hampala, http://dbpedia.org/resource/Haootia, http://dbpedia.org/resource/Harp_seal, http://dbpedia.org/resource/Heaviside's_dolphin, http://dbpedia.org/resource/Hemibarbus, http://dbpedia.org/resource/Hemiculter, http://dbpedia.org/resource/Hemiculterella, http://dbpedia.org/resource/Henicorhynchus, http://dbpedia.org/resource/Hesperocyoninae, http://dbpedia.org/resource/Himalayan_wolf, http://dbpedia.org/resource/Histeridae, http://dbpedia.org/resource/Hoary_fox, http://dbpedia.org/resource/Hokkaido_wolf, http://dbpedia.org/resource/Homoscleromorpha, http://dbpedia.org/resource/Hongshuia, http://dbpedia.org/resource/Hopi_chipmunk, http://dbpedia.org/resource/Horalabiosa, http://dbpedia.org/resource/Horned_guan, http://dbpedia.org/resource/House_mouse, http://dbpedia.org/resource/Hudson_Bay_wolf, http://dbpedia.org/resource/Hybognathus, http://dbpedia.org/resource/Hybopsis, http://dbpedia.org/resource/Hybrid_camel, http://dbpedia.org/resource/Hydrophilidae, http://dbpedia.org/resource/Hydroscaphidae, http://dbpedia.org/resource/Hydrozoa, http://dbpedia.org/resource/Hypophthalmichthys, http://dbpedia.org/resource/Hypselobarbus, http://dbpedia.org/resource/Hypsibarbus, http://dbpedia.org/resource/Iberochondrostoma, http://dbpedia.org/resource/Ide_(fish), http://dbpedia.org/resource/Iguana, http://dbpedia.org/resource/Indian_wolf, http://dbpedia.org/resource/Ischikauia, http://dbpedia.org/resource/Island_fox, http://dbpedia.org/resource/Italian_wolf, http://dbpedia.org/resource/Japanese_eel, http://dbpedia.org/resource/Jugulator_(mammal), http://dbpedia.org/resource/Koi, http://dbpedia.org/resource/Kosswigobarbus, http://dbpedia.org/resource/Kuhl's_lorikeet, http://dbpedia.org/resource/Kuhl's_pipistrelle, http://dbpedia.org/resource/Labeo, http://dbpedia.org/resource/Labeobarbus, http://dbpedia.org/resource/Labeoninae, http://dbpedia.org/resource/Labiobarbus, http://dbpedia.org/resource/Labrador_wolf, http://dbpedia.org/resource/Lamprima_aurata, http://dbpedia.org/resource/Lanner_falcon, http://dbpedia.org/resource/Laubuka, http://dbpedia.org/resource/Lavinia_(fish), http://dbpedia.org/resource/Leaf_beetle, http://dbpedia.org/resource/Least_chipmunk, http://dbpedia.org/resource/Leiocassis, http://dbpedia.org/resource/Leptobarbus, http://dbpedia.org/resource/Leptocyon, http://dbpedia.org/resource/Leptocypris, http://dbpedia.org/resource/Leptogorgia, http://dbpedia.org/resource/Lesser_horseshoe_bat, http://dbpedia.org/resource/Lesser_mouse-eared_bat, http://dbpedia.org/resource/Leuciscinae, http://dbpedia.org/resource/Leuciscus, http://dbpedia.org/resource/Leucos, http://dbpedia.org/resource/Limnodynastes, http://dbpedia.org/resource/Lobocheilos, http://dbpedia.org/resource/Lodgepole_chipmunk, http://dbpedia.org/resource/Long-eared_chipmunk, http://dbpedia.org/resource/Long-eared_hedgehog, http://dbpedia.org/resource/Luciobarbus, http://dbpedia.org/resource/Luciocyprinus, http://dbpedia.org/resource/Luciosoma, http://dbpedia.org/resource/Luxilus, http://dbpedia.org/resource/Lythrurus, http://dbpedia.org/resource/Macaw, http://dbpedia.org/resource/Macrhybopsis, http://dbpedia.org/resource/Mahseer, http://dbpedia.org/resource/Maned_wolf, http://dbpedia.org/resource/Manitoba_wolf, http://dbpedia.org/resource/Margariscus, http://dbpedia.org/resource/Marsh_frog, http://dbpedia.org/resource/Masticbarbus, http://dbpedia.org/resource/Mearns_coyote, http://dbpedia.org/resource/Meat_ant, http://dbpedia.org/resource/Medusozoa, http://dbpedia.org/resource/Megalobrama, http://dbpedia.org/resource/Megalobrama_elongata, http://dbpedia.org/resource/Melolonthinae, http://dbpedia.org/resource/Merriam's_chipmunk, http://dbpedia.org/resource/Mesobola, http://dbpedia.org/resource/Mesocyon, http://dbpedia.org/resource/Mesogobio, http://dbpedia.org/resource/Metatomarctus, http://dbpedia.org/resource/Metorchis_conjunctus, http://dbpedia.org/resource/Metzia, http://dbpedia.org/resource/Mexican_wolf, http://dbpedia.org/resource/Microdevario, http://dbpedia.org/resource/Microphysogobio, http://dbpedia.org/resource/Microrasbora, http://dbpedia.org/resource/Microtomarctus, http://dbpedia.org/resource/Milkweed_butterfly, http://dbpedia.org/resource/Mojarra, http://dbpedia.org/resource/Mordellidae, http://dbpedia.org/resource/Mountain_bluebird, http://dbpedia.org/resource/Mountain_pine_beetle, http://dbpedia.org/resource/Murex, http://dbpedia.org/resource/Musca_vetustissima, http://dbpedia.org/resource/Mystacoleucus, http://dbpedia.org/resource/Myxophaga, http://dbpedia.org/resource/Nanophyetus_salmincola, http://dbpedia.org/resource/Naziritor, http://dbpedia.org/resource/Nematabramis, http://dbpedia.org/resource/Neobola, http://dbpedia.org/resource/Neolissochilus, http://dbpedia.org/resource/Neotamias, http://dbpedia.org/resource/Newfoundland_wolf, http://dbpedia.org/resource/Nicholsicypris, http://dbpedia.org/resource/Nipponocypris, http://dbpedia.org/resource/Nocomis, http://dbpedia.org/resource/Northeastern_coyote, http://dbpedia.org/resource/Northern_quoll, http://dbpedia.org/resource/Northern_snakehead, http://dbpedia.org/resource/Northwestern_wolf, http://dbpedia.org/resource/Notropis, http://dbpedia.org/resource/Nurocyon, http://dbpedia.org/resource/Nyctereutes_donnezani, http://dbpedia.org/resource/Onychostoma, http://dbpedia.org/resource/Opsaridium, http://dbpedia.org/resource/Opsariichthys, http://dbpedia.org/resource/Opsarius, http://dbpedia.org/resource/Oregonichthys, http://dbpedia.org/resource/Oreichthys, http://dbpedia.org/resource/Oreoleuciscus, http://dbpedia.org/resource/Osbornodon, http://dbpedia.org/resource/Osbornodon_iamonensis, http://dbpedia.org/resource/Osteobrama, http://dbpedia.org/resource/Osteochilichthys, http://dbpedia.org/resource/Otarocyon, http://dbpedia.org/resource/Oxetocyon, http://dbpedia.org/resource/Oxyaenidae, http://dbpedia.org/resource/Oxygaster, http://dbpedia.org/resource/Pachychilon, http://dbpedia.org/resource/Pademelon, http://dbpedia.org/resource/Paedocypris_progenetica, http://dbpedia.org/resource/Painted_comber, http://dbpedia.org/resource/Pale_fox, http://dbpedia.org/resource/Palmer's_chipmunk, http://dbpedia.org/resource/Pampas_fox, http://dbpedia.org/resource/Pan's_box_turtle, http://dbpedia.org/resource/Panamint_chipmunk, http://dbpedia.org/resource/Parachela, http://dbpedia.org/resource/Parachondrostoma, http://dbpedia.org/resource/Paracrossochilus, http://dbpedia.org/resource/Paracynarctus, http://dbpedia.org/resource/Paraenhydrocyon, http://dbpedia.org/resource/Paraenhydrocyon_josephi, http://dbpedia.org/resource/Paralaubuca, http://dbpedia.org/resource/Parapsilorhynchus, http://dbpedia.org/resource/Pararasbora, http://dbpedia.org/resource/Parasinilabeo, http://dbpedia.org/resource/Paratomarctus, http://dbpedia.org/resource/Pearly_razorfish, http://dbpedia.org/resource/Pectenocypris, http://dbpedia.org/resource/Pelasgus_(fish), http://dbpedia.org/resource/Percocypris, http://dbpedia.org/resource/Pethia, http://dbpedia.org/resource/Petroleuciscus, http://dbpedia.org/resource/Phenacobius, http://dbpedia.org/resource/Phengodidae, http://dbpedia.org/resource/Philotrox, http://dbpedia.org/resource/Phlaocyon, http://dbpedia.org/resource/Phlaocyon_achoros, http://dbpedia.org/resource/Phlaocyon_annectens, http://dbpedia.org/resource/Phlaocyon_latidens, http://dbpedia.org/resource/Phlaocyon_mariae, http://dbpedia.org/resource/Phlaocyonini, http://dbpedia.org/resource/Phoxinellus, http://dbpedia.org/resource/Phoxinus, http://dbpedia.org/resource/Phyllomedusa, http://dbpedia.org/resource/Phyllomedusa_distincta, http://dbpedia.org/resource/Phyllomedusa_oreades, http://dbpedia.org/resource/Pill_millipede, http://dbpedia.org/resource/Pimephales, http://dbpedia.org/resource/Placocheilus, http://dbpedia.org/resource/Placogobio, http://dbpedia.org/resource/Placozoa, http://dbpedia.org/resource/Plains_coyote, http://dbpedia.org/resource/Platyhelminthes, http://dbpedia.org/resource/Platysmacheilus, http://dbpedia.org/resource/Pleistocene_coyote, http://dbpedia.org/resource/Poeciliidae, http://dbpedia.org/resource/Pogonichthys, http://dbpedia.org/resource/Polynesian_rat, http://dbpedia.org/resource/Polyphaga, http://dbpedia.org/resource/Poropuntius, http://dbpedia.org/resource/Potamidae, http://dbpedia.org/resource/Potamon_potamios, http://dbpedia.org/resource/Predatory_carp, http://dbpedia.org/resource/Probarbus, http://dbpedia.org/resource/Procypris, http://dbpedia.org/resource/Prohesperocyon, http://dbpedia.org/resource/Prolabeops, http://dbpedia.org/resource/Protepicyon, http://dbpedia.org/resource/Protocyon, http://dbpedia.org/resource/Protomarctus, http://dbpedia.org/resource/Prussian_carp, http://dbpedia.org/resource/Przewalski's_horse, http://dbpedia.org/resource/Psalidocyon, http://dbpedia.org/resource/Pseudechis_australis, http://dbpedia.org/resource/Pseudobarbus, http://dbpedia.org/resource/Pseudochondrostoma, http://dbpedia.org/resource/Pseudocrossocheilus, http://dbpedia.org/resource/Pseudogobio, http://dbpedia.org/resource/Pseudohemiculter, http://dbpedia.org/resource/Pseudolaubuca, http://dbpedia.org/resource/Pseudophoxinus, http://dbpedia.org/resource/Pseudopungtungia, http://dbpedia.org/resource/Pseudorasbora, http://dbpedia.org/resource/Pteronotropis, http://dbpedia.org/resource/Ptychidio, http://dbpedia.org/resource/Ptychobarbus, http://dbpedia.org/resource/Ptychocheilus, http://dbpedia.org/resource/Pulex, http://dbpedia.org/resource/Pungtungia, http://dbpedia.org/resource/Puntigrus, http://dbpedia.org/resource/Puntioplites, http://dbpedia.org/resource/Puntius, http://dbpedia.org/resource/Radiata, http://dbpedia.org/resource/Raiamas, http://dbpedia.org/resource/Rakali, http://dbpedia.org/resource/Rasboroides, http://dbpedia.org/resource/Rectoris, http://dbpedia.org/resource/Red-tailed_chipmunk, http://dbpedia.org/resource/Red_flour_beetle, http://dbpedia.org/resource/Reduviidae, http://dbpedia.org/resource/Rhinella, http://dbpedia.org/resource/Rhinichthys, http://dbpedia.org/resource/Rhinogobio, http://dbpedia.org/resource/Rhinotia_hemistictus, http://dbpedia.org/resource/Rhizocyon, http://dbpedia.org/resource/Rhodeus, http://dbpedia.org/resource/Rhynchocypris, http://dbpedia.org/resource/Richardsonius, http://dbpedia.org/resource/Romanogobio, http://dbpedia.org/resource/Rove_beetle, http://dbpedia.org/resource/Rutelinae, http://dbpedia.org/resource/Sahyadria, http://dbpedia.org/resource/Salmophasia, http://dbpedia.org/resource/Sand_cat, http://dbpedia.org/resource/Sarcocheilichthys, http://dbpedia.org/resource/Sardinian_dhole, http://dbpedia.org/resource/Saurogobio, http://dbpedia.org/resource/Savi's_pipistrelle, http://dbpedia.org/resource/Scale_insect, http://dbpedia.org/resource/Scaphiodonichthys, http://dbpedia.org/resource/Scaphognathops, http://dbpedia.org/resource/Scarabaeidae, http://dbpedia.org/resource/Scarabaeoidea, http://dbpedia.org/resource/Scarabaeus_sacer, http://dbpedia.org/resource/Scardinius, http://dbpedia.org/resource/Schismatorhynchos, http://dbpedia.org/resource/Schizocypris, http://dbpedia.org/resource/Schizopyge, http://dbpedia.org/resource/Schizopygopsis, http://dbpedia.org/resource/Schizothorax, http://dbpedia.org/resource/Schizothorax_longibarbus, http://dbpedia.org/resource/Schizothorax_parvus, http://dbpedia.org/resource/Sea_pen, http://dbpedia.org/resource/Semilabeo, http://dbpedia.org/resource/Semiplotus, http://dbpedia.org/resource/Semotilus, http://dbpedia.org/resource/Serotine_bat, http://dbpedia.org/resource/Short-eared_dog, http://dbpedia.org/resource/Shrew, http://dbpedia.org/resource/Side-striped_jackal, http://dbpedia.org/resource/Sikukia, http://dbpedia.org/resource/Silver_carp, http://dbpedia.org/resource/Sinibrama, http://dbpedia.org/resource/Sinilabeo, http://dbpedia.org/resource/Sinocrossocheilus, http://dbpedia.org/resource/Sinocyclocheilus, http://dbpedia.org/resource/Siskiyou_chipmunk, http://dbpedia.org/resource/Small_hive_beetle, http://dbpedia.org/resource/Sonoma_chipmunk, http://dbpedia.org/resource/South_American_fox, http://dbpedia.org/resource/Southern_right_whale, http://dbpedia.org/resource/Southern_toad, http://dbpedia.org/resource/Sparassodonta, http://dbpedia.org/resource/Speothos, http://dbpedia.org/resource/Speothos_pacivorus, http://dbpedia.org/resource/Spermophilus, http://dbpedia.org/resource/Sphaerius, http://dbpedia.org/resource/Spinibarbus, http://dbpedia.org/resource/Springtail, http://dbpedia.org/resource/Spur-thighed_tortoise, http://dbpedia.org/resource/Squalidus, http://dbpedia.org/resource/Squalius, http://dbpedia.org/resource/Stag_beetle, http://dbpedia.org/resource/Stauromedusae, http://dbpedia.org/resource/Stephens_Island_wren, http://dbpedia.org/resource/Steppe_wolf, http://dbpedia.org/resource/Streptophyta, http://dbpedia.org/resource/Subspecies_of_Canis_lupus, http://dbpedia.org/resource/Sundadanio, http://dbpedia.org/resource/Sunkahetanka, http://dbpedia.org/resource/Swallowtail_butterfly, http://dbpedia.org/resource/Systomus, http://dbpedia.org/resource/Taenia_pisiformis, http://dbpedia.org/resource/Takifugu, http://dbpedia.org/resource/Tampichthys, http://dbpedia.org/resource/Tanakia, http://dbpedia.org/resource/Tawny_frogmouth, http://dbpedia.org/resource/Tawny_owl, http://dbpedia.org/resource/Telephone-pole_beetle, http://dbpedia.org/resource/Telestes, http://dbpedia.org/resource/Tench, http://dbpedia.org/resource/Tephrocyon, http://dbpedia.org/resource/Texas_wolf, http://dbpedia.org/resource/Theriodictis, http://dbpedia.org/resource/Thryssocypris, http://dbpedia.org/resource/Thylacoleonidae, http://dbpedia.org/resource/Thynnichthys, http://dbpedia.org/resource/Tibetan_sand_fox, http://dbpedia.org/resource/Tibetan_wolf, http://dbpedia.org/resource/Tomarctus, http://dbpedia.org/resource/Tor_(genus), http://dbpedia.org/resource/Townsend's_chipmunk, http://dbpedia.org/resource/Toxabramis, http://dbpedia.org/resource/Toxascaris_leonina, http://dbpedia.org/resource/Triatoma, http://dbpedia.org/resource/Triatominae, http://dbpedia.org/resource/Tribolodon, http://dbpedia.org/resource/Trigonopoma, http://dbpedia.org/resource/Trigonostigma, http://dbpedia.org/resource/Tropidophoxinellus, http://dbpedia.org/resource/True_toad, http://dbpedia.org/resource/Tubularia, http://dbpedia.org/resource/Tundra_wolf, http://dbpedia.org/resource/Uinta_chipmunk, http://dbpedia.org/resource/Uperoleia, http://dbpedia.org/resource/Urocyon, http://dbpedia.org/resource/Vancouver_Island_wolf, http://dbpedia.org/resource/Varicorhinus, http://dbpedia.org/resource/Vimba, http://dbpedia.org/resource/Vinagarra, http://dbpedia.org/resource/Vulpes_riffautae, http://dbpedia.org/resource/Weevil, http://dbpedia.org/resource/Whirligig_beetle, http://dbpedia.org/resource/Whiskered_bat, http://dbpedia.org/resource/Whistling_kite, http://dbpedia.org/resource/Wrasse, http://dbpedia.org/resource/Xenocyon, http://dbpedia.org/resource/Xenocyprioides, http://dbpedia.org/resource/Xenocypris, http://dbpedia.org/resource/Xenophysogobio, http://dbpedia.org/resource/Xylotrupes, http://dbpedia.org/resource/Yangtze_giant_softshell_turtle, http://dbpedia.org/resource/Yaoshanicus, http://dbpedia.org/resource/Yellow-cheeked_chipmunk, http://dbpedia.org/resource/Yellow-headed_box_turtle, http://dbpedia.org/resource/Yellow-pine_chipmunk, http://dbpedia.org/resource/Yellow_pond_turtle, http://dbpedia.org/resource/Yunnan_box_turtle, http://dbpedia.org/resource/Yuriria_(fish), http://dbpedia.org/resource/Zacco_(fish), http://dbpedia.org/resource/Zopherus, http://dbpedia.org/resource/Acanthizidae, http://dbpedia.org/resource/Acrocephalidae, http://dbpedia.org/resource/Acyrthosiphon_pisum, http://dbpedia.org/resource/Aguiguan_reed_warbler, http://dbpedia.org/resource/Alaotra_grebe, http://dbpedia.org/resource/Aldabra_brush_warbler, http://dbpedia.org/resource/Amoebozoa, http://dbpedia.org/resource/Antillean_cave_rail, http://dbpedia.org/resource/Antioquia_brush_finch, http://dbpedia.org/resource/Arabian_bustard, http://dbpedia.org/resource/Arabian_ostrich, http://dbpedia.org/resource/Archamoebae, http://dbpedia.org/resource/Arkarua, http://dbpedia.org/resource/Ascension_crake, http://dbpedia.org/resource/Ascension_night_heron, http://dbpedia.org/resource/Atitlán_grebe, http://dbpedia.org/resource/Auckland_merganser, http://dbpedia.org/resource/Australian_masked_owl, http://dbpedia.org/resource/Bachman's_warbler, http://dbpedia.org/resource/Bahama_yellowthroat, http://dbpedia.org/resource/Baillon's_crake, http://dbpedia.org/resource/Bar-bellied_cuckooshrike, http://dbpedia.org/resource/Bar-winged_rail, http://dbpedia.org/resource/Bare-eyed_rail, http://dbpedia.org/resource/Barytherium, http://dbpedia.org/resource/Bearded_reedling, http://dbpedia.org/resource/Bearded_tachuri, http://dbpedia.org/resource/Bermuda_night_heron, http://dbpedia.org/resource/Bewick's_wren, http://dbpedia.org/resource/Bibionidae, http://dbpedia.org/resource/Bitis, http://dbpedia.org/resource/Black-browed_babbler, http://dbpedia.org/resource/Black-capped_petrel, http://dbpedia.org/resource/Black-eared_ground_thrush, http://dbpedia.org/resource/Black-faced_pitta, http://dbpedia.org/resource/Black-fronted_parakeet, http://dbpedia.org/resource/Black-lored_waxbill, http://dbpedia.org/resource/Black_francolin, http://dbpedia.org/resource/Black_mamo, http://dbpedia.org/resource/Blackish_cuckooshrike, http://dbpedia.org/resource/Blue-banded_kingfisher, http://dbpedia.org/resource/Blue-wattled_bulbul, http://dbpedia.org/resource/Bonin_grosbeak, http://dbpedia.org/resource/Bonin_nankeen_night_heron, http://dbpedia.org/resource/Bonin_thrush, http://dbpedia.org/resource/Bonin_white-eye, http://dbpedia.org/resource/Brace's_emerald, http://dbpedia.org/resource/Bridled_white-eye, http://dbpedia.org/resource/Brown-banded_antpitta, http://dbpedia.org/resource/Brown-throated_parakeet, http://dbpedia.org/resource/Brown_jacamar, http://dbpedia.org/resource/Brown_thornbill, http://dbpedia.org/resource/Brown_tree_snake, http://dbpedia.org/resource/Buff-banded_rail, http://dbpedia.org/resource/Buff-rumped_woodpecker, http://dbpedia.org/resource/Burrowing_owl, http://dbpedia.org/resource/Bushwren]... [INFO] P: 0.006576155202513203, R: 1.0, F: 0.013066383836978822 -[INFO] Cumulative metrics, sample: 116, P: 0.360, R: 0.435, macro F1: 0.363, macro F1*: 0.394 -[INFO] QALD Cumulative metrics, sample: 116, P: 0.575, R: 0.435, macro F1: 0.363, QALD macro F1: 0.495 -[INFO] Current time: 4799 ms, average: 3543.37 ms/ques, entity detection: 816.75 ms/ques (linking tool: 728.22 ms/ques), relation detection: 1134.17 ms/ques (similarity: 639.47 ms/ques), EDG generation: 51.84 ms/ques +[INFO] Cumulative metrics, sample: 116, P: 0.360, R: 0.435, macro F1: 0.394 +[INFO] QALD Cumulative metrics, sample: 116, P: 0.575, R: 0.435, average F1: 0.363, QALD macro F1: 0.495 +[INFO] Current question consumed time: 4234 ms, average: 3228.62 ms/ques, entity detection: 725.84 ms/ques (linking tool: 649.96 ms/ques), relation detection: 1060.26 ms/ques (similarity: 639.08 ms/ques), EDG generation: 52.98 ms/ques [INFO] Entity Linking P: 0.27586206896551724, R: 0.27586206896551724, micro F1: 0.27586206896551724, macro F1: 0.27586206896551724 [INFO] Relation Linking P: 0.29310344827586204, R: 0.29310344827586204, micro F1: 0.29310344827586204, macro F1: 0.29310344827586204 [INFO] Type Linking P: 0.8793103448275862, R: 0.8793103448275862, micro F1: 0.8793103448275862, macro F1: 0.8793103448275863 @@ -1747,9 +1747,9 @@ Question 116: Which politicians were married to a German? [INFO] Golden Answer: [http://dbpedia.org/resource/Karl_Carstens, http://dbpedia.org/resource/Florence_Prag_Kahn, http://dbpedia.org/resource/Richard_von_Weizsäcker, http://dbpedia.org/resource/Isidor_Straus, http://dbpedia.org/resource/Erich_Honecker, http://dbpedia.org/resource/Walter_Ulbricht, http://dbpedia.org/resource/Otto_of_Stolberg-Wernigerode, http://dbpedia.org/resource/Johannes_Rau, http://dbpedia.org/resource/Sahra_Wagenknecht, http://dbpedia.org/resource/Duke_Adolf_Friedrich_of_Mecklenburg, http://dbpedia.org/resource/Gesine_Schwan] [INFO] Predicted Answer: [http://dbpedia.org/resource/Abd_al-Karim_Qasim, http://dbpedia.org/resource/Abdul_Qadir_(Afghan_communist), http://dbpedia.org/resource/Abdul_Sabur_Farid_Kohistani, http://dbpedia.org/resource/Ada_Colau, http://dbpedia.org/resource/Adam_Price, http://dbpedia.org/resource/Adrian_Flook, http://dbpedia.org/resource/Ahmad_Shah_Ahmadzai, http://dbpedia.org/resource/Alain_Rousset, http://dbpedia.org/resource/Albert_Hawke, http://dbpedia.org/resource/Alexander_Beresford_Hope, http://dbpedia.org/resource/Andrew_Brons, http://dbpedia.org/resource/Andrew_Peacock, http://dbpedia.org/resource/Andrew_Rosindell, http://dbpedia.org/resource/Andrew_Turner_(politician), http://dbpedia.org/resource/Angela_Watkinson, http://dbpedia.org/resource/Annette_Brooke, http://dbpedia.org/resource/Antonio_Fazio, http://dbpedia.org/resource/Antony_Lambton, http://dbpedia.org/resource/Archibald_Primrose,_5th_Earl_of_Rosebery, http://dbpedia.org/resource/Arthur_Eve, http://dbpedia.org/resource/Augustus_O._Bourn, http://dbpedia.org/resource/Barrie_Unsworth, http://dbpedia.org/resource/Barry_Jones_(Australian_politician), http://dbpedia.org/resource/Benjamin_Raspail, http://dbpedia.org/resource/Bonar_Law, http://dbpedia.org/resource/Brian_Howe_(politician), http://dbpedia.org/resource/Byron_Brown, http://dbpedia.org/resource/Carlo_Azeglio_Ciampi, http://dbpedia.org/resource/Carlos_Menem, http://dbpedia.org/resource/Chantal_Bourragué, http://dbpedia.org/resource/Charles_Eden_(politician), http://dbpedia.org/resource/Charles_Grey,_2nd_Earl_Grey, http://dbpedia.org/resource/Charles_Stewart_Parnell, http://dbpedia.org/resource/Chester_A._Kowal, http://dbpedia.org/resource/Chris_Dodd, http://dbpedia.org/resource/Christine_Butler, http://dbpedia.org/resource/Clem_Hawke, http://dbpedia.org/resource/Clement_Davies, http://dbpedia.org/resource/Cliff_Stearns, http://dbpedia.org/resource/Colin_Jordan, http://dbpedia.org/resource/Daisaku_Ikeda, http://dbpedia.org/resource/Daniel_Spagnou, http://dbpedia.org/resource/David_Burnside, http://dbpedia.org/resource/David_Lock, http://dbpedia.org/resource/David_Steel, http://dbpedia.org/resource/Derek_Beackon, http://dbpedia.org/resource/Derek_Conway, http://dbpedia.org/resource/Derek_Simpson_(trade_unionist), http://dbpedia.org/resource/Edward_Smith-Stanley,_14th_Earl_of_Derby, http://dbpedia.org/resource/Edwina_Currie, http://dbpedia.org/resource/Eileen_Gordon, http://dbpedia.org/resource/Elbert_A._Woodward, http://dbpedia.org/resource/Eric_Joyce, http://dbpedia.org/resource/Ernest_Marples, http://dbpedia.org/resource/Eugene_McCarthy, http://dbpedia.org/resource/Fernando_Wood, http://dbpedia.org/resource/Fiona_Jones, http://dbpedia.org/resource/Frédéric_Mitterrand, http://dbpedia.org/resource/Gaius_Antonius_Hybrida, http://dbpedia.org/resource/Gaius_Marius, http://dbpedia.org/resource/Geoffrey_Robinson, http://dbpedia.org/resource/George_Briggs_(New_York_politician), http://dbpedia.org/resource/George_Dodd_(MP), http://dbpedia.org/resource/George_Turner_(British_politician), http://dbpedia.org/resource/Gordon_Bryant, http://dbpedia.org/resource/Govinda_(actor), http://dbpedia.org/resource/Graham_Richardson, http://dbpedia.org/resource/Gregory_Campbell_(politician), http://dbpedia.org/resource/Gwyneth_Dunwoody, http://dbpedia.org/resource/Haji_Mohammad_Chamkani, http://dbpedia.org/resource/Hajibala_Abutalybov, http://dbpedia.org/resource/Happy_Chandler, http://dbpedia.org/resource/Hazel_Hawke, http://dbpedia.org/resource/Henry_Bellingham_(Norfolk_MP), http://dbpedia.org/resource/Howling_Laud_Hope, http://dbpedia.org/resource/Hugh_Dalton, http://dbpedia.org/resource/Hugh_Robertson_(politician), http://dbpedia.org/resource/Hugh_de_Cressingham, http://dbpedia.org/resource/I_Made_Mangku_Pastika, http://dbpedia.org/resource/Ian_Bruce_(politician), http://dbpedia.org/resource/Ian_Malcolm_(politician), http://dbpedia.org/resource/Ieuan_Wyn_Jones, http://dbpedia.org/resource/Jack_Dromey, http://dbpedia.org/resource/Jackie_Ballard, http://dbpedia.org/resource/Jacques_Chaban-Delmas, http://dbpedia.org/resource/James_D._Griffin, http://dbpedia.org/resource/Jay_Lee, http://dbpedia.org/resource/Jerry_Hayes, http://dbpedia.org/resource/Jo_Grimond, http://dbpedia.org/resource/John_Buchan, http://dbpedia.org/resource/John_Button, http://dbpedia.org/resource/John_Clarke_(Baptist_minister), http://dbpedia.org/resource/John_Dawkins, http://dbpedia.org/resource/John_Fox_(congressman), http://dbpedia.org/resource/John_G._Rowland, http://dbpedia.org/resource/John_Hume, http://dbpedia.org/resource/John_Kerin, http://dbpedia.org/resource/John_Morley, http://dbpedia.org/resource/John_Morrissey, http://dbpedia.org/resource/John_Morse_(British_politician), http://dbpedia.org/resource/John_Pardoe, http://dbpedia.org/resource/John_Simon,_1st_Viscount_Simon, http://dbpedia.org/resource/John_T._Hoffman, http://dbpedia.org/resource/John_Tyndall_(politician), http://dbpedia.org/resource/Jonathan_Russell, http://dbpedia.org/resource/Josias_Lyndon, http://dbpedia.org/resource/Juan_Perón, http://dbpedia.org/resource/Keith_Darvill, http://dbpedia.org/resource/Klement_Gottwald, http://dbpedia.org/resource/Liow_Tiong_Lai, http://dbpedia.org/resource/Lord_Edward_Somerset, http://dbpedia.org/resource/Lucius_Quinctius_Cincinnatus, http://dbpedia.org/resource/Luigi_Einaudi, http://dbpedia.org/resource/Marcus_Terentius_Varro_Lucullus, http://dbpedia.org/resource/Margaret_Hodge, http://dbpedia.org/resource/Margaret_Whitlam, http://dbpedia.org/resource/Margaret_Wright_(British_politician), http://dbpedia.org/resource/Mark_Oaten, http://dbpedia.org/resource/Michael_Mates, http://dbpedia.org/resource/Michael_Meadowcroft, http://dbpedia.org/resource/Michèle_Delaunay, http://dbpedia.org/resource/Mike_Woodin, http://dbpedia.org/resource/Mohammad_Hashim_Maiwandwal, http://dbpedia.org/resource/Mohammad_Jihad_al-Laham, http://dbpedia.org/resource/Morgan_Morgans, http://dbpedia.org/resource/Nasirdin_Isanov, http://dbpedia.org/resource/Nathaniel_Lambert, http://dbpedia.org/resource/Neil_Hamilton_(politician), http://dbpedia.org/resource/Nick_St_Aubyn, http://dbpedia.org/resource/Nigel_Dodds, http://dbpedia.org/resource/Norman_Lamb, http://dbpedia.org/resource/Nur_Ahmed_Nur, http://dbpedia.org/resource/Oswald_Mosley, http://dbpedia.org/resource/Patrick_Mercer, http://dbpedia.org/resource/Patrick_Nicholls, http://dbpedia.org/resource/Patsy_Calton, http://dbpedia.org/resource/Paul_Reynaud, http://dbpedia.org/resource/Peter_Brand, http://dbpedia.org/resource/Phil_Cleary, http://dbpedia.org/resource/Phillip_Lynch, http://dbpedia.org/resource/Pritam_Singh, http://dbpedia.org/resource/Reginald_Maudling, http://dbpedia.org/resource/Ricardo_Lagos, http://dbpedia.org/resource/Richard_Edmonds, http://dbpedia.org/resource/Richard_Taylor_(British_politician), http://dbpedia.org/resource/Richard_Younger-Ross, http://dbpedia.org/resource/Robert_McCartney_(Northern_Irish_politician), http://dbpedia.org/resource/Robert_Peel, http://dbpedia.org/resource/Robert_Walpole, http://dbpedia.org/resource/Rodric_Braithwaite, http://dbpedia.org/resource/Ron_Dellums, http://dbpedia.org/resource/Samuel_Ward_(American_statesman), http://dbpedia.org/resource/Seth_Low, http://dbpedia.org/resource/Seán_Garland, http://dbpedia.org/resource/Sibghatullah_Mojaddedi, http://dbpedia.org/resource/Simon_Darby, http://dbpedia.org/resource/Stephen_Day_(British_politician), http://dbpedia.org/resource/Stephen_Hopkins_(politician), http://dbpedia.org/resource/Sue_Doughty, http://dbpedia.org/resource/Sultan_Ali_Keshtmand, http://dbpedia.org/resource/Tam_Galbraith, http://dbpedia.org/resource/Tom_Holmes, http://dbpedia.org/resource/William_Donald_Schaefer, http://dbpedia.org/resource/William_Hamilton_(diplomat), http://dbpedia.org/resource/William_Lamb,_2nd_Viscount_Melbourne, http://dbpedia.org/resource/William_Ponsonby_(British_Army_officer), http://dbpedia.org/resource/William_Ross_(Unionist_politician), http://dbpedia.org/resource/William_Thompson_(Ulster_Unionist_politician), http://dbpedia.org/resource/Adair_Blain, http://dbpedia.org/resource/Alexandros_Koumoundouros, http://dbpedia.org/resource/Alexandros_Mavrokordatos, http://dbpedia.org/resource/Amintore_Fanfani, http://dbpedia.org/resource/Anastas_Mikoyan, http://dbpedia.org/resource/Andreas_Metaxas, http://dbpedia.org/resource/Andrew_Gordon_Magrath, http://dbpedia.org/resource/Andrew_Pelling, http://dbpedia.org/resource/Andrew_Pickens_(congressman), http://dbpedia.org/resource/Austin_Mitchell, http://dbpedia.org/resource/Benjamin_Hawkins, http://dbpedia.org/resource/Benning_Wentworth, http://dbpedia.org/resource/Bill_Waller, http://dbpedia.org/resource/Cemal_Gürsel, http://dbpedia.org/resource/Charles_H._Mills, http://dbpedia.org/resource/Charles_Wentworth_Upham, http://dbpedia.org/resource/Chen_Cheng, http://dbpedia.org/resource/Chen_Guofu, http://dbpedia.org/resource/Chen_Shui-bian, http://dbpedia.org/resource/Chester_Bowles, http://dbpedia.org/resource/Chris_Philp, http://dbpedia.org/resource/Chun_Doo-hwan, http://dbpedia.org/resource/Cyril_Smith, http://dbpedia.org/resource/Darwin_Hindman, http://dbpedia.org/resource/David_Congdon, http://dbpedia.org/resource/English_Council_of_State, http://dbpedia.org/resource/Estanislao_Figueras, http://dbpedia.org/resource/Eugene_Sawyer, http://dbpedia.org/resource/Francesc_Macià, http://dbpedia.org/resource/Gavin_Barwell, http://dbpedia.org/resource/Georgios_Kountouriotis, http://dbpedia.org/resource/Geraint_Davies_(Labour_politician), http://dbpedia.org/resource/Gil_Gutknecht, http://dbpedia.org/resource/Gustavo_Díaz_Ordaz, http://dbpedia.org/resource/Guy_Vander_Jagt, http://dbpedia.org/resource/Harold_George_Nelson, http://dbpedia.org/resource/Henry_S._Foote, http://dbpedia.org/resource/James_S._Rollins, http://dbpedia.org/resource/Jesse_J._Finley, http://dbpedia.org/resource/Jim_Bolger, http://dbpedia.org/resource/Jim_Marurai, http://dbpedia.org/resource/Jock_Nelson, http://dbpedia.org/resource/Jodeen_Carney, http://dbpedia.org/resource/Joe_Clark, http://dbpedia.org/resource/John_A._Logan, http://dbpedia.org/resource/John_Elwes_(politician), http://dbpedia.org/resource/John_Lloyd_Waddy, http://dbpedia.org/resource/John_Slidell, http://dbpedia.org/resource/John_Tipton, http://dbpedia.org/resource/Jordi_Pujol, http://dbpedia.org/resource/José_Montilla, http://dbpedia.org/resource/Katrien_Meire, http://dbpedia.org/resource/Kirk_Fordice, http://dbpedia.org/resource/Kurt_Summers,_Jr., http://dbpedia.org/resource/Kåre_Willoch, http://dbpedia.org/resource/Lawrence_Sullivan_Ross, http://dbpedia.org/resource/Lee_Teng-hui, http://dbpedia.org/resource/Leslie_Haden-Guest,_1st_Baron_Haden-Guest, http://dbpedia.org/resource/Li_Lisan, http://dbpedia.org/resource/Li_Zongren, http://dbpedia.org/resource/Lin_Sen, http://dbpedia.org/resource/Lluís_Companys, http://dbpedia.org/resource/Luke_Gosling, http://dbpedia.org/resource/Lázaro_Cárdenas, http://dbpedia.org/resource/Manuel_Velasco_Coello, http://dbpedia.org/resource/Mark_Field, http://dbpedia.org/resource/Milton_Obote, http://dbpedia.org/resource/Márton_Berzeviczy, http://dbpedia.org/resource/Natasha_Griggs, http://dbpedia.org/resource/Neil_McLean_(politician), http://dbpedia.org/resource/Pasqual_Maragall, http://dbpedia.org/resource/Paul_Everingham, http://dbpedia.org/resource/Pendleton_Murrah, http://dbpedia.org/resource/Pete_McCloskey, http://dbpedia.org/resource/Peter_Thorneycroft, http://dbpedia.org/resource/Plutarco_Elías_Calles, http://dbpedia.org/resource/R._B._Bennett, http://dbpedia.org/resource/Richard_Coote,_1st_Earl_of_Bellomont, http://dbpedia.org/resource/Richard_J._Daley, http://dbpedia.org/resource/Robert_Buron, http://dbpedia.org/resource/Sam_Calder, http://dbpedia.org/resource/Scipio_Africanus, http://dbpedia.org/resource/Steve_O'Connell, http://dbpedia.org/resource/Takeo_Fukuda, http://dbpedia.org/resource/Tan_Yankai, http://dbpedia.org/resource/Vladimir_Ivashko, http://dbpedia.org/resource/Wang_Jingwei, http://dbpedia.org/resource/William_Henry_Gist, http://dbpedia.org/resource/William_Phips, http://dbpedia.org/resource/Yen_Chia-kan, http://dbpedia.org/resource/Zhang_Zuolin, http://dbpedia.org/resource/Álvaro_Obregón, http://dbpedia.org/resource/Abdalá_Bucaram, http://dbpedia.org/resource/Abdel_Hakim_Amer, http://dbpedia.org/resource/Abebe_Aregai, http://dbpedia.org/resource/Adam_Ołdakowski, http://dbpedia.org/resource/Alberto_Dahik, http://dbpedia.org/resource/Alfredo_Palacio, http://dbpedia.org/resource/Andreas_Bausewein, http://dbpedia.org/resource/Antonio_Saca, http://dbpedia.org/resource/Bernard_J._Berry, http://dbpedia.org/resource/Bruno_Erhard_Abegg, http://dbpedia.org/resource/Călin_Popescu-Tăriceanu, http://dbpedia.org/resource/Daniel_Cohn-Bendit, http://dbpedia.org/resource/Diego_Noboa, http://dbpedia.org/resource/Egon_Klepsch, http://dbpedia.org/resource/Eloy_Alfaro, http://dbpedia.org/resource/Endelkachew_Makonnen, http://dbpedia.org/resource/Fabián_Alarcón, http://dbpedia.org/resource/Francisco_Flores_Pérez, http://dbpedia.org/resource/Franz_Müntefering, http://dbpedia.org/resource/Gabriel_García_Moreno, http://dbpedia.org/resource/Giuseppe_Zanardelli, http://dbpedia.org/resource/Guadalupe_Victoria, http://dbpedia.org/resource/Guillermo_Rodríguez_(politician), http://dbpedia.org/resource/Hamdeen_Sabahi, http://dbpedia.org/resource/Henry_Plumb,_Baron_Plumb, http://dbpedia.org/resource/Jaime_Roldós_Aguilera, http://dbpedia.org/resource/James_Hillhouse, http://dbpedia.org/resource/Jean_Lambert, http://dbpedia.org/resource/Jesse_Norman, http://dbpedia.org/resource/Jim_Higgins_(British_politician), http://dbpedia.org/resource/Jim_Weaver_(Oregon_politician), http://dbpedia.org/resource/Joanna_Cherry, http://dbpedia.org/resource/John_Lewis_Ricardo, http://dbpedia.org/resource/John_Prince-Smith, http://dbpedia.org/resource/Jonas_Gahr_Støre, http://dbpedia.org/resource/José_Joaquín_de_Olmedo, http://dbpedia.org/resource/José_María_Velasco_Ibarra, http://dbpedia.org/resource/Kjell_Magne_Bondevik, http://dbpedia.org/resource/Kurt_Schumacher, http://dbpedia.org/resource/Leen_Verbeek, http://dbpedia.org/resource/Lennart_Meri, http://dbpedia.org/resource/Lucio_Gutiérrez, http://dbpedia.org/resource/Maan_Sassen, http://dbpedia.org/resource/Manfred_Rommel, http://dbpedia.org/resource/Manuel_Marín, http://dbpedia.org/resource/Marilyn_Waring, http://dbpedia.org/resource/Martijn_van_Dam, http://dbpedia.org/resource/Martin_Bangemann, http://dbpedia.org/resource/Maximiliano_Hernández_Martínez, http://dbpedia.org/resource/Michelle_Thomson, http://dbpedia.org/resource/Moses_Robinson, http://dbpedia.org/resource/Negasso_Gidada, http://dbpedia.org/resource/Nicholas_Henderson, http://dbpedia.org/resource/Nissanka_Wijeyeratne, http://dbpedia.org/resource/Non-Inscrits, http://dbpedia.org/resource/Omar_Torrijos, http://dbpedia.org/resource/Ortwin_Runde, http://dbpedia.org/resource/Per_Unckel, http://dbpedia.org/resource/Philip_Schuyler, http://dbpedia.org/resource/Pierpont_Edwards, http://dbpedia.org/resource/Pierre-Henri_Teitgen, http://dbpedia.org/resource/Pyotr_Stolypin, http://dbpedia.org/resource/René_Mayer, http://dbpedia.org/resource/Richard_Burke_Jr., http://dbpedia.org/resource/Robert_Ray_(Australian_politician), http://dbpedia.org/resource/Rodrigo_Borja_Cevallos, http://dbpedia.org/resource/Roland_Ries, http://dbpedia.org/resource/Samuel_Wilkeson, http://dbpedia.org/resource/Schafik_Handal, http://dbpedia.org/resource/Sixto_Durán_Ballén, http://dbpedia.org/resource/Stanisław_Gorczyca, http://dbpedia.org/resource/Stanisław_Żelichowski, http://dbpedia.org/resource/Sylvie_Guillaume, http://dbpedia.org/resource/Teun_van_Dijck, http://dbpedia.org/resource/Tommy_Sheppard, http://dbpedia.org/resource/Tomás_Regalado_(Salvadoran_politician), http://dbpedia.org/resource/Vicente_Ramón_Roca, http://dbpedia.org/resource/Vicente_Rocafuerte, http://dbpedia.org/resource/Wayne_Morse, http://dbpedia.org/resource/William_Benton_(senator), http://dbpedia.org/resource/Witold_Gintowt-Dziewałtowski, http://dbpedia.org/resource/Abdolhossein_Hazhir, http://dbpedia.org/resource/Abdul_Rahman_Hassan_Azzam, http://dbpedia.org/resource/Alberto_Romulo, http://dbpedia.org/resource/Alexander_Barrow, http://dbpedia.org/resource/Ali_Saibou, http://dbpedia.org/resource/Andrew_Little_(New_Zealand_politician), http://dbpedia.org/resource/Art_Phillips, http://dbpedia.org/resource/Arthur_R._Outlaw, http://dbpedia.org/resource/Aziz_Sedki, http://dbpedia.org/resource/Bob_McLeod_(politician), http://dbpedia.org/resource/Bob_Russell_(British_politician), http://dbpedia.org/resource/Brian_Gallant, http://dbpedia.org/resource/Brigitte_Barèges, http://dbpedia.org/resource/Clayton_Cosgrove, http://dbpedia.org/resource/David_Alward, http://dbpedia.org/resource/Dennis_Bevington, http://dbpedia.org/resource/Earl_Browder, http://dbpedia.org/resource/Eduardo_Santos, http://dbpedia.org/resource/Enrique_Barón_Crespo, http://dbpedia.org/resource/Eric_Williams, http://dbpedia.org/resource/Ernie_Bridge, http://dbpedia.org/resource/Francisco_Fernández_de_la_Cueva,_8th_Duke_of_Alburquerque, http://dbpedia.org/resource/George_E._McNally, http://dbpedia.org/resource/Georgina_Beyer, http://dbpedia.org/resource/Haj_Ali_Razmara, http://dbpedia.org/resource/Harry_Barron, http://dbpedia.org/resource/Herbert_Ingram, http://dbpedia.org/resource/Herman_Fokker, http://dbpedia.org/resource/James_Brokenshire, http://dbpedia.org/resource/James_McNeill, http://dbpedia.org/resource/Jan_van_der_Jagt, http://dbpedia.org/resource/Jay_Kim, http://dbpedia.org/resource/Joe_Heck, http://dbpedia.org/resource/John_E._Sununu, http://dbpedia.org/resource/John_Edward_Kelley, http://dbpedia.org/resource/John_Hussey,_1st_Baron_Hussey_of_Sleaford, http://dbpedia.org/resource/John_Manley, http://dbpedia.org/resource/John_Rogers_(Irish_lawyer), http://dbpedia.org/resource/Jorge_Sampaio, http://dbpedia.org/resource/Josep_Antoni_Duran_i_Lleida, http://dbpedia.org/resource/José_María_Moncada, http://dbpedia.org/resource/Judith_Adams, http://dbpedia.org/resource/Julio_Argentino_Roca, http://dbpedia.org/resource/Kamal_Hassan_Ali, http://dbpedia.org/resource/Kathalijne_Buitenweg, http://dbpedia.org/resource/Keith_Simpson_(politician), http://dbpedia.org/resource/Kim_Kyu-sik, http://dbpedia.org/resource/Klaus_Berntsen, http://dbpedia.org/resource/Lane_Dwinell, http://dbpedia.org/resource/Leon_C._Phillips, http://dbpedia.org/resource/Leung_Kwok-hung, http://dbpedia.org/resource/Lorraine_Michael, http://dbpedia.org/resource/Luis_Somoza_Debayle, http://dbpedia.org/resource/Madeleine_Moon, http://dbpedia.org/resource/Mamadou_Tandja, http://dbpedia.org/resource/Marielle_de_Sarnez, http://dbpedia.org/resource/Martin_Maginnis, http://dbpedia.org/resource/Max_Rood, http://dbpedia.org/resource/Michael_McLeod_(politician), http://dbpedia.org/resource/Mike_Dow, http://dbpedia.org/resource/Mohammad_Hashim_Khan, http://dbpedia.org/resource/Mohammed_Khalid, http://dbpedia.org/resource/Napoleon_B._Broward, http://dbpedia.org/resource/Napoléon_Joseph_Ney, http://dbpedia.org/resource/Nicholas_Attygalle, http://dbpedia.org/resource/Nicholas_Gilman, http://dbpedia.org/resource/Nick_Hawkins, http://dbpedia.org/resource/Olemic_Thommessen, http://dbpedia.org/resource/Philippe_Germain, http://dbpedia.org/resource/Pieter_van_Geel, http://dbpedia.org/resource/René_Harris, http://dbpedia.org/resource/Robert_Rex, http://dbpedia.org/resource/Roy_Ferguson, http://dbpedia.org/resource/Sam_Jones_(mayor), http://dbpedia.org/resource/Sandy_Stimpson, http://dbpedia.org/resource/Shahryar_Khan, http://dbpedia.org/resource/Simon_Bradstreet, http://dbpedia.org/resource/Squire_Jeremiah, http://dbpedia.org/resource/Supachai_Panitchpakdi, http://dbpedia.org/resource/Sándor_Garbai, http://dbpedia.org/resource/Tim_Lee_Carter, http://dbpedia.org/resource/Victor_Bodiu, http://dbpedia.org/resource/Vito_Marcantonio, http://dbpedia.org/resource/Walter_M._Pierce, http://dbpedia.org/resource/Ahmed_Majdalani, http://dbpedia.org/resource/Andrzej_Czerwiński, http://dbpedia.org/resource/Annita_van_Iersel, http://dbpedia.org/resource/Antonio_Segni, http://dbpedia.org/resource/Arthur_Lee,_1st_Viscount_Lee_of_Fareham, http://dbpedia.org/resource/Cezary_Grabarczyk, http://dbpedia.org/resource/Clément_Mouamba, http://dbpedia.org/resource/Craig_Knowles, http://dbpedia.org/resource/Dariusz_Lipiński, http://dbpedia.org/resource/Donald_Stovel_Macdonald, http://dbpedia.org/resource/Donna_Frye, http://dbpedia.org/resource/Elliot_Morley, http://dbpedia.org/resource/Elżbieta_Radziszewska, http://dbpedia.org/resource/Felipe_González_González, http://dbpedia.org/resource/Franco_Maria_Malfatti, http://dbpedia.org/resource/Frank_Evans_(politician), http://dbpedia.org/resource/Frederick_Walker_Pitkin, http://dbpedia.org/resource/Gary_Gray_(politician), http://dbpedia.org/resource/Gordon_L._Allott, http://dbpedia.org/resource/Harry_G._Leslie, http://dbpedia.org/resource/Imad_Khamis, http://dbpedia.org/resource/James_Bradley_Orman, http://dbpedia.org/resource/Jan_Rokita, http://dbpedia.org/resource/Jim_Harrison_(politician), http://dbpedia.org/resource/John_Ducker, http://dbpedia.org/resource/John_E._Rickards, http://dbpedia.org/resource/John_Endecott, http://dbpedia.org/resource/John_Radcliffe_(physician), http://dbpedia.org/resource/Jonathan_Belcher, http://dbpedia.org/resource/Krystyna_Łybacka, http://dbpedia.org/resource/Laurie_Brereton, http://dbpedia.org/resource/Leader_of_the_Liberal_Democrats, http://dbpedia.org/resource/Luigi_Carlo_Farini, http://dbpedia.org/resource/Maria_Pasło-Wiśniewska, http://dbpedia.org/resource/Małgorzata_Stryjska, http://dbpedia.org/resource/Michael_Hatton, http://dbpedia.org/resource/Michał_Stuligrosz, http://dbpedia.org/resource/Mohammed_bin_Rashid_Al_Maktoum, http://dbpedia.org/resource/Nick_Palmer, http://dbpedia.org/resource/Oldřich_Černík, http://dbpedia.org/resource/Pen_Sovan, http://dbpedia.org/resource/Philippe_Saunier, http://dbpedia.org/resource/Pierre_Chatenet, http://dbpedia.org/resource/Ralph_F._Gates, http://dbpedia.org/resource/Raymond_P._Kogovsek, http://dbpedia.org/resource/Rex_Patterson, http://dbpedia.org/resource/Simon_Guggenheim, http://dbpedia.org/resource/Stefan_Niesiołowski, http://dbpedia.org/resource/Sławomir_Nitras, http://dbpedia.org/resource/Terry_Fields, http://dbpedia.org/resource/Tomasz_Górski_(politician), http://dbpedia.org/resource/Tomasz_Tomczykiewicz, http://dbpedia.org/resource/Urszula_Augustyn, http://dbpedia.org/resource/Ville_Itälä, http://dbpedia.org/resource/Waldy_Dzikowski, http://dbpedia.org/resource/Youssef_Chahed, http://dbpedia.org/resource/Addison_S._McClure, http://dbpedia.org/resource/Albert_Allard, http://dbpedia.org/resource/Alex_J._Groesbeck, http://dbpedia.org/resource/Anders_Nordström, http://dbpedia.org/resource/Arthur_Dunkel, http://dbpedia.org/resource/Bert_Weeks, http://dbpedia.org/resource/Blanche_Bruce, http://dbpedia.org/resource/Bob_MacMillan, http://dbpedia.org/resource/Brian_Masse, http://dbpedia.org/resource/Brian_Mikkelsen, http://dbpedia.org/resource/Brock_Chisholm, http://dbpedia.org/resource/Charles_Phelps_Taft_II, http://dbpedia.org/resource/Cheryl_Hardcastle, http://dbpedia.org/resource/Currie_Dixon, http://dbpedia.org/resource/Dan_Curtis_(politician), http://dbpedia.org/resource/Darrell_Pasloski, http://dbpedia.org/resource/David_Laxton, http://dbpedia.org/resource/David_Logan_(British_politician), http://dbpedia.org/resource/David_Rhys_Grenfell, http://dbpedia.org/resource/Drew_Dilkens, http://dbpedia.org/resource/Edgar_Vincent,_1st_Viscount_D'Abernon, http://dbpedia.org/resource/Edward_LeRoy_Bowerman, http://dbpedia.org/resource/Edward_O._Wolcott, http://dbpedia.org/resource/El-Ouali_Mustapha_Sayed, http://dbpedia.org/resource/Elaine_Taylor_(politician), http://dbpedia.org/resource/Elizabeth_Hanson, http://dbpedia.org/resource/Eric_Wyndham_White, http://dbpedia.org/resource/Francis_G._Newlands, http://dbpedia.org/resource/Frank_W._Burke, http://dbpedia.org/resource/Halfdan_T._Mahler, http://dbpedia.org/resource/Harriett_Baldwin, http://dbpedia.org/resource/Ione_Christensen, http://dbpedia.org/resource/Jack_Metcalf, http://dbpedia.org/resource/James_Fairman_Fielder, http://dbpedia.org/resource/James_Monroe_(congressman), http://dbpedia.org/resource/Jan_Stick, http://dbpedia.org/resource/John_Alden_Dix, http://dbpedia.org/resource/John_Chambers_(politician), http://dbpedia.org/resource/John_Franklin_Fort, http://dbpedia.org/resource/John_Neilson_Gladstone, http://dbpedia.org/resource/John_Scott_Harrison, http://dbpedia.org/resource/Jonathan_T._Updegraff, http://dbpedia.org/resource/Joseph_D._Taylor, http://dbpedia.org/resource/Josiah_Ogden_Hoffman, http://dbpedia.org/resource/Kate_White_(politician), http://dbpedia.org/resource/Keith_Irving, http://dbpedia.org/resource/Larry_Bagnell, http://dbpedia.org/resource/Leah_Manning, http://dbpedia.org/resource/Lisa_Gretzky, http://dbpedia.org/resource/Lois_Moorcroft, http://dbpedia.org/resource/Louis_François_Georges_Baby, http://dbpedia.org/resource/Lucien_Millevoye, http://dbpedia.org/resource/Marcolino_Gomes_Candau, http://dbpedia.org/resource/Mark_Garnier, http://dbpedia.org/resource/Mike_Nixon_(politician), http://dbpedia.org/resource/Monika_Stolz, http://dbpedia.org/resource/Nelson_Dingley_Jr., http://dbpedia.org/resource/Nigel_Huddleston, http://dbpedia.org/resource/Olivier_Long, http://dbpedia.org/resource/Otto_Wels, http://dbpedia.org/resource/Patalione_Kanimoa, http://dbpedia.org/resource/Patrick_Brown_(politician), http://dbpedia.org/resource/Percy_Hatfield, http://dbpedia.org/resource/Ralph_D._Cole, http://dbpedia.org/resource/Richard_M._Bishop, http://dbpedia.org/resource/Robert_Forke, http://dbpedia.org/resource/Ryan_Leef, http://dbpedia.org/resource/Scott_Brison, http://dbpedia.org/resource/Scott_Kent, http://dbpedia.org/resource/Sir_John_Gladstone,_1st_Baronet, http://dbpedia.org/resource/Tammy_Lobato, http://dbpedia.org/resource/Theodore_E._Burton, http://dbpedia.org/resource/Thomas_Crerar, http://dbpedia.org/resource/Thomas_Herbert_Lennox, http://dbpedia.org/resource/Thomas_Howard,_3rd_Viscount_Howard_of_Bindon, http://dbpedia.org/resource/Thomas_J._Walsh, http://dbpedia.org/resource/Thomas_Osborne_Davis_(Canadian_politician), http://dbpedia.org/resource/Vernon_Wallace_Thomson, http://dbpedia.org/resource/Wayne_K._Curry, http://dbpedia.org/resource/Willem_Buys, http://dbpedia.org/resource/William_C._Preston, http://dbpedia.org/resource/Clive_Lewis_(politician), http://dbpedia.org/resource/Danny_Kinahan, http://dbpedia.org/resource/Eddie_McGrady, http://dbpedia.org/resource/Francisco_Luna_Kan, http://dbpedia.org/resource/František_Mareš, http://dbpedia.org/resource/Gabriel_Gustafsson_Oxenstierna, http://dbpedia.org/resource/Gaspar_de_la_Cerda,_8th_Count_of_Galve, http://dbpedia.org/resource/George_Freeman_(politician), http://dbpedia.org/resource/Guy_Ravier, http://dbpedia.org/resource/Henri_Queuille, http://dbpedia.org/resource/Henry_Brooke,_Baron_Brooke_of_Cumnor, http://dbpedia.org/resource/Hishammuddin_Hussein, http://dbpedia.org/resource/Hugh_Fraser_(British_politician), http://dbpedia.org/resource/Huw_Edwards_(politician), http://dbpedia.org/resource/Ioannis_Kolettis, http://dbpedia.org/resource/Ivonne_Ortega_Pacheco, http://dbpedia.org/resource/James_Olmsted, http://dbpedia.org/resource/Jan_Kavan, http://dbpedia.org/resource/Janez_Stanovnik, http://dbpedia.org/resource/Jean-Louis_Tixier-Vignancour, http://dbpedia.org/resource/Jean_Auroux, http://dbpedia.org/resource/Jerry_Lewis_(California_politician), http://dbpedia.org/resource/John_West_(governor), http://dbpedia.org/resource/Josef_Pfitzner, http://dbpedia.org/resource/José_Mariano_Salas, http://dbpedia.org/resource/Ken_Maginnis, http://dbpedia.org/resource/Lewis_Williams_Douglas, http://dbpedia.org/resource/Louis_Marie_de_Milet_de_Mureau, http://dbpedia.org/resource/Luigi_Pelloux, http://dbpedia.org/resource/Marie-Guite_Dufay, http://dbpedia.org/resource/Matthew_Griswold_(governor), http://dbpedia.org/resource/Michel_Dagbert, http://dbpedia.org/resource/Najeeb_Jung, http://dbpedia.org/resource/Nigel_Birch,_Baron_Rhyl, http://dbpedia.org/resource/Nils_Turesson_Bielke, http://dbpedia.org/resource/Patrick_Kanner, http://dbpedia.org/resource/Pau_Claris_i_Casademunt, http://dbpedia.org/resource/René_Haby, http://dbpedia.org/resource/Richard_Drax, http://dbpedia.org/resource/Richard_Ingoldesby, http://dbpedia.org/resource/Richard_Treat, http://dbpedia.org/resource/Richard_Webb_(settler), http://dbpedia.org/resource/Rolando_Zapata_Bello, http://dbpedia.org/resource/Rudolf_Amelunxen, http://dbpedia.org/resource/Sarah_Wollaston, http://dbpedia.org/resource/Shiraz_Shariff, http://dbpedia.org/resource/Simon_Milton_(politician), http://dbpedia.org/resource/Simon_Wright_(politician), http://dbpedia.org/resource/Thomas_H._McKittrick, http://dbpedia.org/resource/Thomas_Penn, http://dbpedia.org/resource/Volkert_Overlander, http://dbpedia.org/resource/William_Hutchinson_(Rhode_Island), http://dbpedia.org/resource/Élie,_duc_Decazes, http://dbpedia.org/resource/A_C_Wharton, http://dbpedia.org/resource/Adeline_Hazan, http://dbpedia.org/resource/Alok_Sanjar, http://dbpedia.org/resource/Charles_Carnan_Ridgely, http://dbpedia.org/resource/Claude_Allègre, http://dbpedia.org/resource/David_Clark_(New_Zealand_politician), http://dbpedia.org/resource/David_Usupashvili, http://dbpedia.org/resource/Ed_Mangano, http://dbpedia.org/resource/Elizabeth_Bobo, http://dbpedia.org/resource/George_Stoneman, http://dbpedia.org/resource/Gregory_Meeks, http://dbpedia.org/resource/Guy_Guzzone, http://dbpedia.org/resource/Ivan_Skoropadsky, http://dbpedia.org/resource/James_E._Van_Zandt, http://dbpedia.org/resource/Jean-Marie_Bockel, http://dbpedia.org/resource/Jim_Strickland_(politician), http://dbpedia.org/resource/Jin_Yi_(Han_dynasty), http://dbpedia.org/resource/John_N._Dalton, http://dbpedia.org/resource/Keith_Best, http://dbpedia.org/resource/Leo_Falcam, http://dbpedia.org/resource/Manuel_Gayoso_de_Lemos, http://dbpedia.org/resource/Mao_Jie, http://dbpedia.org/resource/Martial_Borye_Desrenaudes, http://dbpedia.org/resource/Mike_Spano, http://dbpedia.org/resource/Nelson_Taylor, http://dbpedia.org/resource/Pantelis_Karasevdas, http://dbpedia.org/resource/Park_Eun-sik, http://dbpedia.org/resource/Peter_J._Schmitt, http://dbpedia.org/resource/Philip_La_Follette, http://dbpedia.org/resource/Philippe_Richert, http://dbpedia.org/resource/Robert_Newell_(politician), http://dbpedia.org/resource/Shane_Pendergrass, http://dbpedia.org/resource/Theobald_Otjen, http://dbpedia.org/resource/Thomas_Gore, http://dbpedia.org/resource/Tristam_Burges, http://dbpedia.org/resource/William_S._Hanna, http://dbpedia.org/resource/Aaron_Hobart, http://dbpedia.org/resource/Albert_Johnson_(congressman), http://dbpedia.org/resource/Alexander_Forrest, http://dbpedia.org/resource/Amir_Kabir, http://dbpedia.org/resource/Asbury_Francis_Lever, http://dbpedia.org/resource/Benjamin_F._Hopkins, http://dbpedia.org/resource/Benning_W._Jenness, http://dbpedia.org/resource/Caleb_Carr_(governor), http://dbpedia.org/resource/Chang_Chun-hsiung, http://dbpedia.org/resource/Charles_C._Gossett, http://dbpedia.org/resource/Charles_Crompton, http://dbpedia.org/resource/Charles_E._Stuart, http://dbpedia.org/resource/Charles_G._Atherton, http://dbpedia.org/resource/Charles_Henderson_(Nevada_politician), http://dbpedia.org/resource/Clarence_Emir_Allen, http://dbpedia.org/resource/Coke_R._Stevenson, http://dbpedia.org/resource/Cristóbal_Vaca_de_Castro, http://dbpedia.org/resource/D._Lane_Powers, http://dbpedia.org/resource/Daniel_Sheldon_Norton, http://dbpedia.org/resource/Daniel_Sturgeon, http://dbpedia.org/resource/Daniel_T._Jewett, http://dbpedia.org/resource/Dave_E._Satterfield,_Jr., http://dbpedia.org/resource/David_Atwood, http://dbpedia.org/resource/David_Kilgour, http://dbpedia.org/resource/DeWitt_Clinton_Cregier, http://dbpedia.org/resource/Dick_Thompson_Morgan, http://dbpedia.org/resource/Edmund_Platt, http://dbpedia.org/resource/Edward_Dowse, http://dbpedia.org/resource/Elisha_Baxter, http://dbpedia.org/resource/Elmer_Thomas, http://dbpedia.org/resource/Ephraim_F._Morgan, http://dbpedia.org/resource/Fletcher_Bowron, http://dbpedia.org/resource/Francis_B._Stockbridge, http://dbpedia.org/resource/Frank_A._Mathews,_Jr., http://dbpedia.org/resource/George_B._McClellan_Jr., http://dbpedia.org/resource/George_Sykes_(New_Jersey_politician), http://dbpedia.org/resource/George_Watkins_(politician), http://dbpedia.org/resource/Gillespie_V._Montgomery, http://dbpedia.org/resource/Greg_Laughlin, http://dbpedia.org/resource/Henry_F._Hollis, http://dbpedia.org/resource/Henry_Glen, http://dbpedia.org/resource/Henry_L._Pinckney, http://dbpedia.org/resource/Henry_W._Blair, http://dbpedia.org/resource/Howard_W._Smith, http://dbpedia.org/resource/Ira_Harris, http://dbpedia.org/resource/J._A._O._Preus, http://dbpedia.org/resource/J._Vaughan_Gary, http://dbpedia.org/resource/Jacob_Aaron_Westervelt, http://dbpedia.org/resource/Jacob_Radcliff, http://dbpedia.org/resource/James_B._Pearson, http://dbpedia.org/resource/James_Hartness, http://dbpedia.org/resource/James_Hodge_Codding, http://dbpedia.org/resource/James_P._Richards, http://dbpedia.org/resource/James_Thomas_Heflin, http://dbpedia.org/resource/James_W._Mott, http://dbpedia.org/resource/Jamie_L._Whitten, http://dbpedia.org/resource/Janez_Janša_(performance_artist), http://dbpedia.org/resource/Janko_Kamauf, http://dbpedia.org/resource/Jesse_Slocumb, http://dbpedia.org/resource/Jimmy_Hayes, http://dbpedia.org/resource/John_G._Palfrey, http://dbpedia.org/resource/John_H._Rousselot, http://dbpedia.org/resource/John_J._Jacob_(West_Virginia), http://dbpedia.org/resource/John_M._Patton, http://dbpedia.org/resource/John_Moses_(American_politician), http://dbpedia.org/resource/John_S._Phelps, http://dbpedia.org/resource/John_T._Deweese, http://dbpedia.org/resource/John_Tayler, http://dbpedia.org/resource/John_W._Thomas, http://dbpedia.org/resource/John_Wolcott_Stewart, http://dbpedia.org/resource/Josef_Klaus, http://dbpedia.org/resource/Joseph_C._Yates, http://dbpedia.org/resource/Joseph_W._Fordney, http://dbpedia.org/resource/Joseph_Wilson_Ervin, http://dbpedia.org/resource/José_María_Obando, http://dbpedia.org/resource/Julius_C._Burrows, http://dbpedia.org/resource/Kai_Feng_(politician), http://dbpedia.org/resource/Kinsley_S._Bingham, http://dbpedia.org/resource/L._Mendel_Rivers, http://dbpedia.org/resource/Lafayette_S._Foster, http://dbpedia.org/resource/Lu_Zongyu, http://dbpedia.org/resource/Michael_J._Kirwan, http://dbpedia.org/resource/Mignon_Clyburn, http://dbpedia.org/resource/Miguel_Antonio_Caro, http://dbpedia.org/resource/Milan_Bandić, http://dbpedia.org/resource/Murphy_J._Foster_Jr., http://dbpedia.org/resource/Nathaniel_P._Tallmadge, http://dbpedia.org/resource/Pedro_Álvarez_de_Toledo,_Marquis_of_Villafranca, http://dbpedia.org/resource/Peter_Murphy_(businessman), http://dbpedia.org/resource/Prentiss_Mellen, http://dbpedia.org/resource/Robert_L._King, http://dbpedia.org/resource/Robert_Ridgway_(congressman), http://dbpedia.org/resource/Réal_Ménard, http://dbpedia.org/resource/Sam_Webb, http://dbpedia.org/resource/Samuel_Howell, http://dbpedia.org/resource/Samuel_S._Stratton, http://dbpedia.org/resource/Shi_Tianze, http://dbpedia.org/resource/Simon_J._Schermerhorn, http://dbpedia.org/resource/Sutiyoso, http://dbpedia.org/resource/Tang_Fei, http://dbpedia.org/resource/Thomas_Edward_Campbell, http://dbpedia.org/resource/Tong_Jixu, http://dbpedia.org/resource/Walter_R._Tucker_III, http://dbpedia.org/resource/Whitehead_Hicks, http://dbpedia.org/resource/Wil_Schuurman, http://dbpedia.org/resource/William_Cogswell, http://dbpedia.org/resource/William_E._Stevenson, http://dbpedia.org/resource/William_J._Browning, http://dbpedia.org/resource/William_Kristol, http://dbpedia.org/resource/William_M._Tuck, http://dbpedia.org/resource/William_Moore_McCulloch, http://dbpedia.org/resource/William_Peters_Hepburn, http://dbpedia.org/resource/William_S._Archer, http://dbpedia.org/resource/William_Smyth_(congressman), http://dbpedia.org/resource/Willis_C._Hawley, http://dbpedia.org/resource/Xiao_Ke, http://dbpedia.org/resource/Ximo_Puig, http://dbpedia.org/resource/Zhang_Jinghui, http://dbpedia.org/resource/Zheng_Xiaoxu, http://dbpedia.org/resource/Amory_Holbrook, http://dbpedia.org/resource/Andy_Brandt, http://dbpedia.org/resource/Bob_Bailey_(politician), http://dbpedia.org/resource/Bud_Shuster, http://dbpedia.org/resource/Charles_F._Curry, http://dbpedia.org/resource/Dale_Folwell, http://dbpedia.org/resource/Francisco_de_la_Lastra, http://dbpedia.org/resource/Frank_Vandenbroucke_(politician), http://dbpedia.org/resource/George_Henry_Hoyt, http://dbpedia.org/resource/George_Johnson_Clarke, http://dbpedia.org/resource/Gordon_Banks_(politician), http://dbpedia.org/resource/Gordon_Krantz, http://dbpedia.org/resource/James_Kidd_Flemming, http://dbpedia.org/resource/John_B._McNair, http://dbpedia.org/resource/John_Douglas_Hazen, http://dbpedia.org/resource/John_H._Couch, http://dbpedia.org/resource/John_Motley_Morehead, http://dbpedia.org/resource/John_P._Gaines, http://dbpedia.org/resource/Julius_Debrah, http://dbpedia.org/resource/Lakshmanan_Sathyavagiswaran, http://dbpedia.org/resource/Larry_Langford, http://dbpedia.org/resource/Louis_Tobback, http://dbpedia.org/resource/Marceil_Saddy, http://dbpedia.org/resource/Marilyn_Gladu, http://dbpedia.org/resource/Meir_Lichtenstein, http://dbpedia.org/resource/Mike_Bradley_(politician), http://dbpedia.org/resource/Patricia_Davidson, http://dbpedia.org/resource/Patrick_Cormack, http://dbpedia.org/resource/Paul_Blundy, http://dbpedia.org/resource/Peter_G._Stewart, http://dbpedia.org/resource/Philip_Foster, http://dbpedia.org/resource/Richard_F._Kneip, http://dbpedia.org/resource/Robert_Ghiz, http://dbpedia.org/resource/Samuel_Parker_(Oregon_politician), http://dbpedia.org/resource/Thomas_D._Schall, http://dbpedia.org/resource/Tom_Tait, http://dbpedia.org/resource/William_Dumaresq, http://dbpedia.org/resource/William_G._T'Vault, http://dbpedia.org/resource/William_Pugsley, http://dbpedia.org/resource/Alain_Gournac, http://dbpedia.org/resource/Albert_G._Porter, http://dbpedia.org/resource/Amma_Asante_(politician), http://dbpedia.org/resource/Andrew_Williams_(congressman), http://dbpedia.org/resource/Angélica_Rivera, http://dbpedia.org/resource/Benjamin_Stark, http://dbpedia.org/resource/Berry_Vrbanovic, http://dbpedia.org/resource/Bruce_Fairchild_Barton, http://dbpedia.org/resource/Calvin_T._Hulburd, http://dbpedia.org/resource/Carlos_Ferrero, http://dbpedia.org/resource/Charles-Noël_Barbès, http://dbpedia.org/resource/Charles_Hooks, http://dbpedia.org/resource/Chen_Mingyi, http://dbpedia.org/resource/Clifton_R._Breckinridge, http://dbpedia.org/resource/Conrad_Baker, http://dbpedia.org/resource/Daiene_Vernile, http://dbpedia.org/resource/Daniel_D._Pratt, http://dbpedia.org/resource/Dominique_Braye, http://dbpedia.org/resource/Edward_C._Marshall, http://dbpedia.org/resource/Elbridge_G._Lapham, http://dbpedia.org/resource/Enrique_Martínez_y_Martínez, http://dbpedia.org/resource/Ernesto_Ruffo_Appel, http://dbpedia.org/resource/Fernando_Canales_Clariond, http://dbpedia.org/resource/Francis_Kernan, http://dbpedia.org/resource/Frank_Thompson, http://dbpedia.org/resource/George_H._Mahon, http://dbpedia.org/resource/George_W._Johnson_(governor), http://dbpedia.org/resource/George_William_Palmer_(New_York), http://dbpedia.org/resource/Gill_Furniss, http://dbpedia.org/resource/Guy_Reschenthaler, http://dbpedia.org/resource/Gérard_Larcher, http://dbpedia.org/resource/Harold_Albrecht, http://dbpedia.org/resource/Henry_Bellmon, http://dbpedia.org/resource/Ike_Franklin_Andrews, http://dbpedia.org/resource/Isaac_D._Barnard, http://dbpedia.org/resource/James_D._Williams, http://dbpedia.org/resource/James_Johnson_(Kentucky), http://dbpedia.org/resource/James_T._Patterson, http://dbpedia.org/resource/James_Tien_(politician), http://dbpedia.org/resource/Jean-Michel_Baylet, http://dbpedia.org/resource/Jean-Michel_Bertrand, http://dbpedia.org/resource/John_Alexander_Ahl, http://dbpedia.org/resource/John_Kean_(New_Jersey), http://dbpedia.org/resource/John_Motley_Morehead_III, http://dbpedia.org/resource/John_T._Wait, http://dbpedia.org/resource/Judy_Wasylycia-Leis, http://dbpedia.org/resource/Konstantinos_Tsaldaris, http://dbpedia.org/resource/Liao_Zhigao, http://dbpedia.org/resource/Louise_Haigh, http://dbpedia.org/resource/Luke_P._Poland, http://dbpedia.org/resource/Mark_Foley, http://dbpedia.org/resource/Marwan_Tabbara, http://dbpedia.org/resource/Michael_Harris_(politician), http://dbpedia.org/resource/Naheed_Nenshi, http://dbpedia.org/resource/Nancy_Boyda, http://dbpedia.org/resource/Nicolas_About, http://dbpedia.org/resource/Pascual_Ortiz_Rubio, http://dbpedia.org/resource/Paul_Blomfield, http://dbpedia.org/resource/Pierre_Bédier, http://dbpedia.org/resource/Raj_Saini, http://dbpedia.org/resource/Richard_Hawes, http://dbpedia.org/resource/Robert_Tyndall, http://dbpedia.org/resource/Roberto_Dañino, http://dbpedia.org/resource/Scott_J._Andreassi, http://dbpedia.org/resource/Socrates_Tuttle, http://dbpedia.org/resource/Tomás_Yarrington, http://dbpedia.org/resource/Ulises_Ruiz_Ortiz, http://dbpedia.org/resource/Warner_Miller, http://dbpedia.org/resource/William_Brown_(congressman), http://dbpedia.org/resource/William_Campbell_Preston_Breckinridge, http://dbpedia.org/resource/William_Wright_Southgate, http://dbpedia.org/resource/Zhang_Dingcheng, http://dbpedia.org/resource/Adam_Kinzinger, http://dbpedia.org/resource/Allen_D._Candler, http://dbpedia.org/resource/Andrew_Butler, http://dbpedia.org/resource/Bill_Smith_(Alberta_politician), http://dbpedia.org/resource/C._Jack_Ellis, http://dbpedia.org/resource/Charles_Gordon_Edwards, http://dbpedia.org/resource/Charles_S._Thomas, http://dbpedia.org/resource/Dale_Alford, http://dbpedia.org/resource/David_Wallace_(Indiana_politician), http://dbpedia.org/resource/Dennis_Pennington, http://dbpedia.org/resource/Doug_Collins_(politician), http://dbpedia.org/resource/Dudley_Mays_Hughes, http://dbpedia.org/resource/Edith_Green, http://dbpedia.org/resource/George_Washington_Donaghey, http://dbpedia.org/resource/Gus_Wingfield, http://dbpedia.org/resource/Harry_Kelly_(politician), http://dbpedia.org/resource/Henry_Dworshak, http://dbpedia.org/resource/Herman_Maxwell_Batten, http://dbpedia.org/resource/Iris_Faircloth_Blitch, http://dbpedia.org/resource/Jackie_Walorski, http://dbpedia.org/resource/James_Abercrombie_(Congressman), http://dbpedia.org/resource/James_B._Ray, http://dbpedia.org/resource/Jefferson_F._Long, http://dbpedia.org/resource/Jiang_Hua, http://dbpedia.org/resource/John_Madden_(Tasmanian_politician), http://dbpedia.org/resource/John_Pickler, http://dbpedia.org/resource/John_Reynolds_(Royal_Navy_officer), http://dbpedia.org/resource/Jonathan_Lindley, http://dbpedia.org/resource/Larry_LaRocco, http://dbpedia.org/resource/Linda_Langston, http://dbpedia.org/resource/M._Clifford_Townsend, http://dbpedia.org/resource/Mark_Souder, http://dbpedia.org/resource/Moses_Wisner, http://dbpedia.org/resource/Rahui_Katene, http://dbpedia.org/resource/Robert_Docking, http://dbpedia.org/resource/Robert_Williamson_Steele, http://dbpedia.org/resource/Rong_Yiren, http://dbpedia.org/resource/Samuel_H._Shapiro, http://dbpedia.org/resource/Samuel_Thatcher, http://dbpedia.org/resource/Shelly_Glover, http://dbpedia.org/resource/Sherman_P._Lloyd, http://dbpedia.org/resource/Stanley_L._Greigg, http://dbpedia.org/resource/Tim_Johnson_(Illinois_politician), http://dbpedia.org/resource/Walter_R._Stubbs, http://dbpedia.org/resource/Walter_Washington, http://dbpedia.org/resource/William_Alexander_Smith_(politician), http://dbpedia.org/resource/William_Ephraim_Smith, http://dbpedia.org/resource/William_Ernest_Smith, http://dbpedia.org/resource/William_Jay_Smith_(Tennessee_politician), http://dbpedia.org/resource/William_Laird_Smith, http://dbpedia.org/resource/William_Masters_Smith, http://dbpedia.org/resource/William_Nathan_Harrell_Smith, http://dbpedia.org/resource/William_Robert_Smith, http://dbpedia.org/resource/William_Smith_(1849–1913), http://dbpedia.org/resource/William_Smith_(Maryland), http://dbpedia.org/resource/William_Smith_(South_Carolina_representative), http://dbpedia.org/resource/William_Stephens_Smith, http://dbpedia.org/resource/William_Washington_Larsen, http://dbpedia.org/resource/Willis_J._Bailey, http://dbpedia.org/resource/Andrew_Jackson_Caldwell, http://dbpedia.org/resource/Arthur_W._Mitchell, http://dbpedia.org/resource/Asa_Hodges, http://dbpedia.org/resource/Augusta_Clark, http://dbpedia.org/resource/Charles_Marsh, http://dbpedia.org/resource/Dominique_Riquet, http://dbpedia.org/resource/Don_Taylor_Udall, http://dbpedia.org/resource/Earl_Dewitt_Hutto, http://dbpedia.org/resource/Ehsan_Jafri, http://dbpedia.org/resource/Elvin_Nimrod, http://dbpedia.org/resource/Frederick_Steiwer, http://dbpedia.org/resource/Gerhard_Louis_De_Geer, http://dbpedia.org/resource/Glenn_Hearn, http://dbpedia.org/resource/Gregorio_F._Zaide, http://dbpedia.org/resource/Harvey_Butler_Fergusson, http://dbpedia.org/resource/Henry_Taylor_Blow, http://dbpedia.org/resource/I._Kimbell_Hicks, http://dbpedia.org/resource/Jaime_Nebot, http://dbpedia.org/resource/James_S._Davenport, http://dbpedia.org/resource/Jean-Pierre_Godefroy, http://dbpedia.org/resource/Jesse_Addison_Udall, http://dbpedia.org/resource/Joe_W._Davis, http://dbpedia.org/resource/Joseph_Abbott_(Texas_politician), http://dbpedia.org/resource/Joseph_D._Bedle, http://dbpedia.org/resource/Joshua_B._Lee, http://dbpedia.org/resource/Josiah_Gardner_Abbott, http://dbpedia.org/resource/Josiah_Patterson, http://dbpedia.org/resource/Julian_L._McPhillips, http://dbpedia.org/resource/Lee_Cooke, http://dbpedia.org/resource/Loretta_Spencer, http://dbpedia.org/resource/Martine_Lignières-Cassou, http://dbpedia.org/resource/Miguel_Abadía_Méndez, http://dbpedia.org/resource/Mike_Arnold, http://dbpedia.org/resource/Nancy_Worley, http://dbpedia.org/resource/Paul_Vallely, http://dbpedia.org/resource/Per_Engdahl, http://dbpedia.org/resource/Peter_Koutoujian, http://dbpedia.org/resource/Ralph_Tyler_Smith, http://dbpedia.org/resource/René_Garrec, http://dbpedia.org/resource/Richard_W._Austin, http://dbpedia.org/resource/Robert_L._Williams, http://dbpedia.org/resource/Robert_Lee_Rowe, http://dbpedia.org/resource/Salvador_María_del_Carril, http://dbpedia.org/resource/Samuel_L._Powers, http://dbpedia.org/resource/Samuel_W._Pennypacker, http://dbpedia.org/resource/Santiago_Derqui, http://dbpedia.org/resource/Stojan_Novaković, http://dbpedia.org/resource/Stubbs_(cat), http://dbpedia.org/resource/T._T._Terry, http://dbpedia.org/resource/Teodoro_A._Dehesa_Méndez, http://dbpedia.org/resource/Walter_A._Huxman, http://dbpedia.org/resource/William_Emerson_Barrett, http://dbpedia.org/resource/William_J._Alston, http://dbpedia.org/resource/William_J._Grayson, http://dbpedia.org/resource/Winston_Lackin, http://dbpedia.org/resource/Zhang_Tingyu, http://dbpedia.org/resource/Alexander_Robey_Shepherd, http://dbpedia.org/resource/Andrew_B._Jackson, http://dbpedia.org/resource/Andrew_Canova, http://dbpedia.org/resource/Anson_Burlingame]... [INFO] P: 2.681749475839875E-4, R: 1.0, F: 5.36206098125716E-4 -[INFO] Cumulative metrics, sample: 117, P: 0.356, R: 0.440, macro F1: 0.360, macro F1*: 0.394 -[INFO] QALD Cumulative metrics, sample: 117, P: 0.570, R: 0.440, macro F1: 0.360, QALD macro F1: 0.496 -[INFO] Current time: 2506 ms, average: 3534.50 ms/ques, entity detection: 811.33 ms/ques (linking tool: 723.22 ms/ques), relation detection: 1129.63 ms/ques (similarity: 638.57 ms/ques), EDG generation: 51.49 ms/ques +[INFO] Cumulative metrics, sample: 117, P: 0.356, R: 0.440, macro F1: 0.394 +[INFO] QALD Cumulative metrics, sample: 117, P: 0.570, R: 0.440, average F1: 0.360, QALD macro F1: 0.496 +[INFO] Current question consumed time: 2235 ms, average: 3220.13 ms/ques, entity detection: 720.98 ms/ques (linking tool: 645.46 ms/ques), relation detection: 1056.04 ms/ques (similarity: 638.26 ms/ques), EDG generation: 52.67 ms/ques [INFO] Entity Linking P: 0.28205128205128205, R: 0.28205128205128205, micro F1: 0.28205128205128205, macro F1: 0.28205128205128205 [INFO] Relation Linking P: 0.29914529914529914, R: 0.29914529914529914, micro F1: 0.29914529914529914, macro F1: 0.29914529914529914 [INFO] Type Linking P: 0.8717948717948718, R: 0.8717948717948718, micro F1: 0.8717948717948718, macro F1: 0.8717948717948718 @@ -1762,9 +1762,9 @@ Question 117: How big is the earth's diameter? [INFO] Golden Answer: [1.2742e+07] [INFO] Predicted Answer: [] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 118, P: 0.353, R: 0.436, macro F1: 0.357, macro F1*: 0.390 -[INFO] QALD Cumulative metrics, sample: 118, P: 0.574, R: 0.436, macro F1: 0.357, QALD macro F1: 0.495 -[INFO] Current time: 149 ms, average: 3505.82 ms/ques, entity detection: 804.46 ms/ques (linking tool: 717.09 ms/ques), relation detection: 1120.81 ms/ques (similarity: 633.68 ms/ques), EDG generation: 51.14 ms/ques +[INFO] Cumulative metrics, sample: 118, P: 0.353, R: 0.436, macro F1: 0.390 +[INFO] QALD Cumulative metrics, sample: 118, P: 0.574, R: 0.436, average F1: 0.357, QALD macro F1: 0.495 +[INFO] Current question consumed time: 125 ms, average: 3193.90 ms/ques, entity detection: 714.87 ms/ques (linking tool: 639.99 ms/ques), relation detection: 1047.89 ms/ques (similarity: 633.38 ms/ques), EDG generation: 52.35 ms/ques [INFO] Entity Linking P: 0.288135593220339, R: 0.288135593220339, micro F1: 0.288135593220339, macro F1: 0.288135593220339 [INFO] Relation Linking P: 0.3050847457627119, R: 0.3050847457627119, micro F1: 0.3050847457627119, macro F1: 0.3050847457627119 [INFO] Type Linking P: 0.8728813559322034, R: 0.8728813559322034, micro F1: 0.8728813559322034, macro F1: 0.8728813559322034 @@ -1777,9 +1777,9 @@ Question 118: Is the wife of President Obama called Michelle? [INFO] Golden Answer: [true] [INFO] Predicted Answer: [false] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 119, P: 0.350, R: 0.432, macro F1: 0.354, macro F1*: 0.387 -[INFO] QALD Cumulative metrics, sample: 119, P: 0.569, R: 0.432, macro F1: 0.354, QALD macro F1: 0.491 -[INFO] Current time: 4097 ms, average: 3510.79 ms/ques, entity detection: 801.94 ms/ques (linking tool: 713.19 ms/ques), relation detection: 1121.23 ms/ques (similarity: 633.45 ms/ques), EDG generation: 50.84 ms/ques +[INFO] Cumulative metrics, sample: 119, P: 0.350, R: 0.432, macro F1: 0.387 +[INFO] QALD Cumulative metrics, sample: 119, P: 0.569, R: 0.432, average F1: 0.354, QALD macro F1: 0.491 +[INFO] Current question consumed time: 3344 ms, average: 3195.16 ms/ques, entity detection: 712.27 ms/ques (linking tool: 636.40 ms/ques), relation detection: 1047.88 ms/ques (similarity: 633.05 ms/ques), EDG generation: 51.91 ms/ques [INFO] Entity Linking P: 0.29411764705882354, R: 0.29411764705882354, micro F1: 0.29411764705882354, macro F1: 0.29411764705882354 [INFO] Relation Linking P: 0.31092436974789917, R: 0.31092436974789917, micro F1: 0.31092436974789917, macro F1: 0.31092436974789917 [INFO] Type Linking P: 0.8739495798319328, R: 0.8739495798319328, micro F1: 0.8739495798319328, macro F1: 0.8739495798319328 @@ -1792,9 +1792,9 @@ Question 119: Which U.S. state has the abbreviation MN? [INFO] Golden Answer: [http://dbpedia.org/resource/Minnesota] [INFO] Predicted Answer: [] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 120, P: 0.348, R: 0.429, macro F1: 0.351, macro F1*: 0.384 -[INFO] QALD Cumulative metrics, sample: 120, P: 0.573, R: 0.429, macro F1: 0.351, QALD macro F1: 0.490 -[INFO] Current time: 2663 ms, average: 3503.72 ms/ques, entity detection: 801.39 ms/ques (linking tool: 712.41 ms/ques), relation detection: 1112.62 ms/ques (similarity: 628.53 ms/ques), EDG generation: 50.52 ms/ques +[INFO] Cumulative metrics, sample: 120, P: 0.348, R: 0.429, macro F1: 0.384 +[INFO] QALD Cumulative metrics, sample: 120, P: 0.573, R: 0.429, average F1: 0.351, QALD macro F1: 0.490 +[INFO] Current question consumed time: 2124 ms, average: 3186.23 ms/ques, entity detection: 711.15 ms/ques (linking tool: 635.12 ms/ques), relation detection: 1039.67 ms/ques (similarity: 628.17 ms/ques), EDG generation: 51.48 ms/ques [INFO] Entity Linking P: 0.3, R: 0.3, micro F1: 0.3, macro F1: 0.3 [INFO] Relation Linking P: 0.31666666666666665, R: 0.31666666666666665, micro F1: 0.31666666666666665, macro F1: 0.31666666666666665 [INFO] Type Linking P: 0.875, R: 0.875, micro F1: 0.875, macro F1: 0.875 @@ -1807,9 +1807,9 @@ Question 120: When did Muhammad die? [INFO] Golden Answer: [0632-06-08] [INFO] Predicted Answer: [0632-06-08, 632-6-8, --06-08] [INFO] P: 0.5, R: 1.0, F: 0.6666666666666666 -[INFO] Cumulative metrics, sample: 121, P: 0.349, R: 0.433, macro F1: 0.354, macro F1*: 0.387 -[INFO] QALD Cumulative metrics, sample: 121, P: 0.572, R: 0.433, macro F1: 0.354, QALD macro F1: 0.493 -[INFO] Current time: 1563 ms, average: 3487.69 ms/ques, entity detection: 797.49 ms/ques (linking tool: 708.40 ms/ques), relation detection: 1108.03 ms/ques (similarity: 625.63 ms/ques), EDG generation: 50.14 ms/ques +[INFO] Cumulative metrics, sample: 121, P: 0.349, R: 0.433, macro F1: 0.387 +[INFO] QALD Cumulative metrics, sample: 121, P: 0.572, R: 0.433, average F1: 0.354, QALD macro F1: 0.493 +[INFO] Current question consumed time: 1298 ms, average: 3170.63 ms/ques, entity detection: 707.50 ms/ques (linking tool: 631.44 ms/ques), relation detection: 1035.08 ms/ques (similarity: 625.12 ms/ques), EDG generation: 51.18 ms/ques [INFO] Entity Linking P: 0.2975206611570248, R: 0.2975206611570248, micro F1: 0.2975206611570248, macro F1: 0.2975206611570248 [INFO] Relation Linking P: 0.3140495867768595, R: 0.3140495867768595, micro F1: 0.3140495867768595, macro F1: 0.3140495867768595 [INFO] Type Linking P: 0.8760330578512396, R: 0.8760330578512396, micro F1: 0.8760330578512396, macro F1: 0.8760330578512396 @@ -1822,9 +1822,9 @@ Question 121: What is the atmosphere of the Moon composed of? [INFO] Golden Answer: [http://dbpedia.org/resource/Argon, http://dbpedia.org/resource/Helium, http://dbpedia.org/resource/Hydrogen, http://dbpedia.org/resource/Neon, http://dbpedia.org/resource/Potassium, http://dbpedia.org/resource/Radon, http://dbpedia.org/resource/Sodium] [INFO] Predicted Answer: [yes] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 122, P: 0.346, R: 0.430, macro F1: 0.351, macro F1*: 0.383 -[INFO] QALD Cumulative metrics, sample: 122, P: 0.567, R: 0.430, macro F1: 0.351, QALD macro F1: 0.489 -[INFO] Current time: 2269 ms, average: 3477.70 ms/ques, entity detection: 795.20 ms/ques (linking tool: 705.48 ms/ques), relation detection: 1103.89 ms/ques (similarity: 622.93 ms/ques), EDG generation: 49.87 ms/ques +[INFO] Cumulative metrics, sample: 122, P: 0.346, R: 0.430, macro F1: 0.383 +[INFO] QALD Cumulative metrics, sample: 122, P: 0.567, R: 0.430, average F1: 0.351, QALD macro F1: 0.489 +[INFO] Current question consumed time: 1834 ms, average: 3159.67 ms/ques, entity detection: 705.12 ms/ques (linking tool: 628.57 ms/ques), relation detection: 1030.94 ms/ques (similarity: 622.42 ms/ques), EDG generation: 51.02 ms/ques [INFO] Entity Linking P: 0.29508196721311475, R: 0.29508196721311475, micro F1: 0.29508196721311475, macro F1: 0.29508196721311475 [INFO] Relation Linking P: 0.3114754098360656, R: 0.3114754098360656, micro F1: 0.3114754098360656, macro F1: 0.3114754098360656 [INFO] Type Linking P: 0.8770491803278688, R: 0.8770491803278688, micro F1: 0.8770491803278688, macro F1: 0.8770491803278688 @@ -1837,9 +1837,9 @@ Question 122: Who is the governor of Texas? [INFO] Golden Answer: [Greg Abbott] [INFO] Predicted Answer: [http://dbpedia.org/resource/Catholic_Church, Roman Catholic] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 123, P: 0.343, R: 0.426, macro F1: 0.348, macro F1*: 0.380 -[INFO] QALD Cumulative metrics, sample: 123, P: 0.563, R: 0.426, macro F1: 0.348, QALD macro F1: 0.485 -[INFO] Current time: 4577 ms, average: 3486.63 ms/ques, entity detection: 801.00 ms/ques (linking tool: 711.84 ms/ques), relation detection: 1103.33 ms/ques (similarity: 621.93 ms/ques), EDG generation: 49.51 ms/ques +[INFO] Cumulative metrics, sample: 123, P: 0.343, R: 0.426, macro F1: 0.380 +[INFO] QALD Cumulative metrics, sample: 123, P: 0.563, R: 0.426, average F1: 0.348, QALD macro F1: 0.485 +[INFO] Current question consumed time: 3666 ms, average: 3163.79 ms/ques, entity detection: 709.63 ms/ques (linking tool: 633.57 ms/ques), relation detection: 1029.67 ms/ques (similarity: 621.41 ms/ques), EDG generation: 50.66 ms/ques [INFO] Entity Linking P: 0.2926829268292683, R: 0.2926829268292683, micro F1: 0.2926829268292683, macro F1: 0.2926829268292683 [INFO] Relation Linking P: 0.3089430894308943, R: 0.3089430894308943, micro F1: 0.3089430894308943, macro F1: 0.3089430894308943 [INFO] Type Linking P: 0.8780487804878049, R: 0.8780487804878049, micro F1: 0.8780487804878049, macro F1: 0.8780487804878049 @@ -1852,9 +1852,9 @@ Question 123: Which movies did Kurosawa direct? [INFO] Golden Answer: [http://dbpedia.org/resource/Dersu_Uzala_(1975_film), http://dbpedia.org/resource/Dodes'ka-den, http://dbpedia.org/resource/Dreams_(1990_film), http://dbpedia.org/resource/Drunken_Angel, http://dbpedia.org/resource/High_and_Low_(1963_film), http://dbpedia.org/resource/Horse_(1941_film), http://dbpedia.org/resource/I_Live_in_Fear, http://dbpedia.org/resource/Ikiru, http://dbpedia.org/resource/Kagemusha, http://dbpedia.org/resource/Madadayo, http://dbpedia.org/resource/No_Regrets_for_Our_Youth, http://dbpedia.org/resource/One_Wonderful_Sunday, http://dbpedia.org/resource/Ran_(film), http://dbpedia.org/resource/Rashomon, http://dbpedia.org/resource/Red_Beard, http://dbpedia.org/resource/Rhapsody_in_August, http://dbpedia.org/resource/Sanjuro, http://dbpedia.org/resource/Sanshiro_Sugata, http://dbpedia.org/resource/Sanshiro_Sugata_Part_II, http://dbpedia.org/resource/Scandal_(1950_film), http://dbpedia.org/resource/Seven_Samurai, http://dbpedia.org/resource/Stray_Dog_(film), http://dbpedia.org/resource/The_Bad_Sleep_Well, http://dbpedia.org/resource/The_Hidden_Fortress, http://dbpedia.org/resource/The_Idiot_(1951_film), http://dbpedia.org/resource/The_Lower_Depths_(1957_film), http://dbpedia.org/resource/The_Men_Who_Tread_on_the_Tiger's_Tail, http://dbpedia.org/resource/The_Most_Beautiful, http://dbpedia.org/resource/The_Quiet_Duel, http://dbpedia.org/resource/Those_Who_Make_Tomorrow, http://dbpedia.org/resource/Throne_of_Blood, http://dbpedia.org/resource/Yojimbo_(film)] [INFO] Predicted Answer: [http://dbpedia.org/resource/1776_(film), http://dbpedia.org/resource/1900_(film), http://dbpedia.org/resource/2000_AD_(comics), http://dbpedia.org/resource/21_(2008_film), http://dbpedia.org/resource/61*, http://dbpedia.org/resource/A_Community_Speaks, http://dbpedia.org/resource/A_Family_Thing, http://dbpedia.org/resource/A_Hard_Day's_Night_(film), http://dbpedia.org/resource/A_Serious_Man, http://dbpedia.org/resource/A_Simple_Plan_(film), http://dbpedia.org/resource/A_Sting_in_the_Tale, http://dbpedia.org/resource/A_Very_Private_Affair, http://dbpedia.org/resource/A_View_to_a_Kill, http://dbpedia.org/resource/Aan, http://dbpedia.org/resource/Across_the_Universe_(film), http://dbpedia.org/resource/Act_of_Love_(1953_film), http://dbpedia.org/resource/Actresses_(film), http://dbpedia.org/resource/Airport_1975, http://dbpedia.org/resource/Alam_Ara, http://dbpedia.org/resource/Alice_(1990_film), http://dbpedia.org/resource/All_the_Pretty_Horses_(film), http://dbpedia.org/resource/Almost_Famous, http://dbpedia.org/resource/America's_Sweethearts, http://dbpedia.org/resource/American_Graffiti, http://dbpedia.org/resource/Amphibian_Man_(film), http://dbpedia.org/resource/An_Evening_in_Paris, http://dbpedia.org/resource/Analyze_That, http://dbpedia.org/resource/Analyze_This, http://dbpedia.org/resource/And_God_Created_Woman_(1956_film), http://dbpedia.org/resource/Angel_Heart, http://dbpedia.org/resource/Another_Woman, http://dbpedia.org/resource/Apache_(film), http://dbpedia.org/resource/Apradh, http://dbpedia.org/resource/Aqua_Teen_Hunger_Force_Colon_Movie_Film_for_Theaters, http://dbpedia.org/resource/Army_of_Darkness, http://dbpedia.org/resource/Assault_on_Dome_4, http://dbpedia.org/resource/Atlantic_City_(1980_film), http://dbpedia.org/resource/Attack_of_the_Helping_Hand, http://dbpedia.org/resource/Audition_(1999_film), http://dbpedia.org/resource/Australian_Museum, http://dbpedia.org/resource/Bad_Channels, http://dbpedia.org/resource/Bad_Santa, http://dbpedia.org/resource/Baiju_Bawra_(film), http://dbpedia.org/resource/Bandits_(2001_film), http://dbpedia.org/resource/Barfi!, http://dbpedia.org/resource/Batman:_Mask_of_the_Phantasm, http://dbpedia.org/resource/Batman_(1966_film), http://dbpedia.org/resource/Batman_(1989_film_series), http://dbpedia.org/resource/Batman_(serial), http://dbpedia.org/resource/Batman_Begins, http://dbpedia.org/resource/Batman_Forever, http://dbpedia.org/resource/Batman_Returns, http://dbpedia.org/resource/Batman_and_Robin_(serial), http://dbpedia.org/resource/Batman_v_Superman:_Dawn_of_Justice, http://dbpedia.org/resource/Beach_Party, http://dbpedia.org/resource/Bee_Movie, http://dbpedia.org/resource/Best_Friends_(1982_film), http://dbpedia.org/resource/Billions_(film), http://dbpedia.org/resource/Birdman_of_Alcatraz_(film), http://dbpedia.org/resource/Black_Death_(film), http://dbpedia.org/resource/Black_Sabbath_(film), http://dbpedia.org/resource/Black_Snake_Moan_(film), http://dbpedia.org/resource/Blackadder's_Christmas_Carol, http://dbpedia.org/resource/Blackadder:_Back_&_Forth, http://dbpedia.org/resource/Blackadder:_The_Cavalier_Years, http://dbpedia.org/resource/Blackball_(film), http://dbpedia.org/resource/Blackbeard's_Ghost, http://dbpedia.org/resource/Blackbeard_the_Pirate, http://dbpedia.org/resource/Blades_of_Glory, http://dbpedia.org/resource/Blazing_Samurai, http://dbpedia.org/resource/Blitz_(film), http://dbpedia.org/resource/Blood_Simple, http://dbpedia.org/resource/Bloody_Sunday_(film), http://dbpedia.org/resource/Blow_Out, http://dbpedia.org/resource/Blowup, http://dbpedia.org/resource/Blue_Crush, http://dbpedia.org/resource/Boadicea_(film), http://dbpedia.org/resource/Bobby_(1973_film), http://dbpedia.org/resource/Body_Double, http://dbpedia.org/resource/Borat, http://dbpedia.org/resource/Boulevard_du_Rhum, http://dbpedia.org/resource/Boys_Will_Be_Boys_(film), http://dbpedia.org/resource/Braveheart, http://dbpedia.org/resource/Brothers_of_the_Head, http://dbpedia.org/resource/Bruce_Almighty, http://dbpedia.org/resource/Brute_Force_(1947_film), http://dbpedia.org/resource/Bubba_Ho-Tep, http://dbpedia.org/resource/Buckminster_Fuller:_Thinking_Out_Loud, http://dbpedia.org/resource/Buffalo_'66, http://dbpedia.org/resource/Buffalo_Bill_and_the_Indians,_or_Sitting_Bull's_History_Lesson, http://dbpedia.org/resource/Buffy_the_Vampire_Slayer_Season_Eight, http://dbpedia.org/resource/Bug_(1975_film), http://dbpedia.org/resource/Bug_(2002_film), http://dbpedia.org/resource/Bug_(2006_film), http://dbpedia.org/resource/Bull_Durham, http://dbpedia.org/resource/Butterflies_Are_Free, http://dbpedia.org/resource/Canadian_Bacon, http://dbpedia.org/resource/Carlito's_Way, http://dbpedia.org/resource/Carmen_on_Ice, http://dbpedia.org/resource/Carrie_(1976_film), http://dbpedia.org/resource/Carry_On_(franchise), http://dbpedia.org/resource/Cars_2, http://dbpedia.org/resource/Castle_Keep, http://dbpedia.org/resource/Casualties_of_War, http://dbpedia.org/resource/Cat_Ballou, http://dbpedia.org/resource/Cat_People_(1982_film), http://dbpedia.org/resource/Cats_&_Dogs, http://dbpedia.org/resource/Cats_&_Dogs:_The_Revenge_of_Kitty_Galore, http://dbpedia.org/resource/Chak_De!_India, http://dbpedia.org/resource/Chances_Are_(film), http://dbpedia.org/resource/Chander_Pahar_(film), http://dbpedia.org/resource/Charlie's_Angels:_Full_Throttle, http://dbpedia.org/resource/Chhoti_Si_Baat, http://dbpedia.org/resource/China_Gate_(1998_film), http://dbpedia.org/resource/Chori_Chori_Chupke_Chupke, http://dbpedia.org/resource/City_Slickers, http://dbpedia.org/resource/Cleveland_Smith:_Bounty_Hunter, http://dbpedia.org/resource/Cocktail_(1988_film), http://dbpedia.org/resource/Come_Dance_with_Me_(1959_film), http://dbpedia.org/resource/Comic_Book:_The_Movie, http://dbpedia.org/resource/Congo_(film), http://dbpedia.org/resource/Contempt_(film), http://dbpedia.org/resource/Control_(1987_film), http://dbpedia.org/resource/Conversation_Piece_(film), http://dbpedia.org/resource/Crackerjack_(2002_film), http://dbpedia.org/resource/Creature_from_Black_Lake, http://dbpedia.org/resource/Crimewave, http://dbpedia.org/resource/Criss_Cross_(film), http://dbpedia.org/resource/Cruel_Doubt, http://dbpedia.org/resource/DC_Extended_Universe, http://dbpedia.org/resource/DC_Universe_Animated_Original_Movies, http://dbpedia.org/resource/Dabangg, http://dbpedia.org/resource/Daddy_and_Them, http://dbpedia.org/resource/Dark_Ascension_(film), http://dbpedia.org/resource/De_Palma_(film), http://dbpedia.org/resource/Dear_Brigitte, http://dbpedia.org/resource/Desert_Fury, http://dbpedia.org/resource/Destry_Rides_Again, http://dbpedia.org/resource/Detachment_(film), http://dbpedia.org/resource/Devdas_(2002_Hindi_film), http://dbpedia.org/resource/Dhoom, http://dbpedia.org/resource/Dhoom_2, http://dbpedia.org/resource/Die_Another_Day, http://dbpedia.org/resource/Die_Hard_2, http://dbpedia.org/resource/Dil, http://dbpedia.org/resource/Dilwale_Dulhania_Le_Jayenge, http://dbpedia.org/resource/Dionysus_in_'69, http://dbpedia.org/resource/Disco_Dancer, http://dbpedia.org/resource/Do_Bigha_Zamin, http://dbpedia.org/resource/Doctor_at_Sea_(film), http://dbpedia.org/resource/Doctor_in_the_House_(TV_series), http://dbpedia.org/resource/Don't_Knock_the_Rock, http://dbpedia.org/resource/Don't_Make_Waves, http://dbpedia.org/resource/Don_Juan,_or_If_Don_Juan_Were_a_Woman, http://dbpedia.org/resource/Dr._Kotnis_Ki_Amar_Kahani, http://dbpedia.org/resource/Dr._No_(film), http://dbpedia.org/resource/Dr._Strangelove, http://dbpedia.org/resource/Dracula_(1931_English-language_film), http://dbpedia.org/resource/Dressed_to_Kill_(1980_film), http://dbpedia.org/resource/ESPN_Major_League_Baseball, http://dbpedia.org/resource/Easy_Rider, http://dbpedia.org/resource/Eat_Pray_Love, http://dbpedia.org/resource/Eddie_Presley, http://dbpedia.org/resource/Eight_Iron_Men, http://dbpedia.org/resource/Elmer_Gantry_(film), http://dbpedia.org/resource/Endless_Harmony:_The_Beach_Boys_Story, http://dbpedia.org/resource/Enigma_(2001_film), http://dbpedia.org/resource/Eraserhead, http://dbpedia.org/resource/Escape_from_L.A., http://dbpedia.org/resource/Esther_and_the_King, http://dbpedia.org/resource/Executive_Action_(film), http://dbpedia.org/resource/Executive_Decision, http://dbpedia.org/resource/Fanalysis, http://dbpedia.org/resource/Fantasia_(1940_film), http://dbpedia.org/resource/Fargo_(film), http://dbpedia.org/resource/Farinelli_(film), http://dbpedia.org/resource/Fast_Times_at_Ridgemont_High, http://dbpedia.org/resource/Faster_(2010_film), http://dbpedia.org/resource/Femme_Fatale_(2002_film), http://dbpedia.org/resource/Field_of_Dreams, http://dbpedia.org/resource/Flash_Gordon_(film), http://dbpedia.org/resource/Forces_of_Nature, http://dbpedia.org/resource/Forget_Paris, http://dbpedia.org/resource/Frankenstein_Unbound, http://dbpedia.org/resource/Friday_Night_Lights_(film), http://dbpedia.org/resource/Friday_the_13th_(1980_film), http://dbpedia.org/resource/Frog_Dreaming, http://dbpedia.org/resource/From_Dusk_Till_Dawn_2:_Texas_Blood_Money, http://dbpedia.org/resource/From_Here_to_Eternity, http://dbpedia.org/resource/Futureworld, http://dbpedia.org/resource/Gardish, http://dbpedia.org/resource/Get_to_Know_Your_Rabbit, http://dbpedia.org/resource/Go_Tell_the_Spartans, http://dbpedia.org/resource/Going_Back_(film), http://dbpedia.org/resource/Going_My_Way, http://dbpedia.org/resource/Goldrush:_A_Real_Life_Alaskan_Adventure, http://dbpedia.org/resource/Good_Times,_Wonderful_Times, http://dbpedia.org/resource/Greetings_(1968_film), http://dbpedia.org/resource/Gunfight_at_the_O.K._Corral_(film), http://dbpedia.org/resource/Guzaarish_(2010_film), http://dbpedia.org/resource/Hairspray_(2007_film), http://dbpedia.org/resource/Halloween_(1978_film), http://dbpedia.org/resource/Harry_and_the_Hendersons, http://dbpedia.org/resource/Hearts_of_the_West_(1975_film), http://dbpedia.org/resource/Heera_Panna, http://dbpedia.org/resource/Hello_I_Must_Be_Going_(2012_film), http://dbpedia.org/resource/Henry's_Crime, http://dbpedia.org/resource/Henry_V_(1989_film), http://dbpedia.org/resource/Hera_Pheri_(2000_film), http://dbpedia.org/resource/Heyy_Babyy, http://dbpedia.org/resource/Hi,_Mom!, http://dbpedia.org/resource/Hide_in_Plain_Sight, http://dbpedia.org/resource/High_Fidelity_(film), http://dbpedia.org/resource/High_Society_(1956_film), http://dbpedia.org/resource/High_Time_(film), http://dbpedia.org/resource/His_Majesty_O'Keefe, http://dbpedia.org/resource/Histoire(s)_du_cinéma, http://dbpedia.org/resource/Hitch_(film), http://dbpedia.org/resource/Holiday_Inn_(film), http://dbpedia.org/resource/Home_Movies_(film), http://dbpedia.org/resource/Hoosiers, http://dbpedia.org/resource/How_Rare_a_Possession, http://dbpedia.org/resource/Hum_Tum, http://dbpedia.org/resource/Husbands_and_Wives, http://dbpedia.org/resource/I'll_See_You_in_My_Dreams_(2015_film), http://dbpedia.org/resource/I'm_Sorry,_I'll_Read_That_Again, http://dbpedia.org/resource/I_Build_the_Tower, http://dbpedia.org/resource/I_Surrender_Dear_(1931_film), http://dbpedia.org/resource/I_Walk_Alone, http://dbpedia.org/resource/Ice_Princess, http://dbpedia.org/resource/Icebreaker_(film), http://dbpedia.org/resource/Il_Postino:_The_Postman, http://dbpedia.org/resource/In_Case_of_Adversity, http://dbpedia.org/resource/Indecent_Proposal, http://dbpedia.org/resource/Inland_Empire_(film), http://dbpedia.org/resource/Intolerable_Cruelty, http://dbpedia.org/resource/Intruder_(1989_film), http://dbpedia.org/resource/Israel_Museum, http://dbpedia.org/resource/It's_Murder!, http://dbpedia.org/resource/Iyobinte_Pusthakam, http://dbpedia.org/resource/Jab_We_Met, http://dbpedia.org/resource/Jagged_Edge_(film), http://dbpedia.org/resource/Jayne_Mansfield's_Car, http://dbpedia.org/resource/Jim_Thorpe_–_All-American, http://dbpedia.org/resource/Jobs_(film), http://dbpedia.org/resource/Judgment_(film), http://dbpedia.org/resource/Justice_League_(film), http://dbpedia.org/resource/Kaagaz_Ke_Phool, http://dbpedia.org/resource/Kal_Ho_Naa_Ho, http://dbpedia.org/resource/Karz_(film), http://dbpedia.org/resource/Kasoor, http://dbpedia.org/resource/Khalnayak, http://dbpedia.org/resource/Khiladi_(film_series), http://dbpedia.org/resource/King:_A_Filmed_Record..._Montgomery_to_Memphis, http://dbpedia.org/resource/King_Arthur_(film), http://dbpedia.org/resource/Koi..._Mil_Gaya, http://dbpedia.org/resource/Krrish, http://dbpedia.org/resource/Kuch_Kuch_Hota_Hai, http://dbpedia.org/resource/La_Bamba_(film), http://dbpedia.org/resource/Lancelot_du_Lac_(film), http://dbpedia.org/resource/Lara_Croft_Tomb_Raider:_The_Cradle_of_Life, http://dbpedia.org/resource/Lawman_(film), http://dbpedia.org/resource/Leadbelly_(film), http://dbpedia.org/resource/Leaving_Las_Vegas, http://dbpedia.org/resource/Les_Novices, http://dbpedia.org/resource/Lethal_Weapon, http://dbpedia.org/resource/Letter_to_Brezhnev, http://dbpedia.org/resource/Leviathan_(2014_film), http://dbpedia.org/resource/Little_Fockers, http://dbpedia.org/resource/Little_Murders, http://dbpedia.org/resource/Little_Treasure, http://dbpedia.org/resource/Local_Hero, http://dbpedia.org/resource/Love_&_Mercy_(film), http://dbpedia.org/resource/Love_Actually, http://dbpedia.org/resource/Lovin'_Molly, http://dbpedia.org/resource/Lucky:_No_Time_for_Love, http://dbpedia.org/resource/Lunatics:_A_Love_Story, http://dbpedia.org/resource/MTV_Unplugged, http://dbpedia.org/resource/Mad_City_(film), http://dbpedia.org/resource/Madhumati, http://dbpedia.org/resource/Maine_Pyar_Kiya, http://dbpedia.org/resource/Malèna, http://dbpedia.org/resource/Man,_Woman_and_Child_(film), http://dbpedia.org/resource/Man_with_the_Screaming_Brain, http://dbpedia.org/resource/Maniac_Cop, http://dbpedia.org/resource/Maniac_Cop_2, http://dbpedia.org/resource/Manina,_the_Girl_in_the_Bikini, http://dbpedia.org/resource/Manna_from_Heaven_(film), http://dbpedia.org/resource/Marshall_(film), http://dbpedia.org/resource/Masked_and_Anonymous, http://dbpedia.org/resource/McHale's_Navy_(1997_film), http://dbpedia.org/resource/Meet_the_Fockers, http://dbpedia.org/resource/Meet_the_Parents, http://dbpedia.org/resource/Menno's_Mind, http://dbpedia.org/resource/Mera_Naam_Joker, http://dbpedia.org/resource/Mindwarp_(film), http://dbpedia.org/resource/Miss_Potter, http://dbpedia.org/resource/Mission:_Impossible_(film), http://dbpedia.org/resource/Mission_to_Mars, http://dbpedia.org/resource/Mister_880, http://dbpedia.org/resource/Monster's_Ball, http://dbpedia.org/resource/Monsters,_Inc., http://dbpedia.org/resource/Monsters_University, http://dbpedia.org/resource/Moontrap, http://dbpedia.org/resource/Moses_the_Lawgiver, http://dbpedia.org/resource/Mother_India, http://dbpedia.org/resource/Mr._Bean's_Holiday, http://dbpedia.org/resource/Mr._Billion, http://dbpedia.org/resource/Mr._Jones_(1993_film), http://dbpedia.org/resource/Mr._Music, http://dbpedia.org/resource/Mr._Saturday_Night, http://dbpedia.org/resource/Mr._Woodcock, http://dbpedia.org/resource/Mr._and_Mrs._Bridge, http://dbpedia.org/resource/Mughal-e-Azam, http://dbpedia.org/resource/Murder_a_la_Mod, http://dbpedia.org/resource/Murder_of_a_Cat, http://dbpedia.org/resource/Museum_of_Jewish_Heritage, http://dbpedia.org/resource/My_Cousin_Vinny, http://dbpedia.org/resource/My_Name_Is_Bruce, http://dbpedia.org/resource/Napoleon_(1995_film), http://dbpedia.org/resource/National_Treasure_(franchise), http://dbpedia.org/resource/Neecha_Nagar, http://dbpedia.org/resource/Neptune's_Daughter_(1949_film), http://dbpedia.org/resource/Nero's_Weekend, http://dbpedia.org/resource/Newsroom_South_East, http://dbpedia.org/resource/Night_Mail, http://dbpedia.org/resource/Nosferatu, http://dbpedia.org/resource/Nowhere_to_Run_(1978_film), http://dbpedia.org/resource/Obsession_(1976_film), http://dbpedia.org/resource/Oldboy_(2003_film), http://dbpedia.org/resource/Oldest_Living_Confederate_Widow_Tells_All_(film), http://dbpedia.org/resource/One_Million_Years_B.C., http://dbpedia.org/resource/One_Night_with_the_King, http://dbpedia.org/resource/Operation_Crossroads, http://dbpedia.org/resource/Oz_the_Great_and_Powerful, http://dbpedia.org/resource/Padmashree_Laloo_Prasad_Yadav, http://dbpedia.org/resource/Parental_Guidance_(film), http://dbpedia.org/resource/Parting_Glances, http://dbpedia.org/resource/Partner_(2007_film), http://dbpedia.org/resource/Passage_to_Zarahemla, http://dbpedia.org/resource/Passion_(2012_film), http://dbpedia.org/resource/Paul_(film), http://dbpedia.org/resource/Pearl_Harbor_(film), http://dbpedia.org/resource/Pennies_from_Heaven_(1936_film), http://dbpedia.org/resource/Phantom_of_the_Paradise, http://dbpedia.org/resource/Pirates_of_the_Caribbean:_The_Curse_of_the_Black_Pearl, http://dbpedia.org/resource/Please,_Not_Now!, http://dbpedia.org/resource/Pride_(2014_film), http://dbpedia.org/resource/Primary_Colors_(film), http://dbpedia.org/resource/Prison_Ball, http://dbpedia.org/resource/Promised_Land_(2012_film), http://dbpedia.org/resource/Pulp_Fiction, http://dbpedia.org/resource/Pumping_Iron_II:_The_Women, http://dbpedia.org/resource/Pushing_Tin, http://dbpedia.org/resource/Pyaasa, http://dbpedia.org/resource/Qayamat_Se_Qayamat_Tak, http://dbpedia.org/resource/Ra.One, http://dbpedia.org/resource/Raaz_(2002_film), http://dbpedia.org/resource/Rabbit_Test_(film), http://dbpedia.org/resource/Raging_Bull, http://dbpedia.org/resource/Raising_Cain, http://dbpedia.org/resource/Raja_Harishchandra, http://dbpedia.org/resource/Rang_De_Basanti, http://dbpedia.org/resource/Redacted_(film), http://dbpedia.org/resource/Revolution_OS, http://dbpedia.org/resource/Road_to_Bali, http://dbpedia.org/resource/Road_to_Morocco, http://dbpedia.org/resource/Road_to_Rio, http://dbpedia.org/resource/Road_to_Singapore, http://dbpedia.org/resource/Road_to_Utopia, http://dbpedia.org/resource/Road_to_Zanzibar, http://dbpedia.org/resource/Rock_Around_the_Clock_(film), http://dbpedia.org/resource/Rocket_Gibraltar, http://dbpedia.org/resource/Ronnie_Rocket, http://dbpedia.org/resource/Rope_of_Sand, http://dbpedia.org/resource/Rowdy_Rathore, http://dbpedia.org/resource/Royal_Affairs_in_Versailles, http://dbpedia.org/resource/Run_Silent,_Run_Deep_(1958_film), http://dbpedia.org/resource/Running_Scared_(1986_film), http://dbpedia.org/resource/Running_Time_(film), http://dbpedia.org/resource/Rushmore_(film), http://dbpedia.org/resource/Salaam_Bombay!, http://dbpedia.org/resource/Salaam_Namaste, http://dbpedia.org/resource/Sasquatch,_the_Legend_of_Bigfoot, http://dbpedia.org/resource/Satya_(film), http://dbpedia.org/resource/Scarface_(1983_film), http://dbpedia.org/resource/School_for_Scoundrels_(2006_film), http://dbpedia.org/resource/Scorpio_(film), http://dbpedia.org/resource/Sense_and_Sensibility_(film), http://dbpedia.org/resource/Separate_Tables_(film), http://dbpedia.org/resource/Separate_but_Equal_(film), http://dbpedia.org/resource/Serial_Experiments_Lain, http://dbpedia.org/resource/Serving_Sara, http://dbpedia.org/resource/Shalako_(film), http://dbpedia.org/resource/Sharknado_2:_The_Second_One, http://dbpedia.org/resource/Shree_420, http://dbpedia.org/resource/Silent_Rage, http://dbpedia.org/resource/Singh_Is_Kinng, http://dbpedia.org/resource/Sisters_(1973_film), http://dbpedia.org/resource/Sky_High_(2005_film), http://dbpedia.org/resource/Sling_Blade, http://dbpedia.org/resource/Slumdog_Millionaire, http://dbpedia.org/resource/Snake_Eyes_(film), http://dbpedia.org/resource/Soap_(TV_series), http://dbpedia.org/resource/Some_Folks_Call_It_a_Sling_Blade, http://dbpedia.org/resource/Sorry,_Wrong_Number, http://dbpedia.org/resource/Souten, http://dbpedia.org/resource/South_Park:_Bigger,_Longer_&_Uncut, http://dbpedia.org/resource/South_Sea_Woman, http://dbpedia.org/resource/Spider-Man_(2002_film), http://dbpedia.org/resource/Spider-Man_2, http://dbpedia.org/resource/Spider-Man_3, http://dbpedia.org/resource/Spider-Plant_Man, http://dbpedia.org/resource/Spirits_of_the_Dead, http://dbpedia.org/resource/Still_Smokin_(film), http://dbpedia.org/resource/Storytelling_(film), http://dbpedia.org/resource/Stripped_(film), http://dbpedia.org/resource/Suicide_Squad_(film), http://dbpedia.org/resource/Sunday_Bloody_Sunday_(film), http://dbpedia.org/resource/Sundown:_The_Vampire_in_Retreat, http://dbpedia.org/resource/Sweet_Smell_of_Success, http://dbpedia.org/resource/Sylvia_(2003_film), http://dbpedia.org/resource/Take_a_Giant_Step, http://dbpedia.org/resource/Teenage_Mutant_Ninja_Turtles_(2014_film), http://dbpedia.org/resource/Ten_Tall_Men, http://dbpedia.org/resource/Terminal_Invasion, http://dbpedia.org/resource/Testament_of_Orpheus, http://dbpedia.org/resource/The_Adventures_of_Ichabod_and_Mr._Toad, http://dbpedia.org/resource/The_Astronaut_Farmer, http://dbpedia.org/resource/The_Babe_Ruth_Story, http://dbpedia.org/resource/The_Bachelor_Party, http://dbpedia.org/resource/The_Band_(film), http://dbpedia.org/resource/The_Beach_Boys:_An_American_Band, http://dbpedia.org/resource/The_Beach_Boys:_An_American_Family, http://dbpedia.org/resource/The_Beach_Boys:_Good_Vibrations_Tour, http://dbpedia.org/resource/The_Beach_Boys:_Nashville_Sounds, http://dbpedia.org/resource/The_Bear_and_the_Doll, http://dbpedia.org/resource/The_Bells_of_St._Mary's, http://dbpedia.org/resource/The_Big_Broadcast, http://dbpedia.org/resource/The_Big_Country, http://dbpedia.org/resource/The_Big_Sleep_(1946_film), http://dbpedia.org/resource/The_Black_Adder_(pilot_episode), http://dbpedia.org/resource/The_Black_Dahlia_(film), http://dbpedia.org/resource/The_Blind_Side_(film), http://dbpedia.org/resource/The_Blot, http://dbpedia.org/resource/The_Bonfire_of_the_Vanities_(film), http://dbpedia.org/resource/The_Book_of_Esther_(film), http://dbpedia.org/resource/The_Bride_Is_Much_Too_Beautiful, http://dbpedia.org/resource/The_Buddy_Holly_Story, http://dbpedia.org/resource/The_Cohens_and_Kellys_in_Hollywood, http://dbpedia.org/resource/The_Conversation, http://dbpedia.org/resource/The_Crimson_Pirate, http://dbpedia.org/resource/The_Dark_Knight_Rises, http://dbpedia.org/resource/The_Dark_Knight_Trilogy, http://dbpedia.org/resource/The_Dead_Next_Door, http://dbpedia.org/resource/The_Devil's_Disciple_(1959_film), http://dbpedia.org/resource/The_Diamond_Arm, http://dbpedia.org/resource/The_Duellists, http://dbpedia.org/resource/The_Edifying_and_Joyous_Story_of_Colinot, http://dbpedia.org/resource/The_Escort_(2015_film), http://dbpedia.org/resource/The_Flame_and_the_Arrow, http://dbpedia.org/resource/The_Frighteners, http://dbpedia.org/resource/The_Fury_(1978_film), http://dbpedia.org/resource/The_Gift_(2000_film), http://dbpedia.org/resource/The_Girls_on_the_Beach, http://dbpedia.org/resource/The_Goon_Show, http://dbpedia.org/resource/The_Grand_Maneuver, http://dbpedia.org/resource/The_Great_Santini, http://dbpedia.org/resource/The_Guru_(2002_film), http://dbpedia.org/resource/The_Gypsy_Moths, http://dbpedia.org/resource/The_Happy_Thieves, http://dbpedia.org/resource/The_Hudsucker_Proxy, http://dbpedia.org/resource/The_Island_of_Dr._Moreau_(1977_film), http://dbpedia.org/resource/The_Jerk, http://dbpedia.org/resource/The_Kentuckian, http://dbpedia.org/resource/The_Killers_(1946_film), http://dbpedia.org/resource/The_Ladykillers_(2004_film), http://dbpedia.org/resource/The_Last_Kiss_(2006_film), http://dbpedia.org/resource/The_Legend_of_Boggy_Creek, http://dbpedia.org/resource/The_Legend_of_Frenchie_King, http://dbpedia.org/resource/The_Lego_Batman_Movie, http://dbpedia.org/resource/The_Lego_Movie, http://dbpedia.org/resource/The_Leopard_(1963_film), http://dbpedia.org/resource/The_Life_of_Nephi, http://dbpedia.org/resource/The_Lightkeepers, http://dbpedia.org/resource/The_List_of_Adrian_Messenger, http://dbpedia.org/resource/The_Long_Teeth, http://dbpedia.org/resource/The_Love_Bug_(1997_film), http://dbpedia.org/resource/The_Love_Letter_(1999_film), http://dbpedia.org/resource/The_Lucky_One_(film), http://dbpedia.org/resource/The_Majestic_(film), http://dbpedia.org/resource/The_Man_Who_Wasn't_There_(2001_film), http://dbpedia.org/resource/The_Mark_of_Zorro_(1920_film), http://dbpedia.org/resource/The_Midnight_Man_(1974_film), http://dbpedia.org/resource/The_Monkey's_Uncle, http://dbpedia.org/resource/The_Monster_Squad, http://dbpedia.org/resource/The_Mosquito_Coast, http://dbpedia.org/resource/The_Myth_of_Fingerprints, http://dbpedia.org/resource/The_Night_Heaven_Fell, http://dbpedia.org/resource/The_Night_of_the_Hunter_(film), http://dbpedia.org/resource/The_Nutt_House_(film), http://dbpedia.org/resource/The_Outlaw, http://dbpedia.org/resource/The_Pacific_(miniseries), http://dbpedia.org/resource/The_Panic_in_Needle_Park, http://dbpedia.org/resource/The_Perils_of_Gwendoline_in_the_Land_of_the_Yik-Yak, http://dbpedia.org/resource/The_Plank_(1967_film), http://dbpedia.org/resource/The_Producers_(2005_film), http://dbpedia.org/resource/The_Professionals_(1966_film), http://dbpedia.org/resource/The_Proposition_(1998_film), http://dbpedia.org/resource/The_Queen_(film), http://dbpedia.org/resource/The_Quick_and_the_Dead_(1995_film), http://dbpedia.org/resource/The_Reincarnation_of_Peter_Proud, http://dbpedia.org/resource/The_Road_to_Hong_Kong, http://dbpedia.org/resource/The_Rolling_Stones_Rock_and_Roll_Circus, http://dbpedia.org/resource/The_Royal_Tenenbaums, http://dbpedia.org/resource/The_Savages_(film), http://dbpedia.org/resource/The_Scalphunters, http://dbpedia.org/resource/The_Singing_Ringing_Tree, http://dbpedia.org/resource/The_Sisterhood_of_the_Traveling_Pants_2, http://dbpedia.org/resource/The_Skin, http://dbpedia.org/resource/The_Swimmer_(1968_film), http://dbpedia.org/resource/The_Testaments_of_One_Fold_and_One_Shepherd, http://dbpedia.org/resource/The_Train_(1964_film), http://dbpedia.org/resource/The_Truman_Show, http://dbpedia.org/resource/The_Unforgiven_(1960_film), http://dbpedia.org/resource/The_Usual_Suspects, http://dbpedia.org/resource/The_Viking_Queen, http://dbpedia.org/resource/The_Wedding_Party_(film), http://dbpedia.org/resource/The_Wild_One, http://dbpedia.org/resource/The_Woods_(2006_film), http://dbpedia.org/resource/The_World_Is_Not_Enough, http://dbpedia.org/resource/The_X-Files_(film), http://dbpedia.org/resource/The_Young_Savages, http://dbpedia.org/resource/This_Is_Spinal_Tap, http://dbpedia.org/resource/Thou_Shalt_Not_Kill..._Except, http://dbpedia.org/resource/Three_Sailors_and_a_Girl, http://dbpedia.org/resource/Timequest_(film), http://dbpedia.org/resource/To_Kill_a_Clown, http://dbpedia.org/resource/To_Wong_Foo,_Thanks_for_Everything!_Julie_Newmar, http://dbpedia.org/resource/Tombstone_(film), http://dbpedia.org/resource/Tornado!, http://dbpedia.org/resource/Tough_Guys, http://dbpedia.org/resource/Toy_Story_2, http://dbpedia.org/resource/Tracey_Ullman_Takes_On_New_York, http://dbpedia.org/resource/Transformers:_Age_of_Extinction, http://dbpedia.org/resource/Trapeze_(film), http://dbpedia.org/resource/Tuck_Everlasting_(1981_film), http://dbpedia.org/resource/Tumbledown_(2015_film), http://dbpedia.org/resource/Twilight's_Last_Gleaming, http://dbpedia.org/resource/Two_Weeks_in_September, http://dbpedia.org/resource/U_Turn_(1997_film), http://dbpedia.org/resource/Ulzana's_Raid, http://dbpedia.org/resource/Un_chant_d'amour, http://dbpedia.org/resource/Underdog_(film), http://dbpedia.org/resource/Une_ravissante_idiote, http://dbpedia.org/resource/Valdez_Is_Coming, http://dbpedia.org/resource/Veer-Zaara, http://dbpedia.org/resource/Vengeance_Valley, http://dbpedia.org/resource/Vera_Cruz_(film), http://dbpedia.org/resource/Viva_Maria!, http://dbpedia.org/resource/Waiting_for_Forever, http://dbpedia.org/resource/Warriors_of_Heaven_and_Earth, http://dbpedia.org/resource/Waterloo_(1970_film), http://dbpedia.org/resource/Waxwork_II:_Lost_in_Time, http://dbpedia.org/resource/Wayne's_World_(film), http://dbpedia.org/resource/Wedding_Crashers, http://dbpedia.org/resource/What's_Your_Number%3F, http://dbpedia.org/resource/What_Lies_Beneath, http://dbpedia.org/resource/When_Harry_Met_Sally..., http://dbpedia.org/resource/White_Christmas_(film), http://dbpedia.org/resource/White_on_Rice, http://dbpedia.org/resource/Why_Has_Bodhi-Dharma_Left_for_the_East%3F, http://dbpedia.org/resource/Wings_of_Desire, http://dbpedia.org/resource/Winter_Kills_(film), http://dbpedia.org/resource/Wise_Guys_(1986_film), http://dbpedia.org/resource/Within_the_Woods, http://dbpedia.org/resource/Wrath_of_the_Seas, http://dbpedia.org/resource/XXX_(2002_film), http://dbpedia.org/resource/Zinda_(film), http://dbpedia.org/resource/Zulu_Dawn, http://dbpedia.org/resource/24:_Conspiracy, http://dbpedia.org/resource/A.K._(film), http://dbpedia.org/resource/A_Computer_Animated_Hand, http://dbpedia.org/resource/A_Few_Good_Men, http://dbpedia.org/resource/A_Grin_Without_a_Cat, http://dbpedia.org/resource/A_Mighty_Wind, http://dbpedia.org/resource/A_Mouse_Tale, http://dbpedia.org/resource/A_Night_in_Casablanca, http://dbpedia.org/resource/Abbott_and_Costello_Meet_Captain_Kidd, http://dbpedia.org/resource/Absolution_(1978_film), http://dbpedia.org/resource/Agora_(film), http://dbpedia.org/resource/All_Cheerleaders_Die, http://dbpedia.org/resource/All_Cheerleaders_Die_(2001_film), http://dbpedia.org/resource/All_Power_to_the_People, http://dbpedia.org/resource/Almost_Heroes, http://dbpedia.org/resource/Alpha_and_Omega_(film), http://dbpedia.org/resource/American_Crime_(film), http://dbpedia.org/resource/American_Perfekt, http://dbpedia.org/resource/American_Playhouse, http://dbpedia.org/resource/American_Strays, http://dbpedia.org/resource/Another_Country_(film), http://dbpedia.org/resource/Arthur_Christmas, http://dbpedia.org/resource/Attack_of_the_50_Ft._Woman_(1993_film), http://dbpedia.org/resource/Ava's_Possessions, http://dbpedia.org/resource/Avalon_(2001_film), http://dbpedia.org/resource/Back_to_the_Future_Part_II, http://dbpedia.org/resource/Barbershop_(film), http://dbpedia.org/resource/Barney_Bear, http://dbpedia.org/resource/Barnyard_(film), http://dbpedia.org/resource/Beer_Barrel_Polka, http://dbpedia.org/resource/Behaving_Badly_(film), http://dbpedia.org/resource/Being_Charlie, http://dbpedia.org/resource/Best_in_Show_(film), http://dbpedia.org/resource/Better_Things_(film), http://dbpedia.org/resource/Beyond_Therapy_(film), http://dbpedia.org/resource/Big_Bully_(film), http://dbpedia.org/resource/Big_Hero_6_(film), http://dbpedia.org/resource/Blast_from_the_Past_(film), http://dbpedia.org/resource/Blood_and_Roses, http://dbpedia.org/resource/Blown_Away_(1994_film), http://dbpedia.org/resource/Bolt_(2008_film), http://dbpedia.org/resource/Bombay_Calling, http://dbpedia.org/resource/Bordello_of_Blood, http://dbpedia.org/resource/Box_Elder_(film), http://dbpedia.org/resource/Brave_(2012_film), http://dbpedia.org/resource/Brick_(film), http://dbpedia.org/resource/Bring_It_On:_All_or_Nothing, http://dbpedia.org/resource/Bring_It_On:_Fight_to_the_Finish, http://dbpedia.org/resource/Bring_It_On:_In_It_to_Win_It, http://dbpedia.org/resource/Bring_It_On_(film), http://dbpedia.org/resource/Bring_It_On_Again, http://dbpedia.org/resource/C.I.D._(1955_film), http://dbpedia.org/resource/C.I.D._(1956_film), http://dbpedia.org/resource/C.I.D._(1990_film), http://dbpedia.org/resource/Can_She_Bake_a_Cherry_Pie%3F, http://dbpedia.org/resource/Candide_ou_l'optimisme_au_XXe_siècle, http://dbpedia.org/resource/Candy_(1968_film), http://dbpedia.org/resource/Captain_Kidd_and_the_Slave_Girl, http://dbpedia.org/resource/Carnal_Knowledge, http://dbpedia.org/resource/Charlie_and_the_Chocolate_Factory_(film), http://dbpedia.org/resource/Chicken_Little_(2005_film), http://dbpedia.org/resource/Child's_Play_(1988_film), http://dbpedia.org/resource/Chimera_(1968_film), http://dbpedia.org/resource/Chimera_(2001_film), http://dbpedia.org/resource/Chimère_(film), http://dbpedia.org/resource/Chimères_(film), http://dbpedia.org/resource/Cleopatra_(1912_film), http://dbpedia.org/resource/Cleopatra_(1917_film), http://dbpedia.org/resource/Cleopatra_(1970_film), http://dbpedia.org/resource/Cleopatra_(2003_film), http://dbpedia.org/resource/Cleopatra_(2013_film), http://dbpedia.org/resource/Cleopatra_Jones, http://dbpedia.org/resource/Cockneys_vs_Zombies, http://dbpedia.org/resource/Code_Geass, http://dbpedia.org/resource/Collision_Course_(1989_film), http://dbpedia.org/resource/Comic_Book_Villains, http://dbpedia.org/resource/Conan_the_Barbarian_(2011_film), http://dbpedia.org/resource/Confessions_of_a_Teenage_Drama_Queen, http://dbpedia.org/resource/Conspiracy_(1930_film), http://dbpedia.org/resource/Conspiracy_(1939_film), http://dbpedia.org/resource/Conspiracy_(2001_film), http://dbpedia.org/resource/Conspiracy_(2008_film), http://dbpedia.org/resource/Constantine_(film), http://dbpedia.org/resource/Coronation_Street:_Out_of_Africa, http://dbpedia.org/resource/Coronation_Street:_Romanian_Holiday, http://dbpedia.org/resource/Cosmopolitan_(film), http://dbpedia.org/resource/Cowboy_Bebop:_The_Movie, http://dbpedia.org/resource/Cradle_Will_Rock, http://dbpedia.org/resource/Crank:_High_Voltage, http://dbpedia.org/resource/Crank_(film), http://dbpedia.org/resource/Creation_(2009_film), http://dbpedia.org/resource/Cuba_(film), http://dbpedia.org/resource/Cyborg_(film), http://dbpedia.org/resource/Cyborg_2, http://dbpedia.org/resource/Cyborg_3:_The_Recycler, http://dbpedia.org/resource/Cyborg_Soldier, http://dbpedia.org/resource/Dark_Tide, http://dbpedia.org/resource/David's_Mother, http://dbpedia.org/resource/Days_of_Thunder, http://dbpedia.org/resource/Dead_Season, http://dbpedia.org/resource/Death_Wish_(film), http://dbpedia.org/resource/Debbie_Does_Dallas, http://dbpedia.org/resource/Decline_of_an_Empire, http://dbpedia.org/resource/Delhi_Safari, http://dbpedia.org/resource/Design_Museum, http://dbpedia.org/resource/Desperate_Characters, http://dbpedia.org/resource/Despicable_Me, http://dbpedia.org/resource/Dinosaur_(film), http://dbpedia.org/resource/Divergent_(film), http://dbpedia.org/resource/Dog_Day_Afternoon, http://dbpedia.org/resource/Edie_&_Pen, http://dbpedia.org/resource/Edison_(film), http://dbpedia.org/resource/Ella_Enchanted_(film), http://dbpedia.org/resource/Elvis_&_Nixon, http://dbpedia.org/resource/Epic_(2013_film), http://dbpedia.org/resource/Escape_from_Planet_Earth, http://dbpedia.org/resource/Even_Cowgirls_Get_the_Blues_(film), http://dbpedia.org/resource/FairyTale:_A_True_Story, http://dbpedia.org/resource/Far_from_Vietnam, http://dbpedia.org/resource/Fatal_Attraction, http://dbpedia.org/resource/FernGully:_The_Last_Rainforest, http://dbpedia.org/resource/Ferris_Bueller's_Day_Off, http://dbpedia.org/resource/Final_Fantasy:_The_Spirits_Within, http://dbpedia.org/resource/Final_Offer, http://dbpedia.org/resource/Fired_Up!, http://dbpedia.org/resource/Flesh_Gordon_Meets_the_Cosmic_Cheerleaders, http://dbpedia.org/resource/Flushed_Away, http://dbpedia.org/resource/For_Your_Consideration_(film), http://dbpedia.org/resource/Formosa_Betrayed_(film), http://dbpedia.org/resource/Forty_Thousand_Horsemen, http://dbpedia.org/resource/Four_Christmases, http://dbpedia.org/resource/Freaky_Friday_(1995_film), http://dbpedia.org/resource/Free_Birds, http://dbpedia.org/resource/Fright_Night, http://dbpedia.org/resource/Fright_Night_(2011_film), http://dbpedia.org/resource/Frozen_(2013_film), http://dbpedia.org/resource/Georgia_Rule, http://dbpedia.org/resource/Get_Carter, http://dbpedia.org/resource/Ghostbusters_(2016_film), http://dbpedia.org/resource/Gimme_an_'F', http://dbpedia.org/resource/Girlfriends_(1978_film), http://dbpedia.org/resource/Gone_Fishin'_(film), http://dbpedia.org/resource/Goosebumps_(film), http://dbpedia.org/resource/Guardians_of_the_Galaxy_(film), http://dbpedia.org/resource/H8RZ, http://dbpedia.org/resource/Happy_Feet, http://dbpedia.org/resource/Harry_and_Walter_Go_to_New_York, http://dbpedia.org/resource/Heartbeeps, http://dbpedia.org/resource/Hellgate_(2011_film), http://dbpedia.org/resource/Henry_VIII_and_His_Six_Wives, http://dbpedia.org/resource/Hester_Street_(film), http://dbpedia.org/resource/High_School_Musical, http://dbpedia.org/resource/Hold_That_Ghost, http://dbpedia.org/resource/Home_(2015_film), http://dbpedia.org/resource/Hop_(film), http://dbpedia.org/resource/Horton_Hears_a_Who!_(film), http://dbpedia.org/resource/Hot_Shots!, http://dbpedia.org/resource/Hot_Tub_Time_Machine, http://dbpedia.org/resource/Hotel_Transylvania, http://dbpedia.org/resource/Hubal_(film), http://dbpedia.org/resource/I_Love_You,_Beth_Cooper_(film), http://dbpedia.org/resource/I_Smile_Back, http://dbpedia.org/resource/Ice_Age:_Dawn_of_the_Dinosaurs, http://dbpedia.org/resource/If_Looks_Could_Kill_(film), http://dbpedia.org/resource/In_the_Soup, http://dbpedia.org/resource/Indiscretion_(film), http://dbpedia.org/resource/Inside_Out_(2015_film), http://dbpedia.org/resource/Inspector_Gadget_(film), http://dbpedia.org/resource/Ishtar_(film), http://dbpedia.org/resource/It's_All_Gone_Pete_Tong, http://dbpedia.org/resource/Jason_X, http://dbpedia.org/resource/Jawbreaker_(film), http://dbpedia.org/resource/Jennifer's_Body, http://dbpedia.org/resource/Joe_Versus_the_Volcano, http://dbpedia.org/resource/Joe_the_King, http://dbpedia.org/resource/Judge_Dredd_(film), http://dbpedia.org/resource/Jumpin'_Jack_Flash_(film), http://dbpedia.org/resource/Just_Cause_(film), http://dbpedia.org/resource/Justice_League:_The_Flashpoint_Paradox, http://dbpedia.org/resource/K-19:_The_Widowmaker, http://dbpedia.org/resource/Kenny_(1988_film), http://dbpedia.org/resource/Kicking_&_Screaming_(2005_film), http://dbpedia.org/resource/Kull_the_Conqueror, http://dbpedia.org/resource/Kung_Fu_Panda, http://dbpedia.org/resource/La_Jetée, http://dbpedia.org/resource/La_reina_de_España, http://dbpedia.org/resource/Lady_Jane_(1986_film), http://dbpedia.org/resource/Le_Joli_Mai, http://dbpedia.org/resource/Leather_Jackets_(film), http://dbpedia.org/resource/Legend_of_the_Guardians:_The_Owls_of_Ga'Hoole, http://dbpedia.org/resource/Lesbian_Vampire_Killers, http://dbpedia.org/resource/Let_the_Devil_Wear_Black, http://dbpedia.org/resource/Level_Five_(film), http://dbpedia.org/resource/Liar_Liar, http://dbpedia.org/resource/License_to_Drive, http://dbpedia.org/resource/Lipstick_(film), http://dbpedia.org/resource/Lock,_Stock_and_Two_Smoking_Barrels, http://dbpedia.org/resource/Loggerheads_(film), http://dbpedia.org/resource/Love_Don't_Cost_a_Thing_(film), http://dbpedia.org/resource/Love_in_the_Time_of_Money, http://dbpedia.org/resource/Macross_Plus, http://dbpedia.org/resource/Mad_Max_Beyond_Thunderdome, http://dbpedia.org/resource/Madagascar_(2005_film), http://dbpedia.org/resource/Man_of_the_House_(2005_film), http://dbpedia.org/resource/Margin_Call_(film), http://dbpedia.org/resource/Mars_Needs_Moms, http://dbpedia.org/resource/Maschenka, http://dbpedia.org/resource/Mascots_(2016_film), http://dbpedia.org/resource/Matinee_(1993_film), http://dbpedia.org/resource/Meet_the_Robinsons, http://dbpedia.org/resource/Megamind, http://dbpedia.org/resource/Men_with_Brooms, http://dbpedia.org/resource/Mickey's_Twice_Upon_a_Christmas, http://dbpedia.org/resource/Miles_of_Fire, http://dbpedia.org/resource/Minions_(film), http://dbpedia.org/resource/Monkey_Drummer, http://dbpedia.org/resource/Monsters_vs._Aliens, http://dbpedia.org/resource/Monty_Python_and_the_Holy_Grail, http://dbpedia.org/resource/Moon_(film), http://dbpedia.org/resource/Mr._Lucky_(film), http://dbpedia.org/resource/Mr._Peabody_&_Sherman, http://dbpedia.org/resource/Mrs_Henderson_Presents, http://dbpedia.org/resource/Multiple_Sarcasms, http://dbpedia.org/resource/Muppets_Most_Wanted, http://dbpedia.org/resource/My_Blue_Heaven_(1990_film), http://dbpedia.org/resource/My_Girlfriend's_Boyfriend_(2010_film), http://dbpedia.org/resource/My_Sassy_Girl_(2008_film), http://dbpedia.org/resource/Naked_Lunch_(film), http://dbpedia.org/resource/Nalini_by_Day,_Nancy_by_Night, http://dbpedia.org/resource/Nanook_of_the_North, http://dbpedia.org/resource/National_Lampoon's_Pucked, http://dbpedia.org/resource/National_Lampoon's_Vacation_(film_series), http://dbpedia.org/resource/Nausicaä_of_the_Valley_of_the_Wind_(film), http://dbpedia.org/resource/Neo_Ned, http://dbpedia.org/resource/Never_Again_(2001_film), http://dbpedia.org/resource/Never_Let_Me_Go_(2010_film), http://dbpedia.org/resource/Never_on_Tuesday, http://dbpedia.org/resource/New_Year's_Eve_(film), http://dbpedia.org/resource/Nightbreed, http://dbpedia.org/resource/Nightmare_Classics, http://dbpedia.org/resource/No_Greater_Love_(1996_film), http://dbpedia.org/resource/No_Maps_for_These_Territories, http://dbpedia.org/resource/No_Strings_Attached_(film), http://dbpedia.org/resource/Norman_Loves_Rose, http://dbpedia.org/resource/Octopussy, http://dbpedia.org/resource/Office_Killer, http://dbpedia.org/resource/On_Her_Majesty's_Secret_Service_(film), http://dbpedia.org/resource/One,_Two,_Three, http://dbpedia.org/resource/Open_Season_(2006_film), http://dbpedia.org/resource/Outsourced_(film), http://dbpedia.org/resource/Over_the_Brooklyn_Bridge, http://dbpedia.org/resource/Over_the_Hedge_(film), http://dbpedia.org/resource/Oxford_Blues, http://dbpedia.org/resource/Pandemonium_(film), http://dbpedia.org/resource/Penn_&_Teller_Get_Killed, http://dbpedia.org/resource/Perfume_(2001_film), http://dbpedia.org/resource/Pete_Smalls_Is_Dead, http://dbpedia.org/resource/Photographing_Fairies, http://dbpedia.org/resource/Pirates_of_the_Caribbean_(film_series), http://dbpedia.org/resource/Pixels_(2015_film), http://dbpedia.org/resource/Porco_Rosso, http://dbpedia.org/resource/Prizzi's_Honor, http://dbpedia.org/resource/Protocol_(film), http://dbpedia.org/resource/Puss_in_Boots_(2011_film), http://dbpedia.org/resource/Quest_for_Camelot, http://dbpedia.org/resource/Racing_with_the_Moon, http://dbpedia.org/resource/Ramrod_(film), http://dbpedia.org/resource/Rancho_Notorious, http://dbpedia.org/resource/Rango_(2011_film), http://dbpedia.org/resource/Ratatouille_(film), http://dbpedia.org/resource/Reach_Me, http://dbpedia.org/resource/Reality_Bites, http://dbpedia.org/resource/Repo!_The_Genetic_Opera, http://dbpedia.org/resource/Rio_(2011_film), http://dbpedia.org/resource/Rise_of_the_Guardians, http://dbpedia.org/resource/Rise_of_the_Planet_of_the_Apes, http://dbpedia.org/resource/Robin_Hood:_Men_in_Tights, http://dbpedia.org/resource/Robots_(2005_film), http://dbpedia.org/resource/Rocky, http://dbpedia.org/resource/Rollerball_(1975_film), http://dbpedia.org/resource/Rubber_Johnny, http://dbpedia.org/resource/Safe_(2012_film), http://dbpedia.org/resource/Sanders_of_the_River, http://dbpedia.org/resource/Sans_Soleil, http://dbpedia.org/resource/Satan's_Cheerleaders, http://dbpedia.org/resource/Sausage_Party, http://dbpedia.org/resource/Saw_(2004_film), http://dbpedia.org/resource/Saw_3D, http://dbpedia.org/resource/Saw_II, http://dbpedia.org/resource/Scrooged, http://dbpedia.org/resource/Sexy_Beast, http://dbpedia.org/resource/Shadow_of_the_Vampire, http://dbpedia.org/resource/Shark_Tale, http://dbpedia.org/resource/Short_Circuit_2, http://dbpedia.org/resource/Shrek_2, http://dbpedia.org/resource/Sinister_2, http://dbpedia.org/resource/Sixteen_Candles, http://dbpedia.org/resource/Slaves_of_New_York, http://dbpedia.org/resource/Sleepwalk_with_Me, http://dbpedia.org/resource/Small_Soldiers, http://dbpedia.org/resource/Smoky_(1946_film), http://dbpedia.org/resource/Snatch_(film), http://dbpedia.org/resource/SpaceCamp, http://dbpedia.org/resource/Star_Trek:_First_Contact, http://dbpedia.org/resource/Star_Wars:_Episode_II_–_Attack_of_the_Clones, http://dbpedia.org/resource/Star_Wars:_Episode_I_–_The_Phantom_Menace, http://dbpedia.org/resource/Statues_Also_Die, http://dbpedia.org/resource/Steamboy, http://dbpedia.org/resource/Sticky_Fingers_(1988_film), http://dbpedia.org/resource/Storks_(film), http://dbpedia.org/resource/Strange_Magic_(film), http://dbpedia.org/resource/Sugar_&_Spice, http://dbpedia.org/resource/Sugar_Mountain_(film), http://dbpedia.org/resource/Sunset_Park_(film), http://dbpedia.org/resource/Surf's_Up_(film), http://dbpedia.org/resource/Tangled, http://dbpedia.org/resource/Tarzoon:_Shame_of_the_Jungle, http://dbpedia.org/resource/Ted_&_Venus, http://dbpedia.org/resource/Terror_in_the_Aisles, http://dbpedia.org/resource/Thanks_for_Sharing, http://dbpedia.org/resource/The_Adjutant_of_His_Excellency, http://dbpedia.org/resource/The_Airship_Destroyer, http://dbpedia.org/resource/The_Angry_Birds_Movie, http://dbpedia.org/resource/The_Barbarian_Invasions, http://dbpedia.org/resource/The_Batman_vs._Dracula, http://dbpedia.org/resource/The_Big_Picture_(1989_film), http://dbpedia.org/resource/The_Billion_Dollar_Bubble, http://dbpedia.org/resource/The_Blood_Spattered_Bride, http://dbpedia.org/resource/The_Book_of_Life_(2014_film), http://dbpedia.org/resource/The_Bounty_Hunter_(2010_film), http://dbpedia.org/resource/The_Boys_from_Brazil_(film), http://dbpedia.org/resource/The_Case, http://dbpedia.org/resource/The_Cat_Returns, http://dbpedia.org/resource/The_Charge_of_the_Light_Brigade_(1968_film), http://dbpedia.org/resource/The_Cheerleaders, http://dbpedia.org/resource/The_Chekist, http://dbpedia.org/resource/The_Chosen_One_(2007_film), http://dbpedia.org/resource/The_Chronicles_of_Narnia:_The_Lion,_the_Witch_and_the_Wardrobe, http://dbpedia.org/resource/The_Citizen_(film), http://dbpedia.org/resource/The_Coca-Cola_Kid, http://dbpedia.org/resource/The_Colossus_of_New_York, http://dbpedia.org/resource/The_Conspiracy_(1916_film), http://dbpedia.org/resource/The_Conspiracy_(2012_film), http://dbpedia.org/resource/The_Corporation_(film), http://dbpedia.org/resource/The_Croods, http://dbpedia.org/resource/The_Crush_(1993_film), http://dbpedia.org/resource/The_Dark_Crystal, http://dbpedia.org/resource/The_Day_After_Tomorrow, http://dbpedia.org/resource/The_Day_Christ_Died, http://dbpedia.org/resource/The_Dirty_Dozen, http://dbpedia.org/resource/The_Express, http://dbpedia.org/resource/The_Fall_and_Rise_of_Reginald_Perrin, http://dbpedia.org/resource/The_Flame_Trees_of_Thika, http://dbpedia.org/resource/The_Fly_(1986_film), http://dbpedia.org/resource/The_Fog_of_War, http://dbpedia.org/resource/The_Fortune, http://dbpedia.org/resource/The_Fugitive_(1993_film), http://dbpedia.org/resource/The_Games_of_Countess_Dolingen, http://dbpedia.org/resource/The_Good_Dinosaur, http://dbpedia.org/resource/The_Guns_of_Navarone_(film), http://dbpedia.org/resource/The_Hobbit_(film_series), http://dbpedia.org/resource/The_Hospital, http://dbpedia.org/resource/The_Hot_Rock_(film), http://dbpedia.org/resource/The_Invention_of_Lying, http://dbpedia.org/resource/The_Island_(2005_film), http://dbpedia.org/resource/The_Italian_Job, http://dbpedia.org/resource/The_Jungle_Book_(1994_film), http://dbpedia.org/resource/The_King_of_Braves_GaoGaiGar, http://dbpedia.org/resource/The_Last_Bolshevik, http://dbpedia.org/resource/The_Last_Detail, http://dbpedia.org/resource/The_Last_Stand_(1984_film), http://dbpedia.org/resource/The_Lemon_Sisters, http://dbpedia.org/resource/The_Lighthorsemen_(film), http://dbpedia.org/resource/The_Limey, http://dbpedia.org/resource/The_Little_Shop_of_Horrors, http://dbpedia.org/resource/The_Long_Riders, http://dbpedia.org/resource/The_Lorax_(film), http://dbpedia.org/resource/The_Man_with_the_Golden_Gun_(film), http://dbpedia.org/resource/The_McLaughlin_Group, http://dbpedia.org/resource/The_Missing_Link_(film), http://dbpedia.org/resource/The_Moth_Diaries_(film), http://dbpedia.org/resource/The_Muppet_Movie, http://dbpedia.org/resource/The_Namesake_(film), http://dbpedia.org/resource/The_Nut_Job, http://dbpedia.org/resource/The_Oogieloves_in_the_Big_Balloon_Adventure, http://dbpedia.org/resource/The_Osterman_Weekend_(film), http://dbpedia.org/resource/The_Pacifier, http://dbpedia.org/resource/The_Pallbearer, http://dbpedia.org/resource/The_Patriot_(2000_film), http://dbpedia.org/resource/The_Peanuts_Movie, http://dbpedia.org/resource/The_Positively_True_Adventures_of_the_Alleged_Texas_Cheerleader-Murdering_Mom, http://dbpedia.org/resource/The_Principle, http://dbpedia.org/resource/The_Red_Spectacles, http://dbpedia.org/resource/The_Resurrected, http://dbpedia.org/resource/The_Secret_Diary_of_Sigmund_Freud, http://dbpedia.org/resource/The_Sentinel_(1977_film), http://dbpedia.org/resource/The_Shrink_Is_In, http://dbpedia.org/resource/The_Smurfs_(film), http://dbpedia.org/resource/The_Stepford_Husbands, http://dbpedia.org/resource/The_Stepford_Wives_(2004_film), http://dbpedia.org/resource/The_Story_of_Luke, http://dbpedia.org/resource/The_Sweet_Hereafter_(film), http://dbpedia.org/resource/The_Swinging_Cheerleaders, http://dbpedia.org/resource/The_Sword_and_the_Rose, http://dbpedia.org/resource/The_Thin_Man_(film), http://dbpedia.org/resource/The_Tic_Code, http://dbpedia.org/resource/The_Tudors, http://dbpedia.org/resource/The_Vampire_Lovers, http://dbpedia.org/resource/The_Wild, http://dbpedia.org/resource/The_World's_Greatest_Lover, http://dbpedia.org/resource/Theodore_Rex_(film), http://dbpedia.org/resource/They_Came_to_Cordura, http://dbpedia.org/resource/Thirteen_Days_(film), http://dbpedia.org/resource/Toy_Story_3, http://dbpedia.org/resource/Transcendence_(2014_film), http://dbpedia.org/resource/Transformers:_Dark_of_the_Moon, http://dbpedia.org/resource/Transylvania_6-5000_(1985_film), http://dbpedia.org/resource/Treasure_Planet, http://dbpedia.org/resource/Trees_Lounge, http://dbpedia.org/resource/Turbo:_A_Power_Rangers_Movie]... [INFO] P: 2.813434148056972E-4, R: 1.0, F: 5.625285659037374E-4 -[INFO] Cumulative metrics, sample: 124, P: 0.340, R: 0.431, macro F1: 0.345, macro F1*: 0.380 -[INFO] QALD Cumulative metrics, sample: 124, P: 0.558, R: 0.431, macro F1: 0.345, QALD macro F1: 0.486 -[INFO] Current time: 3150 ms, average: 3483.92 ms/ques, entity detection: 796.04 ms/ques (linking tool: 707.05 ms/ques), relation detection: 1096.90 ms/ques (similarity: 619.45 ms/ques), EDG generation: 49.18 ms/ques +[INFO] Cumulative metrics, sample: 124, P: 0.340, R: 0.431, macro F1: 0.380 +[INFO] QALD Cumulative metrics, sample: 124, P: 0.558, R: 0.431, average F1: 0.345, QALD macro F1: 0.486 +[INFO] Current question consumed time: 2661 ms, average: 3159.73 ms/ques, entity detection: 705.05 ms/ques (linking tool: 629.22 ms/ques), relation detection: 1023.64 ms/ques (similarity: 618.99 ms/ques), EDG generation: 50.25 ms/ques [INFO] Entity Linking P: 0.29838709677419356, R: 0.29838709677419356, micro F1: 0.29838709677419356, macro F1: 0.29838709677419356 [INFO] Relation Linking P: 0.31451612903225806, R: 0.31451612903225806, micro F1: 0.31451612903225806, macro F1: 0.31451612903225806 [INFO] Type Linking P: 0.8709677419354839, R: 0.8709677419354839, micro F1: 0.8709677419354839, macro F1: 0.8709677419354839 @@ -1867,9 +1867,9 @@ Question 124: What was the name of the famous battle in 1836 in San Antonio? [INFO] Golden Answer: [http://dbpedia.org/resource/Battle_of_the_Alamo] [INFO] Predicted Answer: [] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 125, P: 0.338, R: 0.428, macro F1: 0.342, macro F1*: 0.377 -[INFO] QALD Cumulative metrics, sample: 125, P: 0.562, R: 0.428, macro F1: 0.342, QALD macro F1: 0.486 -[INFO] Current time: 3466 ms, average: 3483.78 ms/ques, entity detection: 793.79 ms/ques (linking tool: 704.68 ms/ques), relation detection: 1099.90 ms/ques (similarity: 621.11 ms/ques), EDG generation: 49.98 ms/ques +[INFO] Cumulative metrics, sample: 125, P: 0.338, R: 0.428, macro F1: 0.377 +[INFO] QALD Cumulative metrics, sample: 125, P: 0.562, R: 0.428, average F1: 0.342, QALD macro F1: 0.486 +[INFO] Current question consumed time: 2984 ms, average: 3158.33 ms/ques, entity detection: 702.86 ms/ques (linking tool: 626.95 ms/ques), relation detection: 1025.94 ms/ques (similarity: 620.67 ms/ques), EDG generation: 51.18 ms/ques [INFO] Entity Linking P: 0.304, R: 0.304, micro F1: 0.304, macro F1: 0.304 [INFO] Relation Linking P: 0.32, R: 0.32, micro F1: 0.32, macro F1: 0.32 [INFO] Type Linking P: 0.872, R: 0.872, micro F1: 0.872, macro F1: 0.872 @@ -1882,9 +1882,9 @@ Question 125: Give me the official websites of actors of the television show Cha [INFO] Golden Answer: [http://www.alyssa.com, http://www.briankrause.us] [INFO] Predicted Answer: [http://dbpedia.org/resource/Kaley_Cuoco] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 126, P: 0.335, R: 0.424, macro F1: 0.340, macro F1*: 0.374 -[INFO] QALD Cumulative metrics, sample: 126, P: 0.557, R: 0.424, macro F1: 0.340, QALD macro F1: 0.482 -[INFO] Current time: 1542 ms, average: 3468.37 ms/ques, entity detection: 788.85 ms/ques (linking tool: 699.88 ms/ques), relation detection: 1095.19 ms/ques (similarity: 618.60 ms/ques), EDG generation: 49.81 ms/ques +[INFO] Cumulative metrics, sample: 126, P: 0.335, R: 0.424, macro F1: 0.374 +[INFO] QALD Cumulative metrics, sample: 126, P: 0.557, R: 0.424, average F1: 0.340, QALD macro F1: 0.482 +[INFO] Current question consumed time: 1308 ms, average: 3143.64 ms/ques, entity detection: 698.40 ms/ques (linking tool: 622.60 ms/ques), relation detection: 1021.37 ms/ques (similarity: 618.19 ms/ques), EDG generation: 51.06 ms/ques [INFO] Entity Linking P: 0.30158730158730157, R: 0.30158730158730157, micro F1: 0.30158730158730157, macro F1: 0.30158730158730157 [INFO] Relation Linking P: 0.31746031746031744, R: 0.31746031746031744, micro F1: 0.31746031746031744, macro F1: 0.31746031746031744 [INFO] Type Linking P: 0.873015873015873, R: 0.873015873015873, micro F1: 0.873015873015873, macro F1: 0.8730158730158731 @@ -1897,9 +1897,9 @@ Question 126: How many calories does a baguette have? [INFO] Golden Answer: [263] [INFO] Predicted Answer: [263] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 127, P: 0.340, R: 0.429, macro F1: 0.345, macro F1*: 0.379 -[INFO] QALD Cumulative metrics, sample: 127, P: 0.561, R: 0.429, macro F1: 0.345, QALD macro F1: 0.486 -[INFO] Current time: 836 ms, average: 3447.64 ms/ques, entity detection: 783.78 ms/ques (linking tool: 695.03 ms/ques), relation detection: 1088.60 ms/ques (similarity: 615.73 ms/ques), EDG generation: 49.51 ms/ques +[INFO] Cumulative metrics, sample: 127, P: 0.340, R: 0.429, macro F1: 0.379 +[INFO] QALD Cumulative metrics, sample: 127, P: 0.561, R: 0.429, average F1: 0.345, QALD macro F1: 0.486 +[INFO] Current question consumed time: 705 ms, average: 3124.44 ms/ques, entity detection: 693.71 ms/ques (linking tool: 618.17 ms/ques), relation detection: 1015.18 ms/ques (similarity: 615.34 ms/ques), EDG generation: 50.76 ms/ques [INFO] Entity Linking P: 0.2992125984251969, R: 0.2992125984251969, micro F1: 0.2992125984251969, macro F1: 0.2992125984251969 [INFO] Relation Linking P: 0.31496062992125984, R: 0.31496062992125984, micro F1: 0.31496062992125984, macro F1: 0.31496062992125984 [INFO] Type Linking P: 0.8740157480314961, R: 0.8740157480314961, micro F1: 0.8740157480314961, macro F1: 0.8740157480314961 @@ -1912,9 +1912,9 @@ Question 127: Give me all libraries established before 1400. [INFO] Golden Answer: [http://dbpedia.org/resource/Maughan_Library, http://dbpedia.org/resource/Cambridge_University_Library, http://dbpedia.org/resource/Library_of_Alexandria, http://dbpedia.org/resource/Library_of_Ashurbanipal, http://dbpedia.org/resource/Nanpura_Parsi_Library, http://dbpedia.org/resource/National_and_University_Library_of_the_Republika_Srpska, http://dbpedia.org/resource/Library_of_Pergamum, http://dbpedia.org/resource/Merton_College_Library, http://dbpedia.org/resource/Austrian_National_Library, http://dbpedia.org/resource/Heidelberg_University_Library, http://dbpedia.org/resource/Mother_Irene_Gill_Memorial_Library] [INFO] Predicted Answer: [] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 128, P: 0.338, R: 0.425, macro F1: 0.342, macro F1*: 0.376 -[INFO] QALD Cumulative metrics, sample: 128, P: 0.564, R: 0.425, macro F1: 0.342, QALD macro F1: 0.485 -[INFO] Current time: 822 ms, average: 3427.13 ms/ques, entity detection: 777.66 ms/ques (linking tool: 689.61 ms/ques), relation detection: 1082.50 ms/ques (similarity: 612.46 ms/ques), EDG generation: 49.21 ms/ques +[INFO] Cumulative metrics, sample: 128, P: 0.338, R: 0.425, macro F1: 0.376 +[INFO] QALD Cumulative metrics, sample: 128, P: 0.564, R: 0.425, average F1: 0.342, QALD macro F1: 0.485 +[INFO] Current question consumed time: 663 ms, average: 3105.21 ms/ques, entity detection: 688.29 ms/ques (linking tool: 613.34 ms/ques), relation detection: 1009.32 ms/ques (similarity: 611.99 ms/ques), EDG generation: 50.45 ms/ques [INFO] Entity Linking P: 0.3046875, R: 0.3046875, micro F1: 0.3046875, macro F1: 0.3046875 [INFO] Relation Linking P: 0.3203125, R: 0.3203125, micro F1: 0.3203125, macro F1: 0.3203125 [INFO] Type Linking P: 0.875, R: 0.875, micro F1: 0.875, macro F1: 0.875 @@ -1927,9 +1927,9 @@ Question 128: Give me all Frisian islands that belong to the Netherlands. [INFO] Golden Answer: [http://dbpedia.org/resource/Frisian_Islands] [INFO] Predicted Answer: [http://dbpedia.org/resource/Het_Loo_Palace] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 129, P: 0.335, R: 0.422, macro F1: 0.340, macro F1*: 0.373 -[INFO] QALD Cumulative metrics, sample: 129, P: 0.560, R: 0.422, macro F1: 0.340, QALD macro F1: 0.481 -[INFO] Current time: 14092 ms, average: 3509.81 ms/ques, entity detection: 796.16 ms/ques (linking tool: 707.47 ms/ques), relation detection: 1115.53 ms/ques (similarity: 627.19 ms/ques), EDG generation: 48.98 ms/ques +[INFO] Cumulative metrics, sample: 129, P: 0.335, R: 0.422, macro F1: 0.373 +[INFO] QALD Cumulative metrics, sample: 129, P: 0.560, R: 0.422, average F1: 0.340, QALD macro F1: 0.481 +[INFO] Current question consumed time: 11956 ms, average: 3173.82 ms/ques, entity detection: 704.08 ms/ques (linking tool: 628.71 ms/ques), relation detection: 1038.64 ms/ques (similarity: 626.84 ms/ques), EDG generation: 50.22 ms/ques [INFO] Entity Linking P: 0.3023255813953488, R: 0.3023255813953488, micro F1: 0.3023255813953488, macro F1: 0.3023255813953488 [INFO] Relation Linking P: 0.3178294573643411, R: 0.3178294573643411, micro F1: 0.3178294573643411, macro F1: 0.3178294573643411 [INFO] Type Linking P: 0.875968992248062, R: 0.875968992248062, micro F1: 0.875968992248062, macro F1: 0.875968992248062 @@ -1942,9 +1942,9 @@ Question 129: Which museum exhibits The Scream by Munch? [INFO] Golden Answer: [http://dbpedia.org/resource/National_Gallery_(Norway)] [INFO] Predicted Answer: [http://dbpedia.org/resource/Category:Musical_groups_disestablished_in_1990, http://dbpedia.org/resource/Category:Hardcore_punk_groups_from_Virginia, http://dbpedia.org/resource/Category:Dave_Grohl, http://dbpedia.org/resource/Category:Dischord_Records_artists, http://dbpedia.org/resource/Category:Hardcore_punk_groups_from_Washington,_D.C., http://dbpedia.org/resource/Category:Musical_groups_established_in_1981, http://dbpedia.org/resource/Category:Musical_groups_reestablished_in_2009] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 130, P: 0.332, R: 0.419, macro F1: 0.337, macro F1*: 0.371 -[INFO] QALD Cumulative metrics, sample: 130, P: 0.555, R: 0.419, macro F1: 0.337, QALD macro F1: 0.478 -[INFO] Current time: 1557 ms, average: 3494.78 ms/ques, entity detection: 791.92 ms/ques (linking tool: 703.05 ms/ques), relation detection: 1109.91 ms/ques (similarity: 625.28 ms/ques), EDG generation: 48.69 ms/ques +[INFO] Cumulative metrics, sample: 130, P: 0.332, R: 0.419, macro F1: 0.371 +[INFO] QALD Cumulative metrics, sample: 130, P: 0.555, R: 0.419, average F1: 0.337, QALD macro F1: 0.478 +[INFO] Current question consumed time: 1302 ms, average: 3159.43 ms/ques, entity detection: 700.11 ms/ques (linking tool: 624.68 ms/ques), relation detection: 1033.19 ms/ques (similarity: 624.82 ms/ques), EDG generation: 49.92 ms/ques [INFO] Entity Linking P: 0.3, R: 0.3, micro F1: 0.3, macro F1: 0.3 [INFO] Relation Linking P: 0.3153846153846154, R: 0.3153846153846154, micro F1: 0.3153846153846154, macro F1: 0.3153846153846154 [INFO] Type Linking P: 0.8769230769230769, R: 0.8769230769230769, micro F1: 0.8769230769230769, macro F1: 0.8769230769230769 @@ -1957,9 +1957,9 @@ Question 130: When was the De Beers company founded? [INFO] Golden Answer: [1888-01-01] [INFO] Predicted Answer: [1888] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 131, P: 0.337, R: 0.423, macro F1: 0.342, macro F1*: 0.376 -[INFO] QALD Cumulative metrics, sample: 131, P: 0.559, R: 0.423, macro F1: 0.342, QALD macro F1: 0.482 -[INFO] Current time: 932 ms, average: 3475.22 ms/ques, entity detection: 786.98 ms/ques (linking tool: 698.37 ms/ques), relation detection: 1104.29 ms/ques (similarity: 622.15 ms/ques), EDG generation: 48.37 ms/ques +[INFO] Cumulative metrics, sample: 131, P: 0.337, R: 0.423, macro F1: 0.376 +[INFO] QALD Cumulative metrics, sample: 131, P: 0.559, R: 0.423, average F1: 0.342, QALD macro F1: 0.482 +[INFO] Current question consumed time: 797 ms, average: 3141.40 ms/ques, entity detection: 695.66 ms/ques (linking tool: 620.47 ms/ques), relation detection: 1027.87 ms/ques (similarity: 621.69 ms/ques), EDG generation: 49.60 ms/ques [INFO] Entity Linking P: 0.29770992366412213, R: 0.29770992366412213, micro F1: 0.29770992366412213, macro F1: 0.29770992366412213 [INFO] Relation Linking P: 0.31297709923664124, R: 0.31297709923664124, micro F1: 0.31297709923664124, macro F1: 0.31297709923664124 [INFO] Type Linking P: 0.8778625954198473, R: 0.8778625954198473, micro F1: 0.8778625954198473, macro F1: 0.8778625954198473 @@ -1972,9 +1972,9 @@ Question 131: Who became president after JFK died? [INFO] Golden Answer: [http://dbpedia.org/resource/Lyndon_B._Johnson] [INFO] Predicted Answer: [http://dbpedia.org/resource/Massimo_D'Alema, http://dbpedia.org/resource/Pier_Ferdinando_Casini, http://dbpedia.org/resource/Gianfranco_Fini, http://dbpedia.org/resource/Silvio_Berlusconi, http://dbpedia.org/resource/Giuliano_Amato, http://dbpedia.org/resource/Franco_Marini, http://dbpedia.org/resource/Fausto_Bertinotti, http://dbpedia.org/resource/Javier_González_Fraga, http://dbpedia.org/resource/Domingo_Cavallo, http://dbpedia.org/resource/Eduardo_Duhalde, http://dbpedia.org/resource/María_Julia_Alsogaray, http://dbpedia.org/resource/Oscar_Camilión, http://dbpedia.org/resource/Zulema_María_Eva_Menem, http://dbpedia.org/resource/Guido_di_Tella, http://dbpedia.org/resource/Fernando_de_la_Rúa, http://dbpedia.org/resource/Sultan_Ali_Keshtmand, http://dbpedia.org/resource/José_López_Rega, http://dbpedia.org/resource/Alberto_Teisaire, http://dbpedia.org/resource/Ramón_Carrillo, http://dbpedia.org/resource/Hortensio_Quijano, http://dbpedia.org/resource/Juan_Pistarini, http://dbpedia.org/resource/Isabel_Martínez_de_Perón, http://dbpedia.org/resource/Juan_Atilio_Bramuglia, http://dbpedia.org/resource/Amintore_Fanfani, http://dbpedia.org/resource/Alcide_De_Gasperi, http://dbpedia.org/resource/Giuseppe_Pella, http://dbpedia.org/resource/Mario_Scelba, http://dbpedia.org/resource/Mario_Fernández_Baeza, http://dbpedia.org/resource/Michelle_Bachelet, http://dbpedia.org/resource/Ignacio_Walker, http://dbpedia.org/resource/Jaime_Ravinet, http://dbpedia.org/resource/Felipe_Harboe, http://dbpedia.org/resource/General_Khodaidad, http://dbpedia.org/resource/İsmet_İnönü, http://dbpedia.org/resource/Süleyman_Demirel, http://dbpedia.org/resource/Daniş_Tunalıgil, http://dbpedia.org/resource/Tang_Fei, http://dbpedia.org/resource/Mark_Chen, http://dbpedia.org/resource/Chuang_Kuo-jung, http://dbpedia.org/resource/Annette_Lu, http://dbpedia.org/resource/Frank_Hsieh, http://dbpedia.org/resource/Su_Tseng-chang, http://dbpedia.org/resource/Liao_Feng-teh, http://dbpedia.org/resource/Lai_In-jaw, http://dbpedia.org/resource/Yeh_Chu-lan, http://dbpedia.org/resource/Hu_Chen-pu, http://dbpedia.org/resource/Kang_Young-hoon, http://dbpedia.org/resource/Rafael_Moreno_Valle, http://dbpedia.org/resource/Antonio_Ortiz_Mena, http://dbpedia.org/resource/Antoni_Castells, http://dbpedia.org/resource/Josep_Huguet, http://dbpedia.org/resource/Joan_Puigcercós, http://dbpedia.org/resource/Josep-Lluís_Carod-Rovira, http://dbpedia.org/resource/Jordi_Ausàs, http://dbpedia.org/resource/Li_Yuan-tsu, http://dbpedia.org/resource/Hau_Pei-tsun, http://dbpedia.org/resource/Lee_Huan, http://dbpedia.org/resource/Lien_Chan, http://dbpedia.org/resource/Vincent_Siew, http://dbpedia.org/resource/Wu_Po-hsiung, http://dbpedia.org/resource/Shih_Chi-yang, http://dbpedia.org/resource/Huang_Kun-huei, http://dbpedia.org/resource/Guan_Linzheng, http://dbpedia.org/resource/Yan_Xishan, http://dbpedia.org/resource/Yu_Hanmou, http://dbpedia.org/resource/Zhang_Fakui, http://dbpedia.org/resource/Gu_Zhutong, http://dbpedia.org/resource/Wang_Jingwei, http://dbpedia.org/resource/Zhang_Qun, http://dbpedia.org/resource/Chiang_Kai-shek, http://dbpedia.org/resource/H._H._Kung, http://dbpedia.org/resource/Sun_Fo, http://dbpedia.org/resource/Marta_Cid, http://dbpedia.org/resource/Joan_Carretero, http://dbpedia.org/resource/Xavier_Vendrell, http://dbpedia.org/resource/Josep_Bargalló, http://dbpedia.org/resource/José_Manuel_Puig_Casauranc, http://dbpedia.org/resource/Manuel_C._Téllez, http://dbpedia.org/resource/Chiang_Wei-kuo, http://dbpedia.org/resource/Chiang_Ching-kuo, http://dbpedia.org/resource/Sun_Yun-suan, http://dbpedia.org/resource/José_Vasconcelos, http://dbpedia.org/resource/Rosalía_Arteaga, http://dbpedia.org/resource/Rafael_Correa, http://dbpedia.org/resource/Ana_Ligia_Mixco_Sol_de_Saca, http://dbpedia.org/resource/John_Kerr,_Baron_Kerr_of_Kinlochard, http://dbpedia.org/resource/Lourdes_Rodríguez_de_Flores, http://dbpedia.org/resource/Nicolás_Bravo, http://dbpedia.org/resource/Osvaldo_Hurtado, http://dbpedia.org/resource/Mariano_Suárez, http://dbpedia.org/resource/Carlos_Julio_Arosemena_Monroy, http://dbpedia.org/resource/Andrus_Ansip, http://dbpedia.org/resource/Andres_Tarand, http://dbpedia.org/resource/Mart_Siimann, http://dbpedia.org/resource/Edgar_Savisaar, http://dbpedia.org/resource/Alfredo_Palacio, http://dbpedia.org/resource/Guillermo_Lasso, http://dbpedia.org/resource/Édith_Cresson, http://dbpedia.org/resource/Emma_Bonino, http://dbpedia.org/resource/Hans_van_den_Broek, http://dbpedia.org/resource/Neil_Kinnock, http://dbpedia.org/resource/João_de_Deus_Pinheiro, http://dbpedia.org/resource/Leon_Brittan, http://dbpedia.org/resource/Karel_Van_Miert, http://dbpedia.org/resource/Meles_Zenawi, http://dbpedia.org/resource/Plutarco_Naranjo_Vargas, http://dbpedia.org/resource/Fabián_Alarcón, http://dbpedia.org/resource/Darío_Echandía, http://dbpedia.org/resource/António_Guterres, http://dbpedia.org/resource/Fernando_Teixeira_dos_Santos, http://dbpedia.org/resource/Teresa_Patrício_de_Gouveia, http://dbpedia.org/resource/Carlos_César, http://dbpedia.org/resource/José_Manuel_Barroso, http://dbpedia.org/resource/Vasco_Joaquim_Rocha_Vieira, http://dbpedia.org/resource/Caetano_Luís_Pequito_de_Almeida_Sampaio, http://dbpedia.org/resource/Pedro_Santana_Lopes, http://dbpedia.org/resource/José_Sócrates, http://dbpedia.org/resource/Manuel_Pinho, http://dbpedia.org/resource/Mário_Lino, http://dbpedia.org/resource/António_de_Sousa_Franco, http://dbpedia.org/resource/Norberto_Quirno_Costa, http://dbpedia.org/resource/Ali_Badjo_Gamatié, http://dbpedia.org/resource/Hama_Amadou, http://dbpedia.org/resource/Aldo_Moro, http://dbpedia.org/resource/Giovanni_Leone, http://dbpedia.org/resource/Albert_Coppé, http://dbpedia.org/resource/Sicco_Mansholt, http://dbpedia.org/resource/Altiero_Spinelli, http://dbpedia.org/resource/Raymond_Barre, http://dbpedia.org/resource/Redley_A._Killion, http://dbpedia.org/resource/Jorge_Holguín, http://dbpedia.org/resource/Marco_Fidel_Suárez, http://dbpedia.org/resource/José_María_Campo_Serrano__Minister_of_War__1, http://dbpedia.org/resource/Jean-Claude_Gaudin, http://dbpedia.org/resource/Mariano_Ospina_Pérez, http://dbpedia.org/resource/Juan_Esteban_Pedernera, http://dbpedia.org/resource/Francisco_José_Urrutia_Olano, http://dbpedia.org/resource/Enrique_Olaya_Herrera, http://dbpedia.org/resource/Liborio_Mejía, http://dbpedia.org/resource/Miguel_Abadía_Méndez, http://dbpedia.org/resource/José_Vicente_Concha, http://dbpedia.org/resource/Ramón_González_Valencia, http://dbpedia.org/resource/Santiago_Pérez_de_Manosalbas, http://dbpedia.org/resource/Rodolfo_Frigeri, http://dbpedia.org/resource/José_María_Vernet, http://dbpedia.org/resource/Zoran_Živković_(politician), http://dbpedia.org/resource/Vojislav_Koštunica, http://dbpedia.org/resource/Karen_Olsen_Beck, http://dbpedia.org/resource/Doris_Yankelewitz_Berger, http://dbpedia.org/resource/Juan_Manuel_Villasuso, http://dbpedia.org/resource/Zoran_Đinđić, http://dbpedia.org/resource/Hope_College, http://dbpedia.org/resource/Serge_Vohor, http://dbpedia.org/resource/Edward_Natapei, http://dbpedia.org/resource/Ham_Lini, http://dbpedia.org/resource/Julio_César_Franco, http://dbpedia.org/resource/Jules_Wijdenbosch, http://dbpedia.org/resource/Nizami_Bahmanov, http://dbpedia.org/resource/Nuraddin_Sadigov, http://dbpedia.org/resource/Dadash_Rzayev, http://dbpedia.org/resource/Heydar_Aliyev, http://dbpedia.org/resource/Safar_Abiyev, http://dbpedia.org/resource/Surat_Huseynov, http://dbpedia.org/resource/Rasul_Guliyev, http://dbpedia.org/resource/Ali_Masimov, http://dbpedia.org/resource/Rahim_Huseynov, http://dbpedia.org/resource/Rafig_Asgarov, http://dbpedia.org/resource/Arif_Valiyev, http://dbpedia.org/resource/Panah_Huseynov, http://dbpedia.org/resource/Tofig_Gasimov, http://dbpedia.org/resource/Firuz_Mustafayev, http://dbpedia.org/resource/Valeh_Barshadly, http://dbpedia.org/resource/Lopo_do_Nascimento, http://dbpedia.org/resource/Anahita_Ratebzad, http://dbpedia.org/resource/Nur_Ahmed_Nur, http://dbpedia.org/resource/Mohammad_Najibullah, http://dbpedia.org/resource/Janet_Yellen, http://dbpedia.org/resource/Chuck_Hagel, http://dbpedia.org/resource/David_Petraeus, http://dbpedia.org/resource/Rahm_Emanuel, http://dbpedia.org/resource/Susan_Rice, http://dbpedia.org/resource/Tom_Frieden, http://dbpedia.org/resource/Arne_Duncan, http://dbpedia.org/resource/Ray_LaHood, http://dbpedia.org/resource/Robert_Gibbs, http://dbpedia.org/resource/Anthony_W._Miller, http://dbpedia.org/resource/Gary_Locke, http://dbpedia.org/resource/Samuel_L._Kaplan, http://dbpedia.org/resource/Manson_K._Brown, http://dbpedia.org/resource/Michael_G._Vickers, http://dbpedia.org/resource/Michael_J._Astrue, http://dbpedia.org/resource/Elisse_B._Walter, http://dbpedia.org/resource/Jessica_Rosenworcel, http://dbpedia.org/resource/Jonathan_Adelstein, http://dbpedia.org/resource/Julius_Genachowski, http://dbpedia.org/resource/Michael_O'Rielly, http://dbpedia.org/resource/Mignon_Clyburn, http://dbpedia.org/resource/Ron_Sims, http://dbpedia.org/resource/Edwin_Kneedler, http://dbpedia.org/resource/Ian_Heath_Gershengorn, http://dbpedia.org/resource/Leondra_Kruger, http://dbpedia.org/resource/Mark_Gitenstein, http://dbpedia.org/resource/Regina_Benjamin, http://dbpedia.org/resource/Sri_Srinivasan, http://dbpedia.org/resource/Pamela_E._Bridgewater, http://dbpedia.org/resource/Steven_VanRoekel, http://dbpedia.org/resource/Ann_Wagner, http://dbpedia.org/resource/Gayleatha_B._Brown, http://dbpedia.org/resource/George_A._Krol, http://dbpedia.org/resource/Michele_Thoren_Bond, http://dbpedia.org/resource/Peter_Allgeier, http://dbpedia.org/resource/Gina_Abercrombie-Winstanley, http://dbpedia.org/resource/Karen_Mills, http://dbpedia.org/resource/Michael_Morell, http://dbpedia.org/resource/Richard_Cordray, http://dbpedia.org/resource/Stephanie_O'Sullivan, http://dbpedia.org/resource/Glenn_A._Fine, http://dbpedia.org/resource/Robert_O._Work, http://dbpedia.org/resource/Elizabeth_Littlefield, http://dbpedia.org/resource/Richard_Norland, http://dbpedia.org/resource/Joseph_LeBaron, http://dbpedia.org/resource/Joyce_Anne_Barr, http://dbpedia.org/resource/Cyrus_Amir-Mokri, http://dbpedia.org/resource/Marietta_S._Robinson, http://dbpedia.org/resource/Brian_Deese, http://dbpedia.org/resource/Denise_O'Donnell, http://dbpedia.org/resource/Timothy_Massad, http://dbpedia.org/resource/Howard_Schmidt, http://dbpedia.org/resource/Patricia_A._Butenis, http://dbpedia.org/resource/Patrick_F._Kennedy, http://dbpedia.org/resource/Ann_Stock, http://dbpedia.org/resource/David_S._Cohen_(attorney), http://dbpedia.org/resource/W._Neil_Eggleston, http://dbpedia.org/resource/Carlos_Pascual_(diplomat), http://dbpedia.org/resource/Jeffrey_R._Cellars, http://dbpedia.org/resource/Kathleen_Stephens, http://dbpedia.org/resource/Cameron_R._Hume, http://dbpedia.org/resource/Liliana_Ayalde, http://dbpedia.org/resource/United_States_Special_Envoy_for_Eurasian_Energy, http://dbpedia.org/resource/Mark_Rosenker, http://dbpedia.org/resource/Maura_Connelly, http://dbpedia.org/resource/Ted_Osius, http://dbpedia.org/resource/Kenneth_John_Gonzales, http://dbpedia.org/resource/Kyle_Randolph_Scott, http://dbpedia.org/resource/Brad_Pfaff, http://dbpedia.org/resource/Greg_Delawie, http://dbpedia.org/resource/Jonathan_Coppess, http://dbpedia.org/resource/Rayne_Pegg, http://dbpedia.org/resource/Kim_Darroch, http://dbpedia.org/resource/Lisa_P._Jackson, http://dbpedia.org/resource/Pete_Rouse, http://dbpedia.org/resource/Daniel_Benjamin, http://dbpedia.org/resource/David_L._Boren, http://dbpedia.org/resource/Joe_Biden, http://dbpedia.org/resource/Leon_Panetta, http://dbpedia.org/resource/Richard_Holbrooke, http://dbpedia.org/resource/Robert_Gates, http://dbpedia.org/resource/Francis_Collins, http://dbpedia.org/resource/John_Kerry, http://dbpedia.org/resource/Orrin_Hatch, http://dbpedia.org/resource/Jill_Biden, http://dbpedia.org/resource/Kevin_O'Malley, http://dbpedia.org/resource/William_M._Daley, http://dbpedia.org/resource/Derek_Mitchell, http://dbpedia.org/resource/Gregory_B._Craig, http://dbpedia.org/resource/Michael_Hayden_(general), http://dbpedia.org/resource/Tammy_Duckworth, http://dbpedia.org/resource/William_H._Webster, http://dbpedia.org/resource/Donald_Berwick, http://dbpedia.org/resource/Mary_Voytek, http://dbpedia.org/resource/Larry_Echo_Hawk, http://dbpedia.org/resource/Louis_Susman, http://dbpedia.org/resource/Suzan_G._LeVine, http://dbpedia.org/resource/James_Steinberg, http://dbpedia.org/resource/Rand_Beers, http://dbpedia.org/resource/Stephen_Mull, http://dbpedia.org/resource/Karl_Eikenberry, http://dbpedia.org/resource/Richard_Jones_(U.S._diplomat), http://dbpedia.org/resource/Victoria_Nuland, http://dbpedia.org/resource/Eugene_Louis_Dodaro, http://dbpedia.org/resource/Davita_Vance-Cooks, http://dbpedia.org/resource/William_J._Boarman, http://dbpedia.org/resource/Betty_J._Sapp, http://dbpedia.org/resource/Margaret_Hamburg, http://dbpedia.org/resource/James_P._McAnulty, http://dbpedia.org/resource/Jen_Psaki, http://dbpedia.org/resource/Karen_Kornbluh, http://dbpedia.org/resource/Michael_P._Meehan, http://dbpedia.org/resource/Susan_McCue, http://dbpedia.org/resource/Gail_D._Mathieu, http://dbpedia.org/resource/Mark_Boulware, http://dbpedia.org/resource/Karen_L._Loeffler, http://dbpedia.org/resource/Jamie_M._Morin, http://dbpedia.org/resource/Anthony_L._Gardner, http://dbpedia.org/resource/Deborah_K._Jones, http://dbpedia.org/resource/Jeffrey_L._Bleich, http://dbpedia.org/resource/Patricia_McMahon_Hawkins, http://dbpedia.org/resource/Stephen_W._Preston, http://dbpedia.org/resource/Anne_C._Richard, http://dbpedia.org/resource/John_L._Withers,_II, http://dbpedia.org/resource/Roderick_W._Moore, http://dbpedia.org/resource/Eric_G._John, http://dbpedia.org/resource/Lisa_Kubiske, http://dbpedia.org/resource/Bisa_Williams, http://dbpedia.org/resource/Mark_D._Clookie, http://dbpedia.org/resource/Francis_J._Ricciardone_Jr., http://dbpedia.org/resource/Tim_Broas, http://dbpedia.org/resource/Carrie_Hessler-Radelet, http://dbpedia.org/resource/Jeanne_Hulit, http://dbpedia.org/resource/Betty_E._King, http://dbpedia.org/resource/Charles_Bolden, http://dbpedia.org/resource/Hillary_Clinton, http://dbpedia.org/resource/J._Christopher_Stevens, http://dbpedia.org/resource/Jack_Lew, http://dbpedia.org/resource/Jason_Furman, http://dbpedia.org/resource/Kal_Penn, http://dbpedia.org/resource/Katharine_Abraham, http://dbpedia.org/resource/Robert_Mueller, http://dbpedia.org/resource/Ron_Klain, http://dbpedia.org/resource/Timothy_Geithner, http://dbpedia.org/resource/Adolfo_Carrión_Jr., http://dbpedia.org/resource/Gary_Hart, http://dbpedia.org/resource/George_J._Mitchell, http://dbpedia.org/resource/James_Comey, http://dbpedia.org/resource/Karen_B._Stewart, http://dbpedia.org/resource/Mary_Jo_White, http://dbpedia.org/resource/Michelle_Obama, http://dbpedia.org/resource/Nancy_Pelosi, http://dbpedia.org/resource/Neel_Kashkari, http://dbpedia.org/resource/Ron_Kirk, http://dbpedia.org/resource/Scot_Marciel, http://dbpedia.org/resource/Steven_Chu, http://dbpedia.org/resource/Caroline_Kennedy, http://dbpedia.org/resource/Elizabeth_Warren, http://dbpedia.org/resource/Eric_Holder, http://dbpedia.org/resource/Ray_Mabus, http://dbpedia.org/resource/Steven_E._Koonin, http://dbpedia.org/resource/Ben_Bernanke, http://dbpedia.org/resource/Gary_Gensler, http://dbpedia.org/resource/Gene_Sperling, http://dbpedia.org/resource/Gordon_R._England, http://dbpedia.org/resource/Jane_Lubchenco, http://dbpedia.org/resource/Mary_Schapiro, http://dbpedia.org/resource/Max_Baucus, http://dbpedia.org/resource/Patrick_Murphy_(Pennsylvania_politician), http://dbpedia.org/resource/Tom_Vilsack, http://dbpedia.org/resource/Josh_Earnest, http://dbpedia.org/resource/William_Bratton, http://dbpedia.org/resource/Brian_D._Montgomery, http://dbpedia.org/resource/Elizabeth_Frawley_Bagley, http://dbpedia.org/resource/John_E._Potter, http://dbpedia.org/resource/Megan_Brennan, http://dbpedia.org/resource/Patrick_R._Donahoe, http://dbpedia.org/resource/Sloan_D._Gibson, http://dbpedia.org/resource/Craig_Fugate, http://dbpedia.org/resource/Nancy_L._Ward, http://dbpedia.org/resource/Thomas_R._Nides, http://dbpedia.org/resource/Peter_Cianchette, http://dbpedia.org/resource/Deesha_Dyer, http://dbpedia.org/resource/Johnnie_Carson, http://dbpedia.org/resource/Kathleen_Merrigan, http://dbpedia.org/resource/Alejandro_Daniel_Wolff, http://dbpedia.org/resource/James_B._Cunningham, http://dbpedia.org/resource/Robert_H._Tuttle, http://dbpedia.org/resource/Ryan_Crocker, http://dbpedia.org/resource/Demetrios_Marantis, http://dbpedia.org/resource/Miriam_Sapiro, http://dbpedia.org/resource/Robert_Holleyman, http://dbpedia.org/resource/Louis_Caldera, http://dbpedia.org/resource/William_J._Lynn_III, http://dbpedia.org/resource/Timothy_J._Roemer, http://dbpedia.org/resource/Cody_Keenan, http://dbpedia.org/resource/Michael_Posner_(lawyer), http://dbpedia.org/resource/Juliette_Kayyem, http://dbpedia.org/resource/Marcus_Peacock, http://dbpedia.org/resource/Nancy_Sutley, http://dbpedia.org/resource/Piper_Anne_Wind_Campbell, http://dbpedia.org/resource/Jennifer_Pahlka, http://dbpedia.org/resource/Jim_Messina_(political_staffer), http://dbpedia.org/resource/David_Shedd, http://dbpedia.org/resource/Michael_Sulick, http://dbpedia.org/resource/Dava_Newman, http://dbpedia.org/resource/John_Clint_Williamson, http://dbpedia.org/resource/William_Yosses, http://dbpedia.org/resource/Melody_Barnes, http://dbpedia.org/resource/Edward_Jurith, http://dbpedia.org/resource/Lesa_Roe, http://dbpedia.org/resource/Michael_Botticelli_(civil_servant), http://dbpedia.org/resource/Benjamin_B._Wagner, http://dbpedia.org/resource/Suzan_Johnson_Cook, http://dbpedia.org/resource/Stephen_Kappes, http://dbpedia.org/resource/Stephen_T._Ayers, http://dbpedia.org/resource/Elliot_F._Kaye, http://dbpedia.org/resource/Heather_M._Hodges, http://dbpedia.org/resource/Joseph_Mohorovic, http://dbpedia.org/resource/Robert_S._Adler, http://dbpedia.org/resource/Douglas_Shulman, http://dbpedia.org/resource/Pamela_White, http://dbpedia.org/resource/Scott_H._DeLisi, http://dbpedia.org/resource/Virginia_A._Seitz, http://dbpedia.org/resource/J._Adam_Ereli, http://dbpedia.org/resource/Jeremy_Bernard, http://dbpedia.org/resource/Juan_M._Garcia_III, http://dbpedia.org/resource/Alan_Solomont, http://dbpedia.org/resource/Capricia_Marshall, http://dbpedia.org/resource/Catherine_A._Novelli, http://dbpedia.org/resource/Evan_Ryan, http://dbpedia.org/resource/Rafael_Moure-Eraso, http://dbpedia.org/resource/Dawn_M._Liberi, http://dbpedia.org/resource/Stephen_G._McFarland, http://dbpedia.org/resource/William_E._Todd, http://dbpedia.org/resource/Marianne_Markowitz, http://dbpedia.org/resource/Ferial_Govashiri, http://dbpedia.org/resource/Mary_Ryckman, http://dbpedia.org/resource/Luciana_Borio, http://dbpedia.org/resource/Bruce_Heyman, http://dbpedia.org/resource/Cass_Sunstein, http://dbpedia.org/resource/Dan_Rooney, http://dbpedia.org/resource/Patrick_Leahy, http://dbpedia.org/resource/Carol_Browner, http://dbpedia.org/resource/Harold_Hongju_Koh, http://dbpedia.org/resource/James_L._Jones, http://dbpedia.org/resource/John_R._Allen, http://dbpedia.org/resource/Kurt_M._Campbell, http://dbpedia.org/resource/Stanley_Fischer, http://dbpedia.org/resource/Adriana_Kugler, http://dbpedia.org/resource/Christina_Romer, http://dbpedia.org/resource/Jeffrey_Zients, http://dbpedia.org/resource/Jennifer_Hunt, http://dbpedia.org/resource/Jerome_H._Powell, http://dbpedia.org/resource/Lael_Brainard, http://dbpedia.org/resource/Paul_Volcker, http://dbpedia.org/resource/Rebecca_Blank, http://dbpedia.org/resource/Robert_A._McDonald, http://dbpedia.org/resource/Alan_Krueger, http://dbpedia.org/resource/Anne_W._Patterson, http://dbpedia.org/resource/Bob_Graham, http://dbpedia.org/resource/David_W._Ogden, http://dbpedia.org/resource/John_Podesta, http://dbpedia.org/resource/John_S._Pistole, http://dbpedia.org/resource/Michael_Chertoff, http://dbpedia.org/resource/Russ_Feingold, http://dbpedia.org/resource/Ernest_Moniz, http://dbpedia.org/resource/Eric_Shinseki, http://dbpedia.org/resource/Gregory_Jaczko, http://dbpedia.org/resource/Hilda_Solis, http://dbpedia.org/resource/John_Beyrle, http://dbpedia.org/resource/John_Holdren, http://dbpedia.org/resource/Matthew_Barzun, http://dbpedia.org/resource/John_Berry_(administrator), http://dbpedia.org/resource/Anthony_Foxx, http://dbpedia.org/resource/Ash_Carter, http://dbpedia.org/resource/Austan_Goolsbee, http://dbpedia.org/resource/Deborah_Lee_James, http://dbpedia.org/resource/Denis_McDonough, http://dbpedia.org/resource/Ellen_Tauscher, http://dbpedia.org/resource/Eric_Fanning, http://dbpedia.org/resource/Gina_McCarthy, http://dbpedia.org/resource/Heather_Higginbottom, http://dbpedia.org/resource/Loretta_Lynch, http://dbpedia.org/resource/Maria_Contreras-Sweet, http://dbpedia.org/resource/Michael_Froman, http://dbpedia.org/resource/Penny_Pritzker, http://dbpedia.org/resource/Sally_Jewell, http://dbpedia.org/resource/Shaun_Donovan, http://dbpedia.org/resource/Stephen_W._Rochon, http://dbpedia.org/resource/Sylvia_Mathews_Burwell, http://dbpedia.org/resource/Thomas_Perez, http://dbpedia.org/resource/Tony_Blinken, http://dbpedia.org/resource/Boris_Lushniak, http://dbpedia.org/resource/Robert_O._Blake,_Jr., http://dbpedia.org/resource/Nancy_Jo_Powell, http://dbpedia.org/resource/Philip_D._Murphy, http://dbpedia.org/resource/William_Joseph_Burns, http://dbpedia.org/resource/Mona_Sutphen, http://dbpedia.org/resource/Peter_R._Orszag, http://dbpedia.org/resource/Erica_Groshen, http://dbpedia.org/resource/Teddy_B._Taylor, http://dbpedia.org/resource/John_M._McHugh, http://dbpedia.org/resource/Edith_Ramirez, http://dbpedia.org/resource/Julie_Brill, http://dbpedia.org/resource/William_Drea_Adams, http://dbpedia.org/resource/Robert_B._Murrett, http://dbpedia.org/resource/Robert_Cardillo, http://dbpedia.org/resource/Sheila_Bair, http://dbpedia.org/resource/Lanny_A._Breuer, http://dbpedia.org/resource/Anne_Northup, http://dbpedia.org/resource/Vinai_Thummalapally, http://dbpedia.org/resource/Eileen_Rominger, http://dbpedia.org/resource/Gil_Kerlikowske, http://dbpedia.org/resource/Ron_Bloom, http://dbpedia.org/resource/Nicole_Avant, http://dbpedia.org/resource/John_T._Morton, http://dbpedia.org/resource/Ivo_Daalder, http://dbpedia.org/resource/Julissa_Reynoso, http://dbpedia.org/resource/Kristie_Kenney, http://dbpedia.org/resource/Adrienne_Thomas, http://dbpedia.org/resource/Raul_Yzaguirre, http://dbpedia.org/resource/Stephen_Schwartz_(diplomat), http://dbpedia.org/resource/Danielle_C._Gray, http://dbpedia.org/resource/Stuart_E._Jones, http://dbpedia.org/resource/Tommy_Sowers, http://dbpedia.org/resource/Meredith_Attwell_Baker, http://dbpedia.org/resource/César_Benito_Cabrera, http://dbpedia.org/resource/David_Gompert, http://dbpedia.org/resource/Dwight_L._Bush_Sr., http://dbpedia.org/resource/Bathsheba_Nell_Crocker, http://dbpedia.org/resource/Howard_M._Radzely, http://dbpedia.org/resource/John_Caulfield_(diplomat), http://dbpedia.org/resource/Brian_Mosteller, http://dbpedia.org/resource/Judith_Fergin, http://dbpedia.org/resource/Rajesh_De, http://dbpedia.org/resource/Allan_J._Katz, http://dbpedia.org/resource/Thomas_F._Stephenson, http://dbpedia.org/resource/Franklin_R._Parker, http://dbpedia.org/resource/Jo-Ellen_Darcy, http://dbpedia.org/resource/Anne_S._Ferro, http://dbpedia.org/resource/Anita_Dunn, http://dbpedia.org/resource/David_Axelrod, http://dbpedia.org/resource/Dennis_C._Blair, http://dbpedia.org/resource/James_M._Cole, http://dbpedia.org/resource/John_C._Inglis, http://dbpedia.org/resource/Mark_F._Giuliano, http://dbpedia.org/resource/Randy_Babbitt, http://dbpedia.org/resource/Richard_Stengel, http://dbpedia.org/resource/Robert_Califf, http://dbpedia.org/resource/Stephen_Ostroff, http://dbpedia.org/resource/Cameron_Munter, http://dbpedia.org/resource/John_O._Brennan, http://dbpedia.org/resource/Pete_Geren, http://dbpedia.org/resource/Tom_Wheeler, http://dbpedia.org/resource/Christopher_R._Hill, http://dbpedia.org/resource/Janet_Napolitano, http://dbpedia.org/resource/John_Boehner, http://dbpedia.org/resource/Kathleen_Sebelius, http://dbpedia.org/resource/Mark_Lippert, http://dbpedia.org/resource/Thomas_E._Donilon, http://dbpedia.org/resource/Douglas_Kmiec, http://dbpedia.org/resource/Inez_Tenenbaum, http://dbpedia.org/resource/Neal_Katyal, http://dbpedia.org/resource/Robert_Malley, http://dbpedia.org/resource/Arturo_Valenzuela, http://dbpedia.org/resource/Robert_Groves, http://dbpedia.org/resource/David_Plouffe, http://dbpedia.org/resource/Jennifer_Palmieri, http://dbpedia.org/resource/Marc_Grossman, http://dbpedia.org/resource/Bruce_Reed_(political_operative), http://dbpedia.org/resource/Wendy_J._Olson, http://dbpedia.org/resource/David_D._Pearce, http://dbpedia.org/resource/Jon_Favreau_(speechwriter), http://dbpedia.org/resource/Linda_Thomas-Greenfield, http://dbpedia.org/resource/Rufus_Gifford, http://dbpedia.org/resource/Sarah_Sewall, http://dbpedia.org/resource/Jami_Miscik, http://dbpedia.org/resource/Diana_Farrell, http://dbpedia.org/resource/Andrew_Saul, http://dbpedia.org/resource/Elizabeth_Sherwood-Randall, http://dbpedia.org/resource/Gayle_Smith, http://dbpedia.org/resource/Rose_Gottemoeller, http://dbpedia.org/resource/Tina_Tchen, http://dbpedia.org/resource/Todd_Stern, http://dbpedia.org/resource/Bruce_A._Carlson, http://dbpedia.org/resource/Ann_Marie_Buerkle, http://dbpedia.org/resource/William_Kennard, http://dbpedia.org/resource/Vilma_Socorro_Martínez, http://dbpedia.org/resource/Broderick_D._Johnson, http://dbpedia.org/resource/James_F._Moriarty, http://dbpedia.org/resource/Luis_E._Arreaga, http://dbpedia.org/resource/Michael_H._Decker, http://dbpedia.org/resource/Barry_B._White, http://dbpedia.org/resource/Michael_Leiter, http://dbpedia.org/resource/Bob_Perciasepe, http://dbpedia.org/resource/Heather_Zichal, http://dbpedia.org/resource/Kurt_Volker, http://dbpedia.org/resource/Dennis_Hightower, http://dbpedia.org/resource/Derek_Chollet, http://dbpedia.org/resource/Makila_James, http://dbpedia.org/resource/Pamela_J._H._Slutz, http://dbpedia.org/resource/Reggie_Love, http://dbpedia.org/resource/Priscilla_Guthrie, http://dbpedia.org/resource/Robert_F._Hale, http://dbpedia.org/resource/Joe_Main, http://dbpedia.org/resource/Joseph_C._Szabo, http://dbpedia.org/resource/Dan_Mozena, http://dbpedia.org/resource/Anita_Decker_Breckenridge, http://dbpedia.org/resource/Kristie_Canegallo, http://dbpedia.org/resource/Rob_Nabors, http://dbpedia.org/resource/Joseph_Macmanus, http://dbpedia.org/resource/Kurt_Tong, http://dbpedia.org/resource/Shailagh_Murray, http://dbpedia.org/resource/Charles_E._Samuels,_Jr., http://dbpedia.org/resource/J._Christopher_Giancarlo, http://dbpedia.org/resource/Gustavo_Velasquez, http://dbpedia.org/resource/Dominique_Dawes, http://dbpedia.org/resource/William_K._Reilly, http://dbpedia.org/resource/David_Michaels_(epidemiologist), http://dbpedia.org/resource/James_H._Billington, http://dbpedia.org/resource/Jon_Leibowitz, http://dbpedia.org/resource/Paul_Ryan, http://dbpedia.org/resource/Stuart_E._Eizenstat, http://dbpedia.org/resource/Ben_Rhodes_(White_House_staffer), http://dbpedia.org/resource/Gene_Cretz, http://dbpedia.org/resource/Jim_Leach, http://dbpedia.org/resource/Jeh_Johnson, http://dbpedia.org/resource/Julian_Castro, http://dbpedia.org/resource/Wendy_Sherman, http://dbpedia.org/resource/Christopher_Dell, http://dbpedia.org/resource/Daniel_Poneman, http://dbpedia.org/resource/Fred_Hochberg, http://dbpedia.org/resource/Michael_McFaul, http://dbpedia.org/resource/Allison_Macfarlane, http://dbpedia.org/resource/Colleen_Bell, http://dbpedia.org/resource/Howard_Gutman, http://dbpedia.org/resource/Larry_Leon_Palmer, http://dbpedia.org/resource/Mythili_Raman, http://dbpedia.org/resource/Neal_S._Wolin, http://dbpedia.org/resource/Noah_Mamet, http://dbpedia.org/resource/Stuart_A._Levey, http://dbpedia.org/resource/Craig_Roberts_Stapleton, http://dbpedia.org/resource/Lawrence_Romo, http://dbpedia.org/resource/Gregory_B._Starr, http://dbpedia.org/resource/Isobel_Coleman, http://dbpedia.org/resource/Paul_W._Jones, http://dbpedia.org/resource/Stephen_Friedman_(PFIAB), http://dbpedia.org/resource/John_R._Phillips_(attorney), http://dbpedia.org/resource/Richard_Verma, http://dbpedia.org/resource/Charles_A._Ray, http://dbpedia.org/resource/Roberta_S._Jacobson, http://dbpedia.org/resource/Catherine_Ebert-Gray, http://dbpedia.org/resource/Eduardo_Aguirre, http://dbpedia.org/resource/Hugo_Llorens, http://dbpedia.org/resource/Sonny_Ramaswamy, http://dbpedia.org/resource/Jonathan_Jarvis, http://dbpedia.org/resource/David_J._Hayes, http://dbpedia.org/resource/Michael_C._Ormsby, http://dbpedia.org/resource/Robert_Bauer, http://dbpedia.org/resource/Betsey_Stevenson, http://dbpedia.org/resource/Douglas_Frantz, http://dbpedia.org/resource/W._Scott_Gould, http://dbpedia.org/resource/James_Franklin_Jeffrey, http://dbpedia.org/resource/Marcie_Berman_Ries, http://dbpedia.org/resource/Patrick_Duddy, http://dbpedia.org/resource/Philip_Goldberg, http://dbpedia.org/resource/Richard_Miles_(diplomat), http://dbpedia.org/resource/Tom_Strickland, http://dbpedia.org/resource/Vivek_Kundra, http://dbpedia.org/resource/Rosemary_DiCarlo, http://dbpedia.org/resource/Boris_Bershteyn, http://dbpedia.org/resource/Christopher_Scolese, http://dbpedia.org/resource/Cecilia_Muñoz, http://dbpedia.org/resource/Martha_N._Johnson, http://dbpedia.org/resource/Jon_Huntsman_Jr., http://dbpedia.org/resource/Michael_A._Hammer, http://dbpedia.org/resource/Jon_Wellinghoff, http://dbpedia.org/resource/Thomas_J._Perrelli, http://dbpedia.org/resource/Ronald_P._Spogli, http://dbpedia.org/resource/Thomas_R._Lamont, http://dbpedia.org/resource/Beth_Cobert, http://dbpedia.org/resource/Eric_J._Boswell, http://dbpedia.org/resource/David_Adelman, http://dbpedia.org/resource/Eileen_Donahoe, http://dbpedia.org/resource/James_B._Warlick,_Jr., http://dbpedia.org/resource/Miguel_H._Díaz, http://dbpedia.org/resource/Fay_Hartog-Levin, http://dbpedia.org/resource/Wanda_Nesbitt, http://dbpedia.org/resource/Desirée_Rogers, http://dbpedia.org/resource/Nicole_Lurie, http://dbpedia.org/resource/Laurence_Pope, http://dbpedia.org/resource/Thomas_M._Countryman, http://dbpedia.org/resource/Phillip_Carter_(ambassador), http://dbpedia.org/resource/Luis_CdeBaca, http://dbpedia.org/resource/Cynthia_L._Quarterman, http://dbpedia.org/resource/Leocadia_I._Zak, http://dbpedia.org/resource/Robert_J._Papp_Jr., http://dbpedia.org/resource/Booth_Goodwin, http://dbpedia.org/resource/Donald_B._Verrilli_Jr., http://dbpedia.org/resource/Maura_O'Neill, http://dbpedia.org/resource/John_Koenig_(diplomat), http://dbpedia.org/resource/Brad_Carson, http://dbpedia.org/resource/Daniel_Meltzer, http://dbpedia.org/resource/Howard_Koh, http://dbpedia.org/resource/Jay_Carney, http://dbpedia.org/resource/Ken_Salazar, http://dbpedia.org/resource/Patti_B._Saris, http://dbpedia.org/resource/Samantha_Power, http://dbpedia.org/resource/Christopher_A._Hart, http://dbpedia.org/resource/Keith_B._Alexander, http://dbpedia.org/resource/Michael_S._Rogers, http://dbpedia.org/resource/Richard_Ledgett, http://dbpedia.org/resource/Valerie_Jarrett, http://dbpedia.org/resource/Angella_Reid, http://dbpedia.org/resource/Avril_Haines, http://dbpedia.org/resource/Cameron_Kerry, http://dbpedia.org/resource/Chris_Lu, http://dbpedia.org/resource/Christy_Goldfuss, http://dbpedia.org/resource/David_Pressman, http://dbpedia.org/resource/Donald_C._Winter, http://dbpedia.org/resource/Howard_Shelanski, http://dbpedia.org/resource/John_Bryson, http://dbpedia.org/resource/Mark_Filip, http://dbpedia.org/resource/Otto_J._Wolff, http://dbpedia.org/resource/Rosa_Gumataotao_Rios, http://dbpedia.org/resource/Sarah_Bloom_Raskin, http://dbpedia.org/resource/Alejandro_Mayorkas, http://dbpedia.org/resource/Aneesh_Chopra, http://dbpedia.org/resource/Joseph_Clancy_(Secret_Service), http://dbpedia.org/resource/Julia_Pierson, http://dbpedia.org/resource/Kathryn_Ruemmler, http://dbpedia.org/resource/Kevin_K._Washburn, http://dbpedia.org/resource/Mark_Brzezinski, http://dbpedia.org/resource/Mark_J._Sullivan, http://dbpedia.org/resource/Victor_Mendez, http://dbpedia.org/resource/Nick_Sinai, http://dbpedia.org/resource/Richard_Morningstar, http://dbpedia.org/resource/Daniel_Baer, http://dbpedia.org/resource/Don_Beyer, http://dbpedia.org/resource/A.T._Smith, http://dbpedia.org/resource/David_Huebner, http://dbpedia.org/resource/Michael_T._Flynn, http://dbpedia.org/resource/Nancy-Ann_DeParle, http://dbpedia.org/resource/Victor_Ashe, http://dbpedia.org/resource/Alaina_B._Teplitz, http://dbpedia.org/resource/Alexa_L._Wesner, http://dbpedia.org/resource/Alice_Wells, http://dbpedia.org/resource/Amos_Hochstein, http://dbpedia.org/resource/Amy_J._Hyatt, http://dbpedia.org/resource/Andrew_H._Schapiro, http://dbpedia.org/resource/Andrew_O'Brien_(politician), http://dbpedia.org/resource/Arnold_A._Chacón, http://dbpedia.org/resource/Atul_Keshap, http://dbpedia.org/resource/Azita_Raji, http://dbpedia.org/resource/Bernard_W._Aronson, http://dbpedia.org/resource/Bonnie_Jenkins, http://dbpedia.org/resource/Brett_H._McGurk, http://dbpedia.org/resource/Brian_A._Nichols, http://dbpedia.org/resource/Carlos_R._Moreno, http://dbpedia.org/resource/Catherine_M._Russell, http://dbpedia.org/resource/Christine_A._Elder, http://dbpedia.org/resource/Clifford_Hart, http://dbpedia.org/resource/Crystal_Nix-Hines, http://dbpedia.org/resource/Cynthia_Akuetteh, http://dbpedia.org/resource/Dana_Shell_Smith, http://dbpedia.org/resource/Daniel_B._Shapiro, http://dbpedia.org/resource/Daniel_Fried, http://dbpedia.org/resource/Daniel_Rubinstein, http://dbpedia.org/resource/Daniel_Yohannes, http://dbpedia.org/resource/David_Hale_(diplomat), http://dbpedia.org/resource/David_J._Lane_(ambassador), http://dbpedia.org/resource/David_McKean_(diplomat), http://dbpedia.org/resource/David_Saperstein_(rabbi), http://dbpedia.org/resource/David_Thorne_(diplomat), http://dbpedia.org/resource/Deborah_L._Birx, http://dbpedia.org/resource/Deborah_R._Malac, http://dbpedia.org/resource/Denise_Bauer, http://dbpedia.org/resource/Donald_E._Booth, http://dbpedia.org/resource/Donald_Lu, http://dbpedia.org/resource/Douglas_Lute, http://dbpedia.org/resource/Elisabeth_I._Millard, http://dbpedia.org/resource/Elizabeth_Holzhall_Richard, http://dbpedia.org/resource/Erica_Barks-Ruggles, http://dbpedia.org/resource/Eunice_S._Reddick, http://dbpedia.org/resource/G._Kathleen_Hill, http://dbpedia.org/resource/Geoffrey_R._Pyatt, http://dbpedia.org/resource/Glyn_T._Davies, http://dbpedia.org/resource/Hans_G._Klemm, http://dbpedia.org/resource/Harry_K._Thomas,_Jr., http://dbpedia.org/resource/Helen_La_Lime, http://dbpedia.org/resource/Ian_C._Kelly, http://dbpedia.org/resource/Ira_Forman, http://dbpedia.org/resource/James_C._Swan, http://dbpedia.org/resource/James_Costos, http://dbpedia.org/resource/James_Knight_(diplomat), http://dbpedia.org/resource/James_P._Zumwalt, http://dbpedia.org/resource/Jane_D._Hartley, http://dbpedia.org/resource/Jean_Elizabeth_Manes, http://dbpedia.org/resource/Jeffrey_DeLaurentis, http://dbpedia.org/resource/Jennifer_Zimdahl_Galt, http://dbpedia.org/resource/Jerry_P._Lanier, http://dbpedia.org/resource/Joan_A._Polaschik, http://dbpedia.org/resource/John_B._Emerson, http://dbpedia.org/resource/John_D._Feeley, http://dbpedia.org/resource/John_F._Tefft, http://dbpedia.org/resource/John_L._Estrada, http://dbpedia.org/resource/John_R._Bass, http://dbpedia.org/resource/Jonathan_Finer, http://dbpedia.org/resource/Joseph_W._Westphal, http://dbpedia.org/resource/Joseph_Y._Yun, http://dbpedia.org/resource/Judith_Beth_Cefkin, http://dbpedia.org/resource/Judith_Heumann, http://dbpedia.org/resource/Julie_Furuta-Toy, http://dbpedia.org/resource/Julieta_Valls_Noyes, http://dbpedia.org/resource/Karen_Clark_Stanton, http://dbpedia.org/resource/Kathleen_A._Doherty, http://dbpedia.org/resource/Keith_Harper_(lawyer), http://dbpedia.org/resource/Kelly_Keiderling, http://dbpedia.org/resource/Ken_Hackett, http://dbpedia.org/resource/Kenneth_H._Merten, http://dbpedia.org/resource/Kevin_Whitaker, http://dbpedia.org/resource/Kirk_Wagar, http://dbpedia.org/resource/Lee_S._Wolosky, http://dbpedia.org/resource/Leslie_A._Bassett, http://dbpedia.org/resource/Linda_Swartz_Taglialatela, http://dbpedia.org/resource/Lisa_J._Peterson, http://dbpedia.org/resource/Lucy_Tamlyn, http://dbpedia.org/resource/Macon_Phillips, http://dbpedia.org/resource/Marcia_Bernicat, http://dbpedia.org/resource/Margaret_A._Uyehara, http://dbpedia.org/resource/Marie_L._Yovanovitch, http://dbpedia.org/resource/Mark_B._Childress, http://dbpedia.org/resource/Mark_Gilbert, http://dbpedia.org/resource/Mary_Catherine_Phee, http://dbpedia.org/resource/Matthew_H._Tueller, http://dbpedia.org/resource/Maureen_Cormack, http://dbpedia.org/resource/Michael_Ratney, http://dbpedia.org/resource/Michele_J._Sison, http://dbpedia.org/resource/Nancy_McEldowney, http://dbpedia.org/resource/Nancy_Pettit, http://dbpedia.org/resource/Nancy_Stetson, http://dbpedia.org/resource/P._Michael_McKinley, http://dbpedia.org/resource/Pamela_Hamamoto, http://dbpedia.org/resource/Patricia_Alsup, http://dbpedia.org/resource/Patrick_Gaspard, http://dbpedia.org/resource/Randy_W._Berry, http://dbpedia.org/resource/Richard_G._Olson, http://dbpedia.org/resource/Richard_M._Mills,_Jr., http://dbpedia.org/resource/Robert_Orr_(executive), http://dbpedia.org/resource/Robert_P._Jackson, http://dbpedia.org/resource/Robert_R._King, http://dbpedia.org/resource/Robert_S._Beecroft, http://dbpedia.org/resource/Robert_T._Yamate, http://dbpedia.org/resource/Samuel_D._Heins, http://dbpedia.org/resource/Sarah_E._Mendelson, http://dbpedia.org/resource/Scott_Rauland, http://dbpedia.org/resource/Shari_Villarosa, http://dbpedia.org/resource/Sheila_Gwaltney, http://dbpedia.org/resource/Stephanie_S._Sullivan, http://dbpedia.org/resource/Sung_Kim, http://dbpedia.org/resource/Susan_P._Coppedge, http://dbpedia.org/resource/Susan_S._Jacobs, http://dbpedia.org/resource/Terence_McCulley, http://dbpedia.org/resource/Thomas_C._Krajeski, http://dbpedia.org/resource/Tina_Kaidanow, http://dbpedia.org/resource/Todd_C._Chapman, http://dbpedia.org/resource/Tom_Perriello, http://dbpedia.org/resource/Virginia_E._Palmer, http://dbpedia.org/resource/W._Stuart_Symington_(diplomat), http://dbpedia.org/resource/Wally_Brewster, http://dbpedia.org/resource/William_A._Heidt, http://dbpedia.org/resource/Matthew_Bryza, http://dbpedia.org/resource/Arun_Majumdar, http://dbpedia.org/resource/John_Kirby_(admiral), http://dbpedia.org/resource/Brian_C._Moyer, http://dbpedia.org/resource/Frank_Sanchez, http://dbpedia.org/resource/Janine_A._Davidson, http://dbpedia.org/resource/Mark_Doms, http://dbpedia.org/resource/Jane_Holl_Lute, http://dbpedia.org/resource/James_B._Smith, http://dbpedia.org/resource/Melanne_Verveer, http://dbpedia.org/resource/Jonathan_D._Farrar, http://dbpedia.org/resource/Rocco_Landesman, http://dbpedia.org/resource/Raj_Date, http://dbpedia.org/resource/Winslow_Sargeant, http://dbpedia.org/resource/Conner_Eldridge, http://dbpedia.org/resource/Christopher_A._Kojm, http://dbpedia.org/resource/Stephen_Rapp, http://dbpedia.org/resource/Cheryl_Mills, http://dbpedia.org/resource/Tony_West_(attorney), http://dbpedia.org/resource/Alfonso_E._Lenhardt, http://dbpedia.org/resource/Edward_C._Hugler, http://dbpedia.org/resource/Seth_Harris, http://dbpedia.org/resource/Bill_Corr, http://dbpedia.org/resource/John_Koskinen, http://dbpedia.org/resource/Michèle_Flournoy, http://dbpedia.org/resource/Nisha_Desai_Biswal, http://dbpedia.org/resource/Rashad_Hussain, http://dbpedia.org/resource/C._Steven_McGann, http://dbpedia.org/resource/Frankie_Reed, http://dbpedia.org/resource/Charles_E._Johnson_(government_official), http://dbpedia.org/resource/Erin_C._Conaton, http://dbpedia.org/resource/John_G._Levi, http://dbpedia.org/resource/William_K._Sessions_III, http://dbpedia.org/resource/M._Patricia_Smith, http://dbpedia.org/resource/Vanita_Gupta, http://dbpedia.org/resource/Esther_Brimmer, http://dbpedia.org/resource/Pamela_L._Spratlen, http://dbpedia.org/resource/Bruce_J._Oreck, http://dbpedia.org/resource/Daniel_Pfeiffer, http://dbpedia.org/resource/John_King_Jr., http://dbpedia.org/resource/Lisa_Monaco, http://dbpedia.org/resource/Adam_Namm, http://dbpedia.org/resource/Dennis_M._McCarthy, http://dbpedia.org/resource/Alan_Bersin, http://dbpedia.org/resource/Phil_Schiliro, http://dbpedia.org/resource/Tom_Malinowski, http://dbpedia.org/resource/Jeffrey_D._Levine, http://dbpedia.org/resource/Jesse_Lee_(politician), http://dbpedia.org/resource/Robert_J._Dieter, http://dbpedia.org/resource/Ronald_Rodgers, http://dbpedia.org/resource/Warren_F._(Pete)_Miller,_Jr., http://dbpedia.org/resource/Ertharin_Cousin, http://dbpedia.org/resource/Jeanne_Lambrew, http://dbpedia.org/resource/William_B._Taylor,_Jr., http://dbpedia.org/resource/B._Todd_Jones, http://dbpedia.org/resource/David_Ferriero, http://dbpedia.org/resource/John_Roos, http://dbpedia.org/resource/Michael_L._Connor, http://dbpedia.org/resource/Vivek_Murthy, http://dbpedia.org/resource/James_R._Clapper, http://dbpedia.org/resource/Martin_J._Gruenberg, http://dbpedia.org/resource/Stephen_G._Burns, http://dbpedia.org/resource/Peter_V._Neffenger, http://dbpedia.org/resource/Carolyn_Colvin, http://dbpedia.org/resource/Kin_Moy_(diplomat), http://dbpedia.org/resource/Steven_K._Galson, http://dbpedia.org/resource/Carla_Hayden, http://dbpedia.org/resource/Cynthia_L._Bauerly, http://dbpedia.org/resource/Jerry_Abramson, http://dbpedia.org/resource/Elaine_D._Kaplan, http://dbpedia.org/resource/Deborah_Hersman, http://dbpedia.org/resource/Michael_B._Donley, http://dbpedia.org/resource/Sally_Yates, http://dbpedia.org/resource/Donald_Laverdure, http://dbpedia.org/resource/Janice_Eberly, http://dbpedia.org/resource/David_S._Mao, http://dbpedia.org/resource/Timothy_P._Murphy, http://dbpedia.org/resource/Julianna_Smoot, http://dbpedia.org/resource/Alyssa_Mastromonaco, http://dbpedia.org/resource/Dan_Tangherlini, http://dbpedia.org/resource/David_S._Kris, http://dbpedia.org/resource/Maria_Otero, http://dbpedia.org/resource/Tulinabo_S._Mushingi, http://dbpedia.org/resource/Florence_T._Nakakuni, http://dbpedia.org/resource/Michele_Leonhart, http://dbpedia.org/resource/A._Thomas_McLellan, http://dbpedia.org/resource/Marco_Lopez, http://dbpedia.org/resource/Anne_E._Derse, http://dbpedia.org/resource/Ellen_Moran, http://dbpedia.org/resource/Jacob_Walles, http://dbpedia.org/resource/Robert_Stephen_Ford, http://dbpedia.org/resource/Tracey_Ann_Jacobson, http://dbpedia.org/resource/Earl_Anthony_Wayne, http://dbpedia.org/resource/Cheryl_Martin, http://dbpedia.org/resource/Nicole_Wong, http://dbpedia.org/resource/Megan_Smith, http://dbpedia.org/resource/Donald_Gips, http://dbpedia.org/resource/Gary_Samore, http://dbpedia.org/resource/Lisa_Brown_(lawyer), http://dbpedia.org/resource/Mari_Carmen_Aponte, http://dbpedia.org/resource/Daniel_R._Russel, http://dbpedia.org/resource/Mary_J._Miller, http://dbpedia.org/resource/Lisette_M._Mondello, http://dbpedia.org/resource/Esther_Olavarria, http://dbpedia.org/resource/Karen_W._Pane, http://dbpedia.org/resource/Phyllis_M._Powers, http://dbpedia.org/resource/James_H._Shelton, http://dbpedia.org/resource/Louis_J._Nigro,_Jr., http://dbpedia.org/resource/Eleni_Tsakopoulos_Kounalakis, http://dbpedia.org/resource/Sylvia_Baca, http://dbpedia.org/resource/Ajit_Varadaraj_Pai, http://dbpedia.org/resource/Robert_Hormats, http://dbpedia.org/resource/André_Birotte_Jr., http://dbpedia.org/resource/Peter_A._Selfridge, http://dbpedia.org/resource/William_Brownfield, http://dbpedia.org/resource/Charles_Rivkin, http://dbpedia.org/resource/Katie_Beirne_Fallon, http://dbpedia.org/resource/Donetta_Davidson, http://dbpedia.org/resource/Deborah_McCarthy, http://dbpedia.org/resource/Dorothea-Maria_Rosen, http://dbpedia.org/resource/Helen_Reed-Rowe, http://dbpedia.org/resource/Robert_Bradtke, http://dbpedia.org/resource/Alexander_Arvizu, http://dbpedia.org/resource/Stephen_Seche, http://dbpedia.org/resource/David_L._Carden, http://dbpedia.org/resource/Jake_Sullivan, http://dbpedia.org/resource/Matthew_G._Olsen, http://dbpedia.org/resource/Marilyn_Tavenner, http://dbpedia.org/resource/Margaret_Scobey, http://dbpedia.org/resource/Rose_M._Likins, http://dbpedia.org/resource/William_Braucher_Wood, http://dbpedia.org/resource/Todd_Park, http://dbpedia.org/resource/John_A._Heffern, http://dbpedia.org/resource/Norman_L._Eisen, http://dbpedia.org/resource/Richard_Schmierer, http://dbpedia.org/resource/Thomas_A._Shannon_Jr., http://dbpedia.org/resource/Jeanine_Jackson, http://dbpedia.org/resource/Mary_Beth_Maxwell, http://dbpedia.org/resource/Scott_F._Large, http://dbpedia.org/resource/Thomas_Dougherty, http://dbpedia.org/resource/James_F._Entwistle, http://dbpedia.org/resource/Adrienne_S._O'Neal, http://dbpedia.org/resource/Mary_Burce_Warlick, http://dbpedia.org/resource/Michael_Corbin, http://dbpedia.org/resource/Michael_David_Kirby, http://dbpedia.org/resource/Robert_A._Mandell, http://dbpedia.org/resource/Anne_Slaughter_Andrew, http://dbpedia.org/resource/Walter_E._North, http://dbpedia.org/resource/Charles_C._Adams_Jr., http://dbpedia.org/resource/Ryan_Panchadsaram, http://dbpedia.org/resource/Ramon_Lugo_III, http://dbpedia.org/resource/Andrew_Cuomo]... [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 132, P: 0.335, R: 0.420, macro F1: 0.339, macro F1*: 0.373 -[INFO] QALD Cumulative metrics, sample: 132, P: 0.555, R: 0.420, macro F1: 0.339, QALD macro F1: 0.478 -[INFO] Current time: 4135 ms, average: 3480.22 ms/ques, entity detection: 782.45 ms/ques (linking tool: 693.74 ms/ques), relation detection: 1110.36 ms/ques (similarity: 626.82 ms/ques), EDG generation: 48.08 ms/ques +[INFO] Cumulative metrics, sample: 132, P: 0.335, R: 0.420, macro F1: 0.373 +[INFO] QALD Cumulative metrics, sample: 132, P: 0.555, R: 0.420, average F1: 0.339, QALD macro F1: 0.478 +[INFO] Current question consumed time: 3602 ms, average: 3144.89 ms/ques, entity detection: 691.53 ms/ques (linking tool: 616.33 ms/ques), relation detection: 1033.18 ms/ques (similarity: 626.40 ms/ques), EDG generation: 49.29 ms/ques [INFO] Entity Linking P: 0.30303030303030304, R: 0.30303030303030304, micro F1: 0.30303030303030304, macro F1: 0.30303030303030304 [INFO] Relation Linking P: 0.3106060606060606, R: 0.3106060606060606, micro F1: 0.3106060606060606, macro F1: 0.3106060606060606 [INFO] Type Linking P: 0.8712121212121212, R: 0.8712121212121212, micro F1: 0.8712121212121212, macro F1: 0.8712121212121212 @@ -1987,9 +1987,9 @@ Question 132: Who are the parents of the wife of Juan Carlos I? [INFO] Golden Answer: [http://dbpedia.org/resource/Frederika_of_Hanover, http://dbpedia.org/resource/Paul_of_Greece] [INFO] Predicted Answer: [http://dbpedia.org/resource/Infanta_Cristina_of_Spain, http://dbpedia.org/resource/Infanta_Elena,_Duchess_of_Lugo, http://dbpedia.org/resource/Frederika_of_Hanover, http://dbpedia.org/resource/Paul_of_Greece] [INFO] P: 0.5, R: 1.0, F: 0.6666666666666666 -[INFO] Cumulative metrics, sample: 133, P: 0.336, R: 0.424, macro F1: 0.342, macro F1*: 0.375 -[INFO] QALD Cumulative metrics, sample: 133, P: 0.554, R: 0.424, macro F1: 0.342, QALD macro F1: 0.481 -[INFO] Current time: 1815 ms, average: 3467.70 ms/ques, entity detection: 777.89 ms/ques (linking tool: 689.61 ms/ques), relation detection: 1108.32 ms/ques (similarity: 625.78 ms/ques), EDG generation: 47.83 ms/ques +[INFO] Cumulative metrics, sample: 133, P: 0.336, R: 0.424, macro F1: 0.375 +[INFO] QALD Cumulative metrics, sample: 133, P: 0.554, R: 0.424, average F1: 0.342, QALD macro F1: 0.481 +[INFO] Current question consumed time: 1522 ms, average: 3132.68 ms/ques, entity detection: 687.34 ms/ques (linking tool: 612.52 ms/ques), relation detection: 1031.07 ms/ques (similarity: 625.38 ms/ques), EDG generation: 49.04 ms/ques [INFO] Entity Linking P: 0.3007518796992481, R: 0.3007518796992481, micro F1: 0.3007518796992481, macro F1: 0.3007518796992481 [INFO] Relation Linking P: 0.3082706766917293, R: 0.3082706766917293, micro F1: 0.3082706766917293, macro F1: 0.3082706766917293 [INFO] Type Linking P: 0.8721804511278195, R: 0.8721804511278195, micro F1: 0.8721804511278195, macro F1: 0.8721804511278195 @@ -2002,9 +2002,9 @@ Question 133: Who is the novelist of the work a song of ice and fire? [INFO] Golden Answer: [http://dbpedia.org/resource/George_R._R._Martin] [INFO] Predicted Answer: [http://dbpedia.org/resource/George_R._R._Martin] [INFO] P: 1.0, R: 1.0, F: 1.0 -[INFO] Cumulative metrics, sample: 134, P: 0.341, R: 0.429, macro F1: 0.347, macro F1*: 0.380 -[INFO] QALD Cumulative metrics, sample: 134, P: 0.558, R: 0.429, macro F1: 0.347, QALD macro F1: 0.485 -[INFO] Current time: 507 ms, average: 3445.60 ms/ques, entity detection: 772.28 ms/ques (linking tool: 684.58 ms/ques), relation detection: 1101.60 ms/ques (similarity: 622.04 ms/ques), EDG generation: 47.57 ms/ques +[INFO] Cumulative metrics, sample: 134, P: 0.341, R: 0.429, macro F1: 0.380 +[INFO] QALD Cumulative metrics, sample: 134, P: 0.558, R: 0.429, average F1: 0.347, QALD macro F1: 0.485 +[INFO] Current question consumed time: 421 ms, average: 3112.46 ms/ques, entity detection: 682.35 ms/ques (linking tool: 608.04 ms/ques), relation detection: 1024.71 ms/ques (similarity: 621.61 ms/ques), EDG generation: 48.77 ms/ques [INFO] Entity Linking P: 0.29850746268656714, R: 0.29850746268656714, micro F1: 0.29850746268656714, macro F1: 0.29850746268656714 [INFO] Relation Linking P: 0.30597014925373134, R: 0.30597014925373134, micro F1: 0.30597014925373134, macro F1: 0.30597014925373134 [INFO] Type Linking P: 0.8731343283582089, R: 0.8731343283582089, micro F1: 0.8731343283582089, macro F1: 0.8731343283582089 @@ -2017,9 +2017,9 @@ Question 134: In which studio did the Beatles record their first album? [INFO] Golden Answer: [EMI Studios, London] [INFO] Predicted Answer: [EMI and Trident Studios, London] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 135, P: 0.339, R: 0.425, macro F1: 0.344, macro F1*: 0.377 -[INFO] QALD Cumulative metrics, sample: 135, P: 0.553, R: 0.425, macro F1: 0.344, QALD macro F1: 0.481 -[INFO] Current time: 3258 ms, average: 3444.21 ms/ques, entity detection: 768.51 ms/ques (linking tool: 681.31 ms/ques), relation detection: 1102.45 ms/ques (similarity: 626.38 ms/ques), EDG generation: 47.33 ms/ques +[INFO] Cumulative metrics, sample: 135, P: 0.339, R: 0.425, macro F1: 0.377 +[INFO] QALD Cumulative metrics, sample: 135, P: 0.553, R: 0.425, average F1: 0.344, QALD macro F1: 0.481 +[INFO] Current question consumed time: 2791 ms, average: 3110.07 ms/ques, entity detection: 678.88 ms/ques (linking tool: 605.01 ms/ques), relation detection: 1024.96 ms/ques (similarity: 626.02 ms/ques), EDG generation: 48.53 ms/ques [INFO] Entity Linking P: 0.2962962962962963, R: 0.2962962962962963, micro F1: 0.2962962962962963, macro F1: 0.2962962962962963 [INFO] Relation Linking P: 0.3037037037037037, R: 0.3037037037037037, micro F1: 0.3037037037037037, macro F1: 0.3037037037037037 [INFO] Type Linking P: 0.8740740740740741, R: 0.8740740740740741, micro F1: 0.8740740740740741, macro F1: 0.8740740740740742 @@ -2032,9 +2032,9 @@ Question 135: Which beer brewing companies are located in North-Rhine Westphalia [INFO] Golden Answer: [http://dbpedia.org/resource/Veltins, http://dbpedia.org/resource/Warsteiner, http://dbpedia.org/resource/Krombacher_Brauerei, http://dbpedia.org/resource/Diebels] [INFO] Predicted Answer: [] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 136, P: 0.336, R: 0.422, macro F1: 0.342, macro F1*: 0.374 -[INFO] QALD Cumulative metrics, sample: 136, P: 0.557, R: 0.422, macro F1: 0.342, QALD macro F1: 0.480 -[INFO] Current time: 1511 ms, average: 3430.00 ms/ques, entity detection: 767.23 ms/ques (linking tool: 679.36 ms/ques), relation detection: 1094.82 ms/ques (similarity: 622.08 ms/ques), EDG generation: 47.08 ms/ques +[INFO] Cumulative metrics, sample: 136, P: 0.336, R: 0.422, macro F1: 0.374 +[INFO] QALD Cumulative metrics, sample: 136, P: 0.557, R: 0.422, average F1: 0.342, QALD macro F1: 0.480 +[INFO] Current question consumed time: 1170 ms, average: 3095.81 ms/ques, entity detection: 677.35 ms/ques (linking tool: 603.07 ms/ques), relation detection: 1017.79 ms/ques (similarity: 621.69 ms/ques), EDG generation: 48.27 ms/ques [INFO] Entity Linking P: 0.3014705882352941, R: 0.3014705882352941, micro F1: 0.3014705882352941, macro F1: 0.3014705882352941 [INFO] Relation Linking P: 0.3088235294117647, R: 0.3088235294117647, micro F1: 0.3088235294117647, macro F1: 0.3088235294117647 [INFO] Type Linking P: 0.875, R: 0.875, micro F1: 0.875, macro F1: 0.875 @@ -2047,9 +2047,9 @@ Question 136: When did princess Diana die? [INFO] Golden Answer: [1997-08-31] [INFO] Predicted Answer: [] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 137, P: 0.334, R: 0.419, macro F1: 0.339, macro F1*: 0.372 -[INFO] QALD Cumulative metrics, sample: 137, P: 0.560, R: 0.419, macro F1: 0.339, QALD macro F1: 0.479 -[INFO] Current time: 913 ms, average: 3411.63 ms/ques, entity detection: 763.13 ms/ques (linking tool: 675.63 ms/ques), relation detection: 1088.50 ms/ques (similarity: 618.53 ms/ques), EDG generation: 46.79 ms/ques +[INFO] Cumulative metrics, sample: 137, P: 0.334, R: 0.419, macro F1: 0.372 +[INFO] QALD Cumulative metrics, sample: 137, P: 0.560, R: 0.419, average F1: 0.339, QALD macro F1: 0.479 +[INFO] Current question consumed time: 763 ms, average: 3078.78 ms/ques, entity detection: 673.70 ms/ques (linking tool: 599.75 ms/ques), relation detection: 1011.91 ms/ques (similarity: 618.15 ms/ques), EDG generation: 47.96 ms/ques [INFO] Entity Linking P: 0.30656934306569344, R: 0.30656934306569344, micro F1: 0.30656934306569344, macro F1: 0.30656934306569344 [INFO] Relation Linking P: 0.31386861313868614, R: 0.31386861313868614, micro F1: 0.31386861313868614, macro F1: 0.31386861313868614 [INFO] Type Linking P: 0.8759124087591241, R: 0.8759124087591241, micro F1: 0.8759124087591241, macro F1: 0.8759124087591241 @@ -2062,9 +2062,9 @@ Question 137: Who founded Intel? [INFO] Golden Answer: [http://dbpedia.org/resource/Gordon_Moore, http://dbpedia.org/resource/Robert_Noyce] [INFO] Predicted Answer: [http://dbpedia.org/resource/Alliance_for_Open_Media, http://dbpedia.org/resource/Trusted_Computing_Group] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 138, P: 0.331, R: 0.416, macro F1: 0.337, macro F1*: 0.369 -[INFO] QALD Cumulative metrics, sample: 138, P: 0.556, R: 0.416, macro F1: 0.337, QALD macro F1: 0.476 -[INFO] Current time: 1399 ms, average: 3397.04 ms/ques, entity detection: 759.41 ms/ques (linking tool: 671.92 ms/ques), relation detection: 1084.49 ms/ques (similarity: 616.09 ms/ques), EDG generation: 46.47 ms/ques +[INFO] Cumulative metrics, sample: 138, P: 0.331, R: 0.416, macro F1: 0.369 +[INFO] QALD Cumulative metrics, sample: 138, P: 0.556, R: 0.416, average F1: 0.337, QALD macro F1: 0.476 +[INFO] Current question consumed time: 1191 ms, average: 3065.10 ms/ques, entity detection: 670.33 ms/ques (linking tool: 596.42 ms/ques), relation detection: 1008.06 ms/ques (similarity: 615.71 ms/ques), EDG generation: 47.64 ms/ques [INFO] Entity Linking P: 0.30434782608695654, R: 0.30434782608695654, micro F1: 0.30434782608695654, macro F1: 0.30434782608695654 [INFO] Relation Linking P: 0.3115942028985507, R: 0.3115942028985507, micro F1: 0.3115942028985507, macro F1: 0.3115942028985507 [INFO] Type Linking P: 0.8768115942028986, R: 0.8768115942028986, micro F1: 0.8768115942028986, macro F1: 0.8768115942028986 @@ -2077,9 +2077,9 @@ Question 138: Who is the youngest player in the Premier League? [INFO] Golden Answer: [http://dbpedia.org/resource/Ernie_Pinkney] [INFO] Predicted Answer: [http://dbpedia.org/resource/1997–98_Chelsea_F.C._season, http://dbpedia.org/resource/2004–05_Chelsea_F.C._season, http://dbpedia.org/resource/2006–07_Chelsea_F.C._season, http://dbpedia.org/resource/2011–12_Chelsea_F.C._season, http://dbpedia.org/resource/2013–14_Manchester_City_F.C._season, http://dbpedia.org/resource/2014–15_Chelsea_F.C._season, http://dbpedia.org/resource/1997–98_Derby_County_F.C._season, http://dbpedia.org/resource/1999–2000_Derby_County_F.C._season, http://dbpedia.org/resource/2000–01_Derby_County_F.C._season, http://dbpedia.org/resource/2001–02_Derby_County_F.C._season, http://dbpedia.org/resource/2007–08_Derby_County_F.C._season, http://dbpedia.org/resource/1994–95_Chelsea_F.C._season, http://dbpedia.org/resource/2005–06_Sunderland_A.F.C._season, http://dbpedia.org/resource/2005–06_Wigan_Athletic_F.C._season, http://dbpedia.org/resource/2008–09_West_Bromwich_Albion_F.C._season, http://dbpedia.org/resource/2004–05_West_Bromwich_Albion_F.C._season, http://dbpedia.org/resource/2008–09_Hull_City_A.F.C._season, http://dbpedia.org/resource/1995–96_Newcastle_United_F.C._season, http://dbpedia.org/resource/1996–97_Newcastle_United_F.C._season, http://dbpedia.org/resource/2015–16_A.F.C._Bournemouth_season, http://dbpedia.org/resource/1992–93_Chelsea_F.C._season, http://dbpedia.org/resource/1996–97_Middlesbrough_F.C._season, http://dbpedia.org/resource/1997–98_Arsenal_F.C._season, http://dbpedia.org/resource/2001–02_Arsenal_F.C._season, http://dbpedia.org/resource/2003–04_Arsenal_F.C._season, http://dbpedia.org/resource/2005–06_Arsenal_F.C._season, http://dbpedia.org/resource/2014–15_Arsenal_F.C._season, http://dbpedia.org/resource/2014–15_Aston_Villa_F.C._season, http://dbpedia.org/resource/1994–95_Blackburn_Rovers_F.C._season, http://dbpedia.org/resource/1995–96_Blackburn_Rovers_F.C._season, http://dbpedia.org/resource/1997–98_Newcastle_United_F.C._season, http://dbpedia.org/resource/1998–99_Newcastle_United_F.C._season, http://dbpedia.org/resource/2000–01_Newcastle_United_F.C._season, http://dbpedia.org/resource/2002–03_Newcastle_United_F.C._season, http://dbpedia.org/resource/2006–07_Watford_F.C._season, http://dbpedia.org/resource/2014–15_Queens_Park_Rangers_F.C._season, http://dbpedia.org/resource/2002–03_Everton_F.C._season, http://dbpedia.org/resource/2003–04_Everton_F.C._season, http://dbpedia.org/resource/2014–15_Manchester_United_F.C._season, http://dbpedia.org/resource/2015–16_Manchester_United_F.C._season, http://dbpedia.org/resource/2004–05_Everton_F.C._season, http://dbpedia.org/resource/2005–06_Everton_F.C._season, http://dbpedia.org/resource/2006–07_Everton_F.C._season, http://dbpedia.org/resource/2007–08_Everton_F.C._season, http://dbpedia.org/resource/2005–06_Blackburn_Rovers_F.C._season, http://dbpedia.org/resource/2006–07_Blackburn_Rovers_F.C._season, http://dbpedia.org/resource/2006–07_Portsmouth_F.C._season, http://dbpedia.org/resource/2007–08_Blackburn_Rovers_F.C._season, http://dbpedia.org/resource/2007–08_Portsmouth_F.C._season, http://dbpedia.org/resource/2008–09_Blackburn_Rovers_F.C._season, http://dbpedia.org/resource/2009–10_Blackburn_Rovers_F.C._season, http://dbpedia.org/resource/2009–10_Portsmouth_F.C._season, http://dbpedia.org/resource/2010–11_Blackburn_Rovers_F.C._season, http://dbpedia.org/resource/1993–94_Liverpool_F.C._season, http://dbpedia.org/resource/1994–95_Liverpool_F.C._season, http://dbpedia.org/resource/1995–96_Liverpool_F.C._season, http://dbpedia.org/resource/1996–97_Southampton_F.C._season, http://dbpedia.org/resource/2002–03_Fulham_F.C._season, http://dbpedia.org/resource/2010–11_Birmingham_City_F.C._season, http://dbpedia.org/resource/2011–12_Queens_Park_Rangers_F.C._season, http://dbpedia.org/resource/1999–2000_Leeds_United_A.F.C._season, http://dbpedia.org/resource/2006–07_Middlesbrough_F.C._season, http://dbpedia.org/resource/2016–17_Sunderland_A.F.C._season, http://dbpedia.org/resource/2001–02_Aston_Villa_F.C._season, http://dbpedia.org/resource/2002–03_Aston_Villa_F.C._season, http://dbpedia.org/resource/2012–13_Arsenal_F.C._season, http://dbpedia.org/resource/1992–93_Middlesbrough_F.C._season, http://dbpedia.org/resource/1995–96_Arsenal_F.C._season, http://dbpedia.org/resource/1999–2000_Arsenal_F.C._season, http://dbpedia.org/resource/2000–01_Arsenal_F.C._season, http://dbpedia.org/resource/2002–03_Arsenal_F.C._season, http://dbpedia.org/resource/2001–02_West_Ham_United_F.C._season, http://dbpedia.org/resource/2008–09_Fulham_F.C._season, http://dbpedia.org/resource/2015–16_Aston_Villa_F.C._season, http://dbpedia.org/resource/1992–93_Everton_F.C._season, http://dbpedia.org/resource/2006–07_Tottenham_Hotspur_F.C._season, http://dbpedia.org/resource/1995–96_Tottenham_Hotspur_F.C._season, http://dbpedia.org/resource/1996–97_Tottenham_Hotspur_F.C._season, http://dbpedia.org/resource/2000–01_Tottenham_Hotspur_F.C._season, http://dbpedia.org/resource/2009–10_Arsenal_F.C._season, http://dbpedia.org/resource/2013–14_Arsenal_F.C._season, http://dbpedia.org/resource/2013–14_Fulham_F.C._season, http://dbpedia.org/resource/2009–10_Sunderland_A.F.C._season, http://dbpedia.org/resource/2010–11_Sunderland_A.F.C._season, http://dbpedia.org/resource/2011–12_Sunderland_A.F.C._season, http://dbpedia.org/resource/2000–01_Aston_Villa_F.C._season, http://dbpedia.org/resource/2005–06_Tottenham_Hotspur_F.C._season, http://dbpedia.org/resource/2011–12_West_Bromwich_Albion_F.C._season, http://dbpedia.org/resource/2012–13_West_Bromwich_Albion_F.C._season, http://dbpedia.org/resource/2013–14_West_Bromwich_Albion_F.C._season, http://dbpedia.org/resource/2015–16_Southampton_F.C._season, http://dbpedia.org/resource/2016–17_Burnley_F.C._season, http://dbpedia.org/resource/2016–17_Everton_F.C._season, http://dbpedia.org/resource/2009–10_Wolverhampton_Wanderers_F.C._season, http://dbpedia.org/resource/2011–12_Wolverhampton_Wanderers_F.C._season, http://dbpedia.org/resource/2014–15_Hull_City_A.F.C._season, http://dbpedia.org/resource/1993–94_West_Ham_United_F.C._season, http://dbpedia.org/resource/1995–96_Chelsea_F.C._season, http://dbpedia.org/resource/1996–97_Chelsea_F.C._season, http://dbpedia.org/resource/2016–17_Chelsea_F.C._season, http://dbpedia.org/resource/1998–99_Chelsea_F.C._season, http://dbpedia.org/resource/2002–03_Chelsea_F.C._season, http://dbpedia.org/resource/1994–95_Everton_F.C._season, http://dbpedia.org/resource/1995–96_Everton_F.C._season, http://dbpedia.org/resource/1996–97_Everton_F.C._season, http://dbpedia.org/resource/1997–98_Everton_F.C._season, http://dbpedia.org/resource/2000–01_Bradford_City_A.F.C._season, http://dbpedia.org/resource/2005–06_West_Bromwich_Albion_F.C._season, http://dbpedia.org/resource/2010–11_Wigan_Athletic_F.C._season, http://dbpedia.org/resource/2006–07_West_Ham_United_F.C._season, http://dbpedia.org/resource/2009–10_Burnley_F.C._season, http://dbpedia.org/resource/1998–99_Charlton_Athletic_F.C._season, http://dbpedia.org/resource/2000–01_Charlton_Athletic_F.C._season, http://dbpedia.org/resource/2001–02_Charlton_Athletic_F.C._season, http://dbpedia.org/resource/2002–03_Charlton_Athletic_F.C._season, http://dbpedia.org/resource/2003–04_Charlton_Athletic_F.C._season, http://dbpedia.org/resource/1994–95_Sheffield_Wednesday_F.C._season, http://dbpedia.org/resource/1995–96_Sheffield_Wednesday_F.C._season, http://dbpedia.org/resource/1996–97_Sheffield_Wednesday_F.C._season, http://dbpedia.org/resource/1995–96_Queens_Park_Rangers_F.C._season, http://dbpedia.org/resource/2016–17_Arsenal_F.C._season, http://dbpedia.org/resource/2015–16_Tottenham_Hotspur_F.C._season, http://dbpedia.org/resource/2011–12_Liverpool_F.C._season, http://dbpedia.org/resource/2005–06_West_Ham_United_F.C._season, http://dbpedia.org/resource/2014–15_West_Ham_United_F.C._season, http://dbpedia.org/resource/2015–16_West_Ham_United_F.C._season, http://dbpedia.org/resource/1995–96_Southampton_F.C._season, http://dbpedia.org/resource/1998–99_Southampton_F.C._season, http://dbpedia.org/resource/1999–2000_Southampton_F.C._season, http://dbpedia.org/resource/2000–01_Southampton_F.C._season, http://dbpedia.org/resource/2014–15_Southampton_F.C._season, http://dbpedia.org/resource/1992–93_Nottingham_Forest_F.C._season, http://dbpedia.org/resource/1999–2000_Middlesbrough_F.C._season, http://dbpedia.org/resource/2000–01_Everton_F.C._season, http://dbpedia.org/resource/1997–98_Sheffield_Wednesday_F.C._season, http://dbpedia.org/resource/2016–17_Leicester_City_F.C._season, http://dbpedia.org/resource/1994–95_Ipswich_Town_F.C._season, http://dbpedia.org/resource/2000–01_Ipswich_Town_F.C._season, http://dbpedia.org/resource/2014–15_Sunderland_A.F.C._season, http://dbpedia.org/resource/2013–14_Sunderland_A.F.C._season, http://dbpedia.org/resource/2008–09_Arsenal_F.C._season, http://dbpedia.org/resource/2010–11_Arsenal_F.C._season, http://dbpedia.org/resource/2004–05_Portsmouth_F.C._season, http://dbpedia.org/resource/2014–15_Crystal_Palace_F.C._season, http://dbpedia.org/resource/2001–02_Liverpool_F.C._season, http://dbpedia.org/resource/2004–05_Liverpool_F.C._season, http://dbpedia.org/resource/2005–06_Liverpool_F.C._season, http://dbpedia.org/resource/2006–07_Liverpool_F.C._season, http://dbpedia.org/resource/2007–08_Liverpool_F.C._season, http://dbpedia.org/resource/2008–09_Liverpool_F.C._season, http://dbpedia.org/resource/2009–10_Liverpool_F.C._season, http://dbpedia.org/resource/2010–11_Liverpool_F.C._season, http://dbpedia.org/resource/2013–14_Liverpool_F.C._season, http://dbpedia.org/resource/2014–15_Liverpool_F.C._season, http://dbpedia.org/resource/2015–16_Liverpool_F.C._season, http://dbpedia.org/resource/2002–03_Leeds_United_A.F.C._season, http://dbpedia.org/resource/2003–04_Leeds_United_A.F.C._season, http://dbpedia.org/resource/2005–06_Aston_Villa_F.C._season, http://dbpedia.org/resource/2014–15_Manchester_City_F.C._season, http://dbpedia.org/resource/2016–17_Liverpool_F.C._season, http://dbpedia.org/resource/1996–97_Liverpool_F.C._season, http://dbpedia.org/resource/1997–98_Liverpool_F.C._season, http://dbpedia.org/resource/1998–99_Liverpool_F.C._season, http://dbpedia.org/resource/1999–2000_Liverpool_F.C._season, http://dbpedia.org/resource/2015–16_Everton_F.C._season, http://dbpedia.org/resource/2013–14_Hull_City_A.F.C._season, http://dbpedia.org/resource/1997–98_Leeds_United_A.F.C._season, http://dbpedia.org/resource/1998–99_Leeds_United_A.F.C._season, http://dbpedia.org/resource/2005–06_Middlesbrough_F.C._season, http://dbpedia.org/resource/2006–07_Charlton_Athletic_F.C._season, http://dbpedia.org/resource/2007–08_West_Ham_United_F.C._season, http://dbpedia.org/resource/2008–09_Manchester_City_F.C._season, http://dbpedia.org/resource/2013–14_Cardiff_City_F.C._season, http://dbpedia.org/resource/1999–2000_Sunderland_A.F.C._season, http://dbpedia.org/resource/2000–01_Sunderland_A.F.C._season, http://dbpedia.org/resource/2001–02_Sunderland_A.F.C._season, http://dbpedia.org/resource/2002–03_Sunderland_A.F.C._season, http://dbpedia.org/resource/2003–04_Southampton_F.C._season, http://dbpedia.org/resource/2013–14_Crystal_Palace_F.C._season, http://dbpedia.org/resource/2007–08_Aston_Villa_F.C._season, http://dbpedia.org/resource/2013–14_Newcastle_United_F.C._season, http://dbpedia.org/resource/2015–16_Manchester_City_F.C._season, http://dbpedia.org/resource/2015–16_Newcastle_United_F.C._season, http://dbpedia.org/resource/2007–08_Fulham_F.C._season, http://dbpedia.org/resource/1994–95_Norwich_City_F.C._season, http://dbpedia.org/resource/2015–16_Norwich_City_F.C._season, http://dbpedia.org/resource/2015–16_Watford_F.C._season, http://dbpedia.org/resource/2001–02_Leicester_City_F.C._season, http://dbpedia.org/resource/1994–95_Crystal_Palace_F.C._season, http://dbpedia.org/resource/1996–97_Sunderland_A.F.C._season, http://dbpedia.org/resource/1992–93_Blackburn_Rovers_F.C._season, http://dbpedia.org/resource/1995–96_Middlesbrough_F.C._season, http://dbpedia.org/resource/2004–05_Newcastle_United_F.C._season, http://dbpedia.org/resource/2007–08_Newcastle_United_F.C._season, http://dbpedia.org/resource/2008–09_Newcastle_United_F.C._season, http://dbpedia.org/resource/2011–12_Newcastle_United_F.C._season, http://dbpedia.org/resource/2012–13_Newcastle_United_F.C._season, http://dbpedia.org/resource/2014–15_Newcastle_United_F.C._season, http://dbpedia.org/resource/2016–17_Tottenham_Hotspur_F.C._season, http://dbpedia.org/resource/2014–15_Everton_F.C._season, http://dbpedia.org/resource/2006–07_Reading_F.C._season, http://dbpedia.org/resource/2015–16_Sunderland_A.F.C._season, http://dbpedia.org/resource/2013–14_Chelsea_F.C._season, http://dbpedia.org/resource/1994–95_Coventry_City_F.C._season, http://dbpedia.org/resource/1995–96_Coventry_City_F.C._season, http://dbpedia.org/resource/1996–97_Coventry_City_F.C._season, http://dbpedia.org/resource/1997–98_Coventry_City_F.C._season, http://dbpedia.org/resource/1998–99_Coventry_City_F.C._season, http://dbpedia.org/resource/1999–2000_Coventry_City_F.C._season, http://dbpedia.org/resource/2000–01_Coventry_City_F.C._season, http://dbpedia.org/resource/2014–15_Tottenham_Hotspur_F.C._season, http://dbpedia.org/resource/2015–16_Arsenal_F.C._season, http://dbpedia.org/resource/2016–17_Crystal_Palace_F.C._season, http://dbpedia.org/resource/2002–03_West_Bromwich_Albion_F.C._season, http://dbpedia.org/resource/2003–04_Wolverhampton_Wanderers_F.C._season, http://dbpedia.org/resource/2011–12_Swansea_City_A.F.C._season, http://dbpedia.org/resource/2012–13_Swansea_City_A.F.C._season, http://dbpedia.org/resource/2013–14_Swansea_City_A.F.C._season, http://dbpedia.org/resource/2014–15_Swansea_City_A.F.C._season, http://dbpedia.org/resource/2015–16_Crystal_Palace_F.C._season, http://dbpedia.org/resource/2015–16_Swansea_City_A.F.C._season, http://dbpedia.org/resource/2016–17_Swansea_City_A.F.C._season, http://dbpedia.org/resource/2016–17_West_Ham_United_F.C._season, http://dbpedia.org/resource/2016–17_Manchester_City_F.C._season, http://dbpedia.org/resource/2004–05_Charlton_Athletic_F.C._season, http://dbpedia.org/resource/1994–95_Wimbledon_F.C._season, http://dbpedia.org/resource/1996–97_Wimbledon_F.C._season, http://dbpedia.org/resource/1999–2000_Wimbledon_F.C._season, http://dbpedia.org/resource/1998–99_Sheffield_Wednesday_F.C._season, http://dbpedia.org/resource/1999–2000_Sheffield_Wednesday_F.C._season, http://dbpedia.org/resource/2016–17_A.F.C._Bournemouth_season, http://dbpedia.org/resource/1995–96_West_Ham_United_F.C._season, http://dbpedia.org/resource/1996–97_West_Ham_United_F.C._season, http://dbpedia.org/resource/1997–98_West_Ham_United_F.C._season, http://dbpedia.org/resource/1999–2000_West_Ham_United_F.C._season, http://dbpedia.org/resource/2000–01_Leeds_United_A.F.C._season, http://dbpedia.org/resource/2000–01_West_Ham_United_F.C._season, http://dbpedia.org/resource/2006–07_Arsenal_F.C._season, http://dbpedia.org/resource/2007–08_Arsenal_F.C._season, http://dbpedia.org/resource/2011–12_Arsenal_F.C._season, http://dbpedia.org/resource/2007–08_Middlesbrough_F.C._season, http://dbpedia.org/resource/2008–09_Middlesbrough_F.C._season, http://dbpedia.org/resource/2016–17_Middlesbrough_F.C._season, http://dbpedia.org/resource/2006–07_Sheffield_United_F.C._season, http://dbpedia.org/resource/1994–95_Leicester_City_F.C._season, http://dbpedia.org/resource/1997–98_Leicester_City_F.C._season, http://dbpedia.org/resource/1998–99_Leicester_City_F.C._season, http://dbpedia.org/resource/1999–2000_Leicester_City_F.C._season, http://dbpedia.org/resource/2006–07_Wigan_Athletic_F.C._season, http://dbpedia.org/resource/2007–08_Wigan_Athletic_F.C._season, http://dbpedia.org/resource/2008–09_Wigan_Athletic_F.C._season, http://dbpedia.org/resource/1999–2000_Chelsea_F.C._season, http://dbpedia.org/resource/2000–01_Chelsea_F.C._season, http://dbpedia.org/resource/2001–02_Chelsea_F.C._season, http://dbpedia.org/resource/2003–04_Chelsea_F.C._season, http://dbpedia.org/resource/2005–06_Chelsea_F.C._season, http://dbpedia.org/resource/2007–08_Chelsea_F.C._season, http://dbpedia.org/resource/2012–13_Chelsea_F.C._season, http://dbpedia.org/resource/2015–16_Chelsea_F.C._season, http://dbpedia.org/resource/1992–93_Norwich_City_F.C._season, http://dbpedia.org/resource/1993–94_Aston_Villa_F.C._season, http://dbpedia.org/resource/1994–95_Aston_Villa_F.C._season, http://dbpedia.org/resource/1995–96_Aston_Villa_F.C._season, http://dbpedia.org/resource/1996–97_Blackburn_Rovers_F.C._season, http://dbpedia.org/resource/1997–98_Blackburn_Rovers_F.C._season, http://dbpedia.org/resource/1998–99_Aston_Villa_F.C._season, http://dbpedia.org/resource/1998–99_Blackburn_Rovers_F.C._season, http://dbpedia.org/resource/2002–03_Blackburn_Rovers_F.C._season, http://dbpedia.org/resource/2002–03_Tottenham_Hotspur_F.C._season, http://dbpedia.org/resource/2003–04_Portsmouth_F.C._season, http://dbpedia.org/resource/2004–05_Blackburn_Rovers_F.C._season, http://dbpedia.org/resource/2007–08_Sunderland_A.F.C._season, http://dbpedia.org/resource/2008–09_Sunderland_A.F.C._season, http://dbpedia.org/resource/1994–95_Leeds_United_A.F.C._season, http://dbpedia.org/resource/1997–98_Crystal_Palace_F.C._season, http://dbpedia.org/resource/2001–02_Blackburn_Rovers_F.C._season, http://dbpedia.org/resource/2003–04_Leicester_City_F.C._season, http://dbpedia.org/resource/2001–02_Southampton_F.C._season, http://dbpedia.org/resource/2010–11_Blackpool_F.C._season, http://dbpedia.org/resource/2012–13_Sunderland_A.F.C._season, http://dbpedia.org/resource/2014–15_West_Bromwich_Albion_F.C._season, http://dbpedia.org/resource/2015–16_West_Bromwich_Albion_F.C._season, http://dbpedia.org/resource/2016–17_West_Bromwich_Albion_F.C._season, http://dbpedia.org/resource/2001–02_Bolton_Wanderers_F.C._season, http://dbpedia.org/resource/2002–03_Bolton_Wanderers_F.C._season, http://dbpedia.org/resource/2003–04_Bolton_Wanderers_F.C._season, http://dbpedia.org/resource/2004–05_Bolton_Wanderers_F.C._season, http://dbpedia.org/resource/2006–07_Bolton_Wanderers_F.C._season, http://dbpedia.org/resource/1994–95_Manchester_City_F.C._season, http://dbpedia.org/resource/2000–01_Middlesbrough_F.C._season, http://dbpedia.org/resource/2012–13_Reading_F.C._season, http://dbpedia.org/resource/2014–15_Leicester_City_F.C._season, http://dbpedia.org/resource/2016–17_Southampton_F.C._season, http://dbpedia.org/resource/1997–98_Bolton_Wanderers_F.C._season, http://dbpedia.org/resource/2010–11_West_Bromwich_Albion_F.C._season, http://dbpedia.org/resource/2000–01_Leicester_City_F.C._season, http://dbpedia.org/resource/2016–17_Watford_F.C._season, http://dbpedia.org/resource/1994–95_Queens_Park_Rangers_F.C._season, http://dbpedia.org/resource/2009–10_Bolton_Wanderers_F.C._season, http://dbpedia.org/resource/2014–15_Burnley_F.C._season, http://dbpedia.org/resource/1994–95_Arsenal_F.C._season, http://dbpedia.org/resource/1996–97_Derby_County_F.C._season, http://dbpedia.org/resource/2009–10_West_Ham_United_F.C._season, http://dbpedia.org/resource/1993–94_Chelsea_F.C._season, http://dbpedia.org/resource/2007–08_Manchester_City_F.C._season, http://dbpedia.org/resource/2015–16_Leicester_City_F.C._season, http://dbpedia.org/resource/1994–95_West_Ham_United_F.C._season, http://dbpedia.org/resource/1995–96_Nottingham_Forest_F.C._season, http://dbpedia.org/resource/1996–97_Nottingham_Forest_F.C._season, http://dbpedia.org/resource/1994–95_Nottingham_Forest_F.C._season, http://dbpedia.org/resource/1992–93_Ipswich_Town_F.C._season, http://dbpedia.org/resource/Martin_Atkinson, http://dbpedia.org/resource/Rob_Styles, http://dbpedia.org/resource/Mike_Jones_(referee), http://dbpedia.org/resource/Paul_Danson, http://dbpedia.org/resource/Ray_Olivier, http://dbpedia.org/resource/Jonathan_Moss_(referee), http://dbpedia.org/resource/Keith_Stroud, http://dbpedia.org/resource/Steve_Tanner_(referee), http://dbpedia.org/resource/Rodger_Gifford, http://dbpedia.org/resource/Stuart_Attwell, http://dbpedia.org/resource/Mark_Halsey, http://dbpedia.org/resource/Neil_Swarbrick, http://dbpedia.org/resource/Neale_Barry, http://dbpedia.org/resource/David_Elleray, http://dbpedia.org/resource/Robbie_Hart, http://dbpedia.org/resource/Andre_Marriner, http://dbpedia.org/resource/Anthony_Taylor_(referee), http://dbpedia.org/resource/Matt_Messias, http://dbpedia.org/resource/Lee_Probert, http://dbpedia.org/resource/Robert_Madley, http://dbpedia.org/resource/Howard_King_(referee), http://dbpedia.org/resource/Gerald_Ashby, http://dbpedia.org/resource/Graham_Poll, http://dbpedia.org/resource/Howard_Webb, http://dbpedia.org/resource/Peter_Jones_(referee), http://dbpedia.org/resource/Michael_Oliver_(referee), http://dbpedia.org/resource/Wendy_Toms, http://dbpedia.org/resource/Keith_Cooper, http://dbpedia.org/resource/Paul_Taylor_(referee), http://dbpedia.org/resource/Chris_Foy_(referee), http://dbpedia.org/resource/Steve_Dunn_(referee), http://dbpedia.org/resource/Terry_Holbrook, http://dbpedia.org/resource/Mike_Riley_(referee), http://dbpedia.org/resource/Paul_Alcock, http://dbpedia.org/resource/Alan_Wilkie, http://dbpedia.org/resource/Alan_Wiley, http://dbpedia.org/resource/Keith_Hackett, http://dbpedia.org/resource/Mark_Clattenburg, http://dbpedia.org/resource/Mike_Dean_(referee), http://dbpedia.org/resource/Barry_Knight_(referee), http://dbpedia.org/resource/Steve_Bennett_(referee), http://dbpedia.org/resource/Phil_Joslin_(referee), http://dbpedia.org/resource/Brian_Hill_(referee), http://dbpedia.org/resource/Uriah_Rennie, http://dbpedia.org/resource/John_Brooks_(referee), http://dbpedia.org/resource/Dermot_Gallagher, http://dbpedia.org/resource/Peter_Walton_(referee), http://dbpedia.org/resource/Lee_Mason, http://dbpedia.org/resource/Darren_Drysdale, http://dbpedia.org/resource/Gary_Willard, http://dbpedia.org/resource/Keren_Barratt, http://dbpedia.org/resource/Martin_Bodenham, http://dbpedia.org/resource/Paul_Rejer, http://dbpedia.org/resource/Craig_Pawson, http://dbpedia.org/resource/Phil_Dowd, http://dbpedia.org/resource/Eddie_Wolstenholme, http://dbpedia.org/resource/Keith_Burge, http://dbpedia.org/resource/Stephen_Lodge_(referee), http://dbpedia.org/resource/David_Allison_(referee), http://dbpedia.org/resource/Andrew_Madley] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 139, P: 0.329, R: 0.413, macro F1: 0.334, macro F1*: 0.366 -[INFO] QALD Cumulative metrics, sample: 139, P: 0.552, R: 0.413, macro F1: 0.334, QALD macro F1: 0.473 -[INFO] Current time: 2094 ms, average: 3387.67 ms/ques, entity detection: 756.83 ms/ques (linking tool: 669.29 ms/ques), relation detection: 1080.34 ms/ques (similarity: 613.48 ms/ques), EDG generation: 46.23 ms/ques +[INFO] Cumulative metrics, sample: 139, P: 0.329, R: 0.413, macro F1: 0.366 +[INFO] QALD Cumulative metrics, sample: 139, P: 0.552, R: 0.413, average F1: 0.334, QALD macro F1: 0.473 +[INFO] Current question consumed time: 1820 ms, average: 3056.14 ms/ques, entity detection: 667.95 ms/ques (linking tool: 594.03 ms/ques), relation detection: 1004.13 ms/ques (similarity: 613.08 ms/ques), EDG generation: 47.40 ms/ques [INFO] Entity Linking P: 0.302158273381295, R: 0.302158273381295, micro F1: 0.302158273381295, macro F1: 0.302158273381295 [INFO] Relation Linking P: 0.30935251798561153, R: 0.30935251798561153, micro F1: 0.30935251798561153, macro F1: 0.30935251798561153 [INFO] Type Linking P: 0.8776978417266187, R: 0.8776978417266187, micro F1: 0.8776978417266187, macro F1: 0.8776978417266187 @@ -2097,9 +2097,9 @@ Question 139: Which instruments does Cat Stevens play? *piano *] [INFO] P: 0.7692307692307693, R: 1.0, F: 0.8695652173913044 -[INFO] Cumulative metrics, sample: 140, P: 0.332, R: 0.417, macro F1: 0.338, macro F1*: 0.370 -[INFO] QALD Cumulative metrics, sample: 140, P: 0.553, R: 0.417, macro F1: 0.338, QALD macro F1: 0.476 -[INFO] Current time: 1724 ms, average: 3375.79 ms/ques, entity detection: 752.94 ms/ques (linking tool: 665.70 ms/ques), relation detection: 1076.82 ms/ques (similarity: 613.54 ms/ques), EDG generation: 45.96 ms/ques +[INFO] Cumulative metrics, sample: 140, P: 0.332, R: 0.417, macro F1: 0.370 +[INFO] QALD Cumulative metrics, sample: 140, P: 0.553, R: 0.417, average F1: 0.338, QALD macro F1: 0.476 +[INFO] Current question consumed time: 1498 ms, average: 3045.01 ms/ques, entity detection: 664.48 ms/ques (linking tool: 590.82 ms/ques), relation detection: 1000.68 ms/ques (similarity: 613.16 ms/ques), EDG generation: 47.12 ms/ques [INFO] Entity Linking P: 0.3, R: 0.3, micro F1: 0.3, macro F1: 0.3 [INFO] Relation Linking P: 0.30714285714285716, R: 0.30714285714285716, micro F1: 0.30714285714285716, macro F1: 0.30714285714285716 [INFO] Type Linking P: 0.8785714285714286, R: 0.8785714285714286, micro F1: 0.8785714285714286, macro F1: 0.8785714285714286 @@ -2112,9 +2112,9 @@ Question 140: How many rivers and lakes are in South Carolina? [INFO] Golden Answer: [15] [INFO] Predicted Answer: [39, 15] [INFO] P: 0.5, R: 1.0, F: 0.6666666666666666 -[INFO] Cumulative metrics, sample: 141, P: 0.333, R: 0.422, macro F1: 0.340, macro F1*: 0.372 -[INFO] QALD Cumulative metrics, sample: 141, P: 0.553, R: 0.422, macro F1: 0.340, QALD macro F1: 0.478 -[INFO] Current time: 7446 ms, average: 3404.65 ms/ques, entity detection: 752.91 ms/ques (linking tool: 665.24 ms/ques), relation detection: 1087.99 ms/ques (similarity: 628.68 ms/ques), EDG generation: 45.79 ms/ques +[INFO] Cumulative metrics, sample: 141, P: 0.333, R: 0.422, macro F1: 0.372 +[INFO] QALD Cumulative metrics, sample: 141, P: 0.553, R: 0.422, average F1: 0.340, QALD macro F1: 0.478 +[INFO] Current question consumed time: 6466 ms, average: 3069.28 ms/ques, entity detection: 664.19 ms/ques (linking tool: 590.21 ms/ques), relation detection: 1010.77 ms/ques (similarity: 628.27 ms/ques), EDG generation: 46.98 ms/ques [INFO] Entity Linking P: 0.2978723404255319, R: 0.2978723404255319, micro F1: 0.2978723404255319, macro F1: 0.2978723404255319 [INFO] Relation Linking P: 0.3049645390070922, R: 0.3049645390070922, micro F1: 0.3049645390070922, macro F1: 0.3049645390070922 [INFO] Type Linking P: 0.8794326241134752, R: 0.8794326241134752, micro F1: 0.8794326241134752, macro F1: 0.8794326241134752 @@ -2127,9 +2127,9 @@ Question 141: Who was called Rodzilla? [INFO] Golden Answer: [http://dbpedia.org/resource/Rodney_Blake] [INFO] Predicted Answer: [] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 142, P: 0.331, R: 0.419, macro F1: 0.338, macro F1*: 0.370 -[INFO] QALD Cumulative metrics, sample: 142, P: 0.556, R: 0.419, macro F1: 0.338, QALD macro F1: 0.478 -[INFO] Current time: 350 ms, average: 3383.14 ms/ques, entity detection: 748.85 ms/ques (linking tool: 661.43 ms/ques), relation detection: 1080.58 ms/ques (similarity: 624.46 ms/ques), EDG generation: 45.51 ms/ques +[INFO] Cumulative metrics, sample: 142, P: 0.331, R: 0.419, macro F1: 0.370 +[INFO] QALD Cumulative metrics, sample: 142, P: 0.556, R: 0.419, average F1: 0.338, QALD macro F1: 0.478 +[INFO] Current question consumed time: 266 ms, average: 3049.54 ms/ques, entity detection: 660.44 ms/ques (linking tool: 586.70 ms/ques), relation detection: 1003.87 ms/ques (similarity: 624.01 ms/ques), EDG generation: 46.68 ms/ques [INFO] Entity Linking P: 0.3028169014084507, R: 0.3028169014084507, micro F1: 0.3028169014084507, macro F1: 0.3028169014084507 [INFO] Relation Linking P: 0.30985915492957744, R: 0.30985915492957744, micro F1: 0.30985915492957744, macro F1: 0.30985915492957744 [INFO] Type Linking P: 0.8802816901408451, R: 0.8802816901408451, micro F1: 0.8802816901408451, macro F1: 0.8802816901408451 @@ -2142,9 +2142,9 @@ Question 142: Give me the capitals of all countries in Africa. [INFO] Golden Answer: [http://dbpedia.org/resource/Algiers, http://dbpedia.org/resource/Luanda, http://dbpedia.org/resource/Ouagadougou, http://dbpedia.org/resource/Bujumbura, http://dbpedia.org/resource/Yaoundé, http://dbpedia.org/resource/Praia, http://dbpedia.org/resource/Bangui, http://dbpedia.org/resource/N'Djamena, http://dbpedia.org/resource/Moroni,_Comoros, http://dbpedia.org/resource/Kinshasa, http://dbpedia.org/resource/Djibouti_(city), http://dbpedia.org/resource/Cairo, http://dbpedia.org/resource/Malabo, http://dbpedia.org/resource/Asmara, http://dbpedia.org/resource/Addis_Ababa, http://dbpedia.org/resource/Libreville, http://dbpedia.org/resource/Accra, http://dbpedia.org/resource/Conakry, http://dbpedia.org/resource/Bissau, http://dbpedia.org/resource/Abidjan, http://dbpedia.org/resource/Yamoussoukro, http://dbpedia.org/resource/Nairobi, http://dbpedia.org/resource/Maseru, http://dbpedia.org/resource/Monrovia, http://dbpedia.org/resource/Tripoli, http://dbpedia.org/resource/Antananarivo, http://dbpedia.org/resource/Lilongwe, http://dbpedia.org/resource/Bamako, http://dbpedia.org/resource/Nouakchott, http://dbpedia.org/resource/Port_Louis, http://dbpedia.org/resource/Rabat, http://dbpedia.org/resource/Maputo, http://dbpedia.org/resource/Windhoek, http://dbpedia.org/resource/Niamey, http://dbpedia.org/resource/Abuja, http://dbpedia.org/resource/Brazzaville, http://dbpedia.org/resource/Kigali, http://dbpedia.org/resource/Laayoune, http://dbpedia.org/resource/Tifariti, http://dbpedia.org/resource/Dakar, http://dbpedia.org/resource/Victoria,_Seychelles, http://dbpedia.org/resource/Freetown, http://dbpedia.org/resource/Mogadishu, http://dbpedia.org/resource/Bloemfontein, http://dbpedia.org/resource/Cape_Town, http://dbpedia.org/resource/Pretoria, http://dbpedia.org/resource/Juba, http://dbpedia.org/resource/Khartoum, http://dbpedia.org/resource/Mbabane, http://dbpedia.org/resource/Lobamba, http://dbpedia.org/resource/São_Tomé, http://dbpedia.org/resource/Dodoma, http://dbpedia.org/resource/Lomé, http://dbpedia.org/resource/Tunis, http://dbpedia.org/resource/Lusaka, http://dbpedia.org/resource/Harare, http://dbpedia.org/resource/Hargeisa, http://dbpedia.org/resource/Porto-Novo, http://dbpedia.org/resource/Banjul] [INFO] Predicted Answer: [http://dbpedia.org/resource/Katsina] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 143, P: 0.329, R: 0.416, macro F1: 0.336, macro F1*: 0.367 -[INFO] QALD Cumulative metrics, sample: 143, P: 0.552, R: 0.416, macro F1: 0.336, QALD macro F1: 0.474 -[INFO] Current time: 4460 ms, average: 3390.67 ms/ques, entity detection: 748.69 ms/ques (linking tool: 661.19 ms/ques), relation detection: 1085.13 ms/ques (similarity: 626.04 ms/ques), EDG generation: 45.80 ms/ques +[INFO] Cumulative metrics, sample: 143, P: 0.329, R: 0.416, macro F1: 0.367 +[INFO] QALD Cumulative metrics, sample: 143, P: 0.552, R: 0.416, average F1: 0.336, QALD macro F1: 0.474 +[INFO] Current question consumed time: 3742 ms, average: 3054.38 ms/ques, entity detection: 659.42 ms/ques (linking tool: 585.66 ms/ques), relation detection: 1007.67 ms/ques (similarity: 625.48 ms/ques), EDG generation: 47.06 ms/ques [INFO] Entity Linking P: 0.3006993006993007, R: 0.3006993006993007, micro F1: 0.3006993006993007, macro F1: 0.3006993006993007 [INFO] Relation Linking P: 0.3076923076923077, R: 0.3076923076923077, micro F1: 0.3076923076923077, macro F1: 0.3076923076923077 [INFO] Type Linking P: 0.8811188811188811, R: 0.8811188811188811, micro F1: 0.8811188811188811, macro F1: 0.881118881118881 @@ -2157,9 +2157,9 @@ Question 143: Which bridges are of the same type as the Manhattan Bridge? [INFO] Golden Answer: [http://dbpedia.org/resource/Brooklyn_Bridge, http://dbpedia.org/resource/Pont_d'Aquitaine, http://dbpedia.org/resource/Manhattan_Bridge, http://dbpedia.org/resource/Triborough_Bridge, http://dbpedia.org/resource/Williamsburg_Bridge, http://dbpedia.org/resource/Howrah_Bridge, http://dbpedia.org/resource/Mid-Hudson_Bridge, http://dbpedia.org/resource/Waco_Suspension_Bridge, http://dbpedia.org/resource/South_Tenth_Street_Bridge, http://dbpedia.org/resource/Kingston–Port_Ewen_Suspension_Bridge, http://dbpedia.org/resource/Varodd_Bridge, http://dbpedia.org/resource/Satok_Bridge, http://dbpedia.org/resource/Zunyi_Bridge, http://dbpedia.org/resource/Çanakkale_1915_Bridge, http://dbpedia.org/resource/Andy_Warhol_Bridge, http://dbpedia.org/resource/Menai_Suspension_Bridge, http://dbpedia.org/resource/Akashi_Kaikyō_Bridge, http://dbpedia.org/resource/Golden_Gate_Bridge, http://dbpedia.org/resource/Carquinez_Bridge, http://dbpedia.org/resource/Anthony_Wayne_Bridge, http://dbpedia.org/resource/Tajik–Afghan_Friendship_Bridge, http://dbpedia.org/resource/Hakata–Ōshima_Bridge, http://dbpedia.org/resource/Innoshima_Bridge, http://dbpedia.org/resource/Kurushima-Kaikyō_Bridge, http://dbpedia.org/resource/Ōnaruto_Bridge, http://dbpedia.org/resource/Gateway_Bridge_(Illinois–Iowa), http://dbpedia.org/resource/Runyang_Yangtze_River_Bridge, http://dbpedia.org/resource/Jim_Stynes_Bridge, http://dbpedia.org/resource/Choluteca_Bridge, http://dbpedia.org/resource/Kangaroo_Point_Bridge, http://dbpedia.org/resource/Daly's_bridge, http://dbpedia.org/resource/Yichang_Yangtze_River_Highway_Bridge, http://dbpedia.org/resource/Albert_Bridge,_London, http://dbpedia.org/resource/Clifton_Suspension_Bridge, http://dbpedia.org/resource/Hammersmith_Bridge, http://dbpedia.org/resource/Mackinac_Bridge, http://dbpedia.org/resource/Thousand_Islands_Bridge, http://dbpedia.org/resource/Bosphorus_Bridge, http://dbpedia.org/resource/Bronx–Whitestone_Bridge, http://dbpedia.org/resource/Fatih_Sultan_Mehmet_Bridge, http://dbpedia.org/resource/San_Francisco–Oakland_Bay_Bridge, http://dbpedia.org/resource/Throgs_Neck_Bridge, http://dbpedia.org/resource/Yavuz_Sultan_Selim_Bridge, http://dbpedia.org/resource/Tamar_Bridge, http://dbpedia.org/resource/Emmerich_Rhine_Bridge, http://dbpedia.org/resource/Roberto_Clemente_Bridge, http://dbpedia.org/resource/Tacoma_Narrows_Bridge_(1950), http://dbpedia.org/resource/U.S._Grant_Bridge, http://dbpedia.org/resource/Pierre_Laporte_Bridge, http://dbpedia.org/resource/Haicang_Bridge, http://dbpedia.org/resource/Nicholas_Chain_Bridge, http://dbpedia.org/resource/Fort_Edmonton_Footbridge, http://dbpedia.org/resource/Peak_Walk, http://dbpedia.org/resource/EG_LNG_Pipeline_Suspension_Bridge, http://dbpedia.org/resource/Ambassador_Bridge, http://dbpedia.org/resource/Chesapeake_Bay_Bridge, http://dbpedia.org/resource/Great_Belt_Fixed_Link, http://dbpedia.org/resource/Wheeling_Suspension_Bridge, http://dbpedia.org/resource/Hal_W._Adams_Bridge, http://dbpedia.org/resource/A._Murray_MacKay_Bridge, http://dbpedia.org/resource/Angus_L._Macdonald_Bridge, http://dbpedia.org/resource/Huangpu_Bridge, http://dbpedia.org/resource/Humen_Pearl_River_Bridge, http://dbpedia.org/resource/Ozimek_Suspension_Bridge, http://dbpedia.org/resource/Kutai_Kartanegara_Bridge, http://dbpedia.org/resource/Jiangyin_Yangtze_River_Bridge, http://dbpedia.org/resource/Tajik–Afghan_bridge_at_Tem-Demogan, http://dbpedia.org/resource/Xiling_Yangtze_River_Bridge, http://dbpedia.org/resource/Yangluo_Yangtze_River_Bridge, http://dbpedia.org/resource/St._Johns_Bridge, http://dbpedia.org/resource/Bear_Mountain_Bridge, http://dbpedia.org/resource/Niagara_Falls_Suspension_Bridge, http://dbpedia.org/resource/Osman_Gazi_Bridge, http://dbpedia.org/resource/Roebling's_Delaware_Aqueduct, http://dbpedia.org/resource/Severn_Bridge, http://dbpedia.org/resource/Xihoumen_Bridge, http://dbpedia.org/resource/Askøy_Bridge, http://dbpedia.org/resource/Höga_Kusten_Bridge, http://dbpedia.org/resource/Osterøy_Bridge, http://dbpedia.org/resource/Sotra_Bridge, http://dbpedia.org/resource/Fort_Steuben_Bridge, http://dbpedia.org/resource/Beaver_Bridge_(Arkansas), http://dbpedia.org/resource/Bidwell_Bar_Bridge, http://dbpedia.org/resource/Grand_Tower_Pipeline_Bridge, http://dbpedia.org/resource/Chain_Bridge_(Massachusetts), http://dbpedia.org/resource/Simon_Kenton_Memorial_Bridge, http://dbpedia.org/resource/Two_Cent_Bridge, http://dbpedia.org/resource/Waldo–Hancock_Bridge, http://dbpedia.org/resource/Rombak_Bridge, http://dbpedia.org/resource/Tjeldsund_Bridge, http://dbpedia.org/resource/Samora_Machel_Bridge, http://dbpedia.org/resource/Lumberville–Raven_Rock_Bridge, http://dbpedia.org/resource/Liede_Bridge, http://dbpedia.org/resource/Pingsheng_Bridge, http://dbpedia.org/resource/Shantou_Bay_Bridge, http://dbpedia.org/resource/Lawas_Bridge, http://dbpedia.org/resource/25_de_Abril_Bridge, http://dbpedia.org/resource/Tacoma_Narrows_Bridge_(1940), http://dbpedia.org/resource/Älvsborg_Bridge, http://dbpedia.org/resource/Humber_Bridge, http://dbpedia.org/resource/Vincent_Thomas_Bridge, http://dbpedia.org/resource/Claiborne_Pell_Newport_Bridge, http://dbpedia.org/resource/Forth_Road_Bridge, http://dbpedia.org/resource/Millennium_Bridge,_London, http://dbpedia.org/resource/Chain_Bridge_(Budapest), http://dbpedia.org/resource/Ogdensburg–Prescott_International_Bridge, http://dbpedia.org/resource/Bridge_for_Kids, http://dbpedia.org/resource/Market_Street_Bridge_(Steubenville), http://dbpedia.org/resource/Guy_West_Bridge, http://dbpedia.org/resource/Dazi_Bridge, http://dbpedia.org/resource/Semey_Bridge, http://dbpedia.org/resource/Erfjord_Bridge, http://dbpedia.org/resource/Amu_Darya_Pipeline_Bridge, http://dbpedia.org/resource/Hennepin_Avenue_Bridge, http://dbpedia.org/resource/Luding_Bridge, http://dbpedia.org/resource/Fourth_Nanjing_Yangtze_Bridge, http://dbpedia.org/resource/Lions_Gate_Bridge, http://dbpedia.org/resource/Riegelsville_Bridge, http://dbpedia.org/resource/Tancarville_Bridge, http://dbpedia.org/resource/Walter_Taylor_Bridge, http://dbpedia.org/resource/Angers_Bridge, http://dbpedia.org/resource/Mount_Hope_Bridge, http://dbpedia.org/resource/Hakuchō_Bridge, http://dbpedia.org/resource/Hampden_Bridge_(Kangaroo_Valley), http://dbpedia.org/resource/Hercilio_Luz_Bridge, http://dbpedia.org/resource/Hirado_Bridge, http://dbpedia.org/resource/Kattel_Bridge, http://dbpedia.org/resource/Kuala_Krai_Bridge, http://dbpedia.org/resource/Namhae_Bridge, http://dbpedia.org/resource/Porthill_Bridge, http://dbpedia.org/resource/Puente_de_Occidente, http://dbpedia.org/resource/Thuận_Phước_Bridge, http://dbpedia.org/resource/Yeongjong_Bridge, http://dbpedia.org/resource/Yi_Sun-sin_Bridge, http://dbpedia.org/resource/Seaway_International_Bridge, http://dbpedia.org/resource/Beipan_River_Guanxing_Highway_Bridge, http://dbpedia.org/resource/Beipan_River_Hukun_Expressway_Bridge, http://dbpedia.org/resource/Sidu_River_Bridge, http://dbpedia.org/resource/Yangmingtan_Bridge, http://dbpedia.org/resource/Aizhai_Bridge, http://dbpedia.org/resource/Gwangandaegyo, http://dbpedia.org/resource/Île_d'Orléans_Bridge, http://dbpedia.org/resource/Matadi_Bridge, http://dbpedia.org/resource/Bømla_Bridge, http://dbpedia.org/resource/Fyksesund_Bridge, http://dbpedia.org/resource/Gjemnessund_Bridge, http://dbpedia.org/resource/Hardanger_Bridge, http://dbpedia.org/resource/Kvalsund_Bridge, http://dbpedia.org/resource/Lokkaren_Bridge, http://dbpedia.org/resource/Lysefjord_Bridge, http://dbpedia.org/resource/Nærøysund_Bridge, http://dbpedia.org/resource/Stord_Bridge, http://dbpedia.org/resource/Elisabeth_Bridge_(Budapest), http://dbpedia.org/resource/Sewells_Road_Bridge, http://dbpedia.org/resource/Arapuni_Suspension_Bridge, http://dbpedia.org/resource/Puente_Colgante_(Manila), http://dbpedia.org/resource/Allen_Street_Bridge_disaster, http://dbpedia.org/resource/Otowi_Suspension_Bridge, http://dbpedia.org/resource/Royal_Gorge_Bridge__Royal_Gorge_Bridge__1, http://dbpedia.org/resource/Malacca_Strait_Bridge__Malacca_Strait_Bridge__1] [INFO] Predicted Answer: [http://dbpedia.org/resource/Suspension_bridge] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 144, P: 0.326, R: 0.413, macro F1: 0.333, macro F1*: 0.364 -[INFO] QALD Cumulative metrics, sample: 144, P: 0.548, R: 0.413, macro F1: 0.333, QALD macro F1: 0.471 -[INFO] Current time: 1994 ms, average: 3380.97 ms/ques, entity detection: 747.90 ms/ques (linking tool: 660.42 ms/ques), relation detection: 1080.55 ms/ques (similarity: 624.84 ms/ques), EDG generation: 45.62 ms/ques +[INFO] Cumulative metrics, sample: 144, P: 0.326, R: 0.413, macro F1: 0.364 +[INFO] QALD Cumulative metrics, sample: 144, P: 0.548, R: 0.413, average F1: 0.333, QALD macro F1: 0.471 +[INFO] Current question consumed time: 1708 ms, average: 3045.03 ms/ques, entity detection: 658.56 ms/ques (linking tool: 584.88 ms/ques), relation detection: 1003.35 ms/ques (similarity: 624.27 ms/ques), EDG generation: 46.88 ms/ques [INFO] Entity Linking P: 0.2986111111111111, R: 0.2986111111111111, micro F1: 0.2986111111111111, macro F1: 0.2986111111111111 [INFO] Relation Linking P: 0.3055555555555556, R: 0.3055555555555556, micro F1: 0.3055555555555556, macro F1: 0.3055555555555556 [INFO] Type Linking P: 0.8819444444444444, R: 0.8819444444444444, micro F1: 0.8819444444444444, macro F1: 0.8819444444444444 @@ -2172,9 +2172,9 @@ Question 144: How many companies were founded by the founder of Facebook? [INFO] Golden Answer: [3] [INFO] Predicted Answer: [2] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 145, P: 0.324, R: 0.410, macro F1: 0.331, macro F1*: 0.362 -[INFO] QALD Cumulative metrics, sample: 145, P: 0.545, R: 0.410, macro F1: 0.331, QALD macro F1: 0.468 -[INFO] Current time: 3641 ms, average: 3382.77 ms/ques, entity detection: 746.61 ms/ques (linking tool: 659.17 ms/ques), relation detection: 1082.75 ms/ques (similarity: 627.26 ms/ques), EDG generation: 45.92 ms/ques +[INFO] Cumulative metrics, sample: 145, P: 0.324, R: 0.410, macro F1: 0.362 +[INFO] QALD Cumulative metrics, sample: 145, P: 0.545, R: 0.410, average F1: 0.331, QALD macro F1: 0.468 +[INFO] Current question consumed time: 3202 ms, average: 3046.11 ms/ques, entity detection: 657.47 ms/ques (linking tool: 583.85 ms/ques), relation detection: 1005.07 ms/ques (similarity: 626.87 ms/ques), EDG generation: 47.17 ms/ques [INFO] Entity Linking P: 0.296551724137931, R: 0.296551724137931, micro F1: 0.296551724137931, macro F1: 0.296551724137931 [INFO] Relation Linking P: 0.30344827586206896, R: 0.30344827586206896, micro F1: 0.30344827586206896, macro F1: 0.30344827586206896 [INFO] Type Linking P: 0.8758620689655172, R: 0.8758620689655172, micro F1: 0.8758620689655172, macro F1: 0.8758620689655172 @@ -2187,9 +2187,9 @@ Question 145: Show me hiking trails in the Grand Canyon where there's no danger [INFO] Golden Answer: [http://dbpedia.org/resource/Bright_Angel_Trail] [INFO] Predicted Answer: [http://dbpedia.org/resource/Deer_Creek_Trail, http://dbpedia.org/resource/Dripping_Springs_Trail, http://dbpedia.org/resource/Royal_Arch_Route, http://dbpedia.org/resource/Boucher_Trail, http://dbpedia.org/resource/Clear_Creek_(Arizona), http://dbpedia.org/resource/Grand_Canyon, Clear Creek Trail Junction with North Kaibab Trail, http://dbpedia.org/resource/Colorado_River, Point Imperial, Uncle Jim Trailhead, Cliff Spring Trailhead, http://dbpedia.org/resource/Thunder_River_Trail, North Rim, http://dbpedia.org/resource/Indian_Garden, Indian Hollow, Grand Canyon North Rim, http://dbpedia.org/resource/Hermit's_Rest, Garnet Canyon, Red Canyon, http://dbpedia.org/resource/Phantom_Ranch, http://dbpedia.org/resource/Lipan_Point, Widforss Trailhead, http://dbpedia.org/resource/Hermit_Trail, Eremita Mesa, South Rim, South Rim, Hermit Trail junction, http://dbpedia.org/resource/Tonto_Trail, http://dbpedia.org/resource/South_Bass_Trail] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 146, P: 0.322, R: 0.407, macro F1: 0.329, macro F1*: 0.359 -[INFO] QALD Cumulative metrics, sample: 146, P: 0.541, R: 0.407, macro F1: 0.329, QALD macro F1: 0.465 -[INFO] Current time: 2926 ms, average: 3379.64 ms/ques, entity detection: 745.43 ms/ques (linking tool: 657.01 ms/ques), relation detection: 1077.95 ms/ques (similarity: 624.26 ms/ques), EDG generation: 45.92 ms/ques +[INFO] Cumulative metrics, sample: 146, P: 0.322, R: 0.407, macro F1: 0.359 +[INFO] QALD Cumulative metrics, sample: 146, P: 0.541, R: 0.407, average F1: 0.329, QALD macro F1: 0.465 +[INFO] Current question consumed time: 2386 ms, average: 3041.60 ms/ques, entity detection: 656.01 ms/ques (linking tool: 581.68 ms/ques), relation detection: 1000.44 ms/ques (similarity: 623.80 ms/ques), EDG generation: 47.16 ms/ques [INFO] Entity Linking P: 0.2945205479452055, R: 0.2945205479452055, micro F1: 0.2945205479452055, macro F1: 0.2945205479452055 [INFO] Relation Linking P: 0.3013698630136986, R: 0.3013698630136986, micro F1: 0.3013698630136986, macro F1: 0.3013698630136986 [INFO] Type Linking P: 0.8767123287671232, R: 0.8767123287671232, micro F1: 0.8767123287671232, macro F1: 0.8767123287671232 @@ -2202,9 +2202,9 @@ Question 146: Which book has the most pages? [INFO] Golden Answer: [http://dbpedia.org/resource/The_Tolkien_Reader] [INFO] Predicted Answer: [http://dbpedia.org/resource/1356_(novel), http://dbpedia.org/resource/1985_(Anthony_Burgess_novel), http://dbpedia.org/resource/700_Sundays, http://dbpedia.org/resource/A_Brief_History_of_Seven_Killings, http://dbpedia.org/resource/A_Brief_History_of_Time, http://dbpedia.org/resource/A_Christmas_Carol, http://dbpedia.org/resource/A_Defence_of_Masochism, http://dbpedia.org/resource/A_Distant_Mirror, http://dbpedia.org/resource/A_General_History_of_the_Pyrates, http://dbpedia.org/resource/A_History_of_Western_Philosophy, http://dbpedia.org/resource/A_Moveable_Feast, http://dbpedia.org/resource/A_New_System_of_Domestic_Cookery, http://dbpedia.org/resource/Achtung_–_Panzer!, http://dbpedia.org/resource/All_the_Pretty_Horses_(novel), http://dbpedia.org/resource/American_Gods, http://dbpedia.org/resource/Amsterdam_(novel), http://dbpedia.org/resource/An_Age, http://dbpedia.org/resource/An_Insider's_View_of_Mormon_Origins, http://dbpedia.org/resource/Appley_Dapply's_Nursery_Rhymes, http://dbpedia.org/resource/BASIC_Computer_Games, http://dbpedia.org/resource/Back_When_We_Were_Grownups, http://dbpedia.org/resource/Ball_Four, http://dbpedia.org/resource/Basic_Chess_Endings, http://dbpedia.org/resource/Becoming_Batman:_The_Possibility_of_a_Superhero, http://dbpedia.org/resource/Belle_et_Sébastien, http://dbpedia.org/resource/Beowulf:_A_Translation_and_Commentary, http://dbpedia.org/resource/Best_SF:_1968, http://dbpedia.org/resource/Big_Bang_(book), http://dbpedia.org/resource/Bigfoot:_The_Life_and_Times_of_a_Legend, http://dbpedia.org/resource/Black_Flags:_The_Rise_of_ISIS, http://dbpedia.org/resource/Black_Holes_and_Time_Warps, http://dbpedia.org/resource/Bring_Up_the_Bodies, http://dbpedia.org/resource/Brown_Bear,_Brown_Bear,_What_Do_You_See%3F, http://dbpedia.org/resource/Career_of_Evil, http://dbpedia.org/resource/Carmilla, http://dbpedia.org/resource/Cecily_Parsley's_Nursery_Rhymes, http://dbpedia.org/resource/Chander_Pahar, http://dbpedia.org/resource/Chaos:_Making_a_New_Science, http://dbpedia.org/resource/Cloudy_with_a_Chance_of_Meatballs, http://dbpedia.org/resource/Comic_novel, http://dbpedia.org/resource/Coming_to_Power, http://dbpedia.org/resource/Computer_Graphics:_Principles_and_Practice, http://dbpedia.org/resource/Critical_Path_(book), http://dbpedia.org/resource/Dead_Babies_(novel), http://dbpedia.org/resource/Doctor_Zhivago_(novel), http://dbpedia.org/resource/Dot_and_the_Kangaroo, http://dbpedia.org/resource/Dracula, http://dbpedia.org/resource/Dracula's_Guest_and_Other_Weird_Stories, http://dbpedia.org/resource/Dracula_the_Un-dead, http://dbpedia.org/resource/Dreadnought_(book), http://dbpedia.org/resource/Earthly_Powers, http://dbpedia.org/resource/Earthworks_(novel), http://dbpedia.org/resource/Eat,_Pray,_Love, http://dbpedia.org/resource/Endymion_(Disraeli_novel), http://dbpedia.org/resource/Enigma_(novel), http://dbpedia.org/resource/Exit_to_Eden, http://dbpedia.org/resource/Famous_Impostors, http://dbpedia.org/resource/Fanny_Hill, http://dbpedia.org/resource/Finnegans_Wake, http://dbpedia.org/resource/First_Folio, http://dbpedia.org/resource/G._(novel), http://dbpedia.org/resource/Game_of_Shadows, http://dbpedia.org/resource/Goethe's_Faust, http://dbpedia.org/resource/Going_Bovine, http://dbpedia.org/resource/Greybeard, http://dbpedia.org/resource/Hadassah:_One_Night_with_the_King, http://dbpedia.org/resource/Heat_and_Dust, http://dbpedia.org/resource/Helmet_for_My_Pillow, http://dbpedia.org/resource/Historia_Plantarum_(Theophrastus), http://dbpedia.org/resource/Holiday_(novel), http://dbpedia.org/resource/Horton_Hatches_the_Egg, http://dbpedia.org/resource/Hotel_du_Lac, http://dbpedia.org/resource/Hothouse_(novel), http://dbpedia.org/resource/How_Late_It_Was,_How_Late, http://dbpedia.org/resource/Howl's_Moving_Castle, http://dbpedia.org/resource/I_Am_Number_Four, http://dbpedia.org/resource/I_Am_Ozzy, http://dbpedia.org/resource/In_Praise_of_Idleness_and_Other_Essays, http://dbpedia.org/resource/In_a_Free_State, http://dbpedia.org/resource/Inferno_(Niven_and_Pournelle_novel), http://dbpedia.org/resource/Interzone_(book), http://dbpedia.org/resource/Job_(novel), http://dbpedia.org/resource/Joseph_Smith:_Rough_Stone_Rolling, http://dbpedia.org/resource/Joseph_Smith:_The_Making_of_a_Prophet, http://dbpedia.org/resource/Juiced:_Wild_Times,_Rampant_'Roids,_Smash_Hits_&_How_Baseball_Got_Big, http://dbpedia.org/resource/Killing_Floor_(novel), http://dbpedia.org/resource/Lady_Athlyne, http://dbpedia.org/resource/Last_Orders, http://dbpedia.org/resource/Life_&_Times_of_Michael_K, http://dbpedia.org/resource/Life_of_Pi, http://dbpedia.org/resource/Lothair_(novel), http://dbpedia.org/resource/Make_Love!_The_Bruce_Campbell_Way, http://dbpedia.org/resource/Miss_Betty, http://dbpedia.org/resource/Mommie_Dearest, http://dbpedia.org/resource/Moneyball, http://dbpedia.org/resource/Moon_Tiger, http://dbpedia.org/resource/Mortal_Stakes, http://dbpedia.org/resource/Never_Cry_Wolf, http://dbpedia.org/resource/No_Time_Like_Tomorrow, http://dbpedia.org/resource/Non-Stop_(novel), http://dbpedia.org/resource/Observations_Concerning_the_Increase_of_Mankind,_Peopling_of_Countries,_etc., http://dbpedia.org/resource/Offshore_(novel), http://dbpedia.org/resource/On_Chesil_Beach, http://dbpedia.org/resource/On_Stranger_Tides, http://dbpedia.org/resource/Operating_Manual_for_Spaceship_Earth, http://dbpedia.org/resource/Oscar_and_Lucinda, http://dbpedia.org/resource/Paddy_Clarke_Ha_Ha_Ha, http://dbpedia.org/resource/Panzer_Leader_(book), http://dbpedia.org/resource/Panzerschlachten, http://dbpedia.org/resource/Personal_Reminiscences_of_Henry_Irving, http://dbpedia.org/resource/Pet_Sematary, http://dbpedia.org/resource/Power:_A_New_Social_Analysis, http://dbpedia.org/resource/Psmith,_Journalist, http://dbpedia.org/resource/Puss_in_Boots, http://dbpedia.org/resource/Reasons_and_Persons, http://dbpedia.org/resource/Red_Sox_Rule, http://dbpedia.org/resource/Report_on_Probability_A, http://dbpedia.org/resource/Resurrection_(novel), http://dbpedia.org/resource/Sacred_Hunger, http://dbpedia.org/resource/Saint_Maybe, http://dbpedia.org/resource/Satyricon, http://dbpedia.org/resource/Saville_(novel), http://dbpedia.org/resource/Schindler's_Ark, http://dbpedia.org/resource/Seduction_of_the_Innocent, http://dbpedia.org/resource/Shame_(Rushdie_novel), http://dbpedia.org/resource/Shiloh_(Naylor_novel), http://dbpedia.org/resource/Shoeless_Joe_(novel), http://dbpedia.org/resource/Slashdot, http://dbpedia.org/resource/Snowbound:_The_Record_of_a_Theatrical_Touring_Party, http://dbpedia.org/resource/Something_to_Answer_For, http://dbpedia.org/resource/Space_Opera_(1974_anthology), http://dbpedia.org/resource/Stalingrad_(book), http://dbpedia.org/resource/Stand_on_Zanzibar, http://dbpedia.org/resource/Staying_On, http://dbpedia.org/resource/Stopped_at_Stalingrad, http://dbpedia.org/resource/Story_of_O, http://dbpedia.org/resource/Studies_of_the_Book_of_Mormon, http://dbpedia.org/resource/Sybil_(novel), http://dbpedia.org/resource/TMZ, http://dbpedia.org/resource/The_AWK_Programming_Language, http://dbpedia.org/resource/The_Ancestor's_Tale, http://dbpedia.org/resource/The_Autobiography_of_Benjamin_Franklin, http://dbpedia.org/resource/The_Basketball_Diaries, http://dbpedia.org/resource/The_Bee_on_the_Comb, http://dbpedia.org/resource/The_Bible_Unearthed, http://dbpedia.org/resource/The_Bill_James_Historical_Baseball_Abstract, http://dbpedia.org/resource/The_Blind_Assassin, http://dbpedia.org/resource/The_Bone_People, http://dbpedia.org/resource/The_Brothers_Karamazov, http://dbpedia.org/resource/The_Canopy_of_Time, http://dbpedia.org/resource/The_Cathedral_and_the_Bazaar, http://dbpedia.org/resource/The_Conservationist, http://dbpedia.org/resource/The_Demolished_Man, http://dbpedia.org/resource/The_Dunwich_Horror, http://dbpedia.org/resource/The_Elected_Member, http://dbpedia.org/resource/The_Empire_Strikes_Out, http://dbpedia.org/resource/The_English_Patient, http://dbpedia.org/resource/The_Fairy_Caravan, http://dbpedia.org/resource/The_Famished_Road, http://dbpedia.org/resource/The_Fellowship_of_the_Ring, http://dbpedia.org/resource/The_Fifteen_Decisive_Battles_of_the_World, http://dbpedia.org/resource/The_Finkler_Question, http://dbpedia.org/resource/The_Fractal_Geometry_of_Nature, http://dbpedia.org/resource/The_Gathering_(Enright_novel), http://dbpedia.org/resource/The_Genetical_Theory_of_Natural_Selection, http://dbpedia.org/resource/The_Ghost_Road, http://dbpedia.org/resource/The_Glass_Menagerie, http://dbpedia.org/resource/The_Glory_of_Their_Times, http://dbpedia.org/resource/The_God_of_Small_Things, http://dbpedia.org/resource/The_Golden_One, http://dbpedia.org/resource/The_Heart_of_the_Matter, http://dbpedia.org/resource/The_Image_(novel), http://dbpedia.org/resource/The_Influence_of_Sea_Power_upon_History, http://dbpedia.org/resource/The_Inheritance_of_Loss, http://dbpedia.org/resource/The_Jewel_of_Seven_Stars, http://dbpedia.org/resource/The_Lady_of_the_Shroud, http://dbpedia.org/resource/The_Lair_of_the_White_Worm, http://dbpedia.org/resource/The_Last_Night_of_the_Yankee_Dynasty, http://dbpedia.org/resource/The_Legend_of_Sleepy_Hollow, http://dbpedia.org/resource/The_Line_of_Beauty, http://dbpedia.org/resource/The_Lion,_the_Witch_and_the_Wardrobe, http://dbpedia.org/resource/The_Long_Winter_(novel), http://dbpedia.org/resource/The_Loved_One, http://dbpedia.org/resource/The_Mauritius_Command, http://dbpedia.org/resource/The_Mind_of_God, http://dbpedia.org/resource/The_Moment_of_Eclipse, http://dbpedia.org/resource/The_Mystery_of_the_Sea, http://dbpedia.org/resource/The_Naked_and_the_Dead, http://dbpedia.org/resource/The_Narrow_Road_to_the_Deep_North_(novel), http://dbpedia.org/resource/The_Natural, http://dbpedia.org/resource/The_Old_Man_of_Lochnagar, http://dbpedia.org/resource/The_Practice_of_Programming, http://dbpedia.org/resource/The_Primal_Urge, http://dbpedia.org/resource/The_Primrose_Path_(Stoker_novel), http://dbpedia.org/resource/The_Princess_Bride, http://dbpedia.org/resource/The_Principles_of_Mathematics, http://dbpedia.org/resource/The_Problems_of_Philosophy, http://dbpedia.org/resource/The_Process_(novel), http://dbpedia.org/resource/The_Remains_of_the_Day, http://dbpedia.org/resource/The_Scarlet_Pimpernel, http://dbpedia.org/resource/The_Sea,_the_Sea, http://dbpedia.org/resource/The_Sea_(novel), http://dbpedia.org/resource/The_Secret_Life_of_Bees, http://dbpedia.org/resource/The_Sense_of_an_Ending, http://dbpedia.org/resource/The_Shoulder_of_Shasta, http://dbpedia.org/resource/The_Siege_of_Krishnapur, http://dbpedia.org/resource/The_Silmarillion, http://dbpedia.org/resource/The_Silver_Pigs, http://dbpedia.org/resource/The_Sly_Old_Cat, http://dbpedia.org/resource/The_Snake's_Pass, http://dbpedia.org/resource/The_Soul_of_Baseball, http://dbpedia.org/resource/The_Stand, http://dbpedia.org/resource/The_Story_of_Little_Black_Sambo, http://dbpedia.org/resource/The_Story_of_Miss_Moppet, http://dbpedia.org/resource/The_Story_of_a_Fierce_Bad_Rabbit, http://dbpedia.org/resource/The_Stronghold, http://dbpedia.org/resource/The_Swan_Thieves, http://dbpedia.org/resource/The_Tailor_of_Gloucester, http://dbpedia.org/resource/The_Tale_of_Benjamin_Bunny, http://dbpedia.org/resource/The_Tale_of_Ginger_and_Pickles, http://dbpedia.org/resource/The_Tale_of_Jemima_Puddle-Duck, http://dbpedia.org/resource/The_Tale_of_Johnny_Town-Mouse, http://dbpedia.org/resource/The_Tale_of_Kitty-in-Boots, http://dbpedia.org/resource/The_Tale_of_Little_Pig_Robinson, http://dbpedia.org/resource/The_Tale_of_Mr._Jeremy_Fisher, http://dbpedia.org/resource/The_Tale_of_Mr._Tod, http://dbpedia.org/resource/The_Tale_of_Mrs._Tiggy-Winkle, http://dbpedia.org/resource/The_Tale_of_Mrs._Tittlemouse, http://dbpedia.org/resource/The_Tale_of_Peter_Rabbit, http://dbpedia.org/resource/The_Tale_of_Pigling_Bland, http://dbpedia.org/resource/The_Tale_of_Samuel_Whiskers_or_The_Roly-Poly_Pudding, http://dbpedia.org/resource/The_Tale_of_Squirrel_Nutkin, http://dbpedia.org/resource/The_Tale_of_Timmy_Tiptoes, http://dbpedia.org/resource/The_Tale_of_Tom_Kitten, http://dbpedia.org/resource/The_Tale_of_Two_Bad_Mice, http://dbpedia.org/resource/The_Tale_of_the_Flopsy_Bunnies, http://dbpedia.org/resource/The_Tale_of_the_Pie_and_the_Patty-Pan, http://dbpedia.org/resource/The_Third_Mind, http://dbpedia.org/resource/The_Thirty-Nine_Steps, http://dbpedia.org/resource/The_Three_Musketeers, http://dbpedia.org/resource/The_Universal_Baseball_Association,_Inc.,_J._Henry_Waugh,_Prop., http://dbpedia.org/resource/The_Unix_Programming_Environment, http://dbpedia.org/resource/The_Unwinding, http://dbpedia.org/resource/The_Voyage_Out, http://dbpedia.org/resource/The_Waste_Land, http://dbpedia.org/resource/The_Water-Babies,_A_Fairy_Tale_for_a_Land_Baby, http://dbpedia.org/resource/The_Watter's_Mou', http://dbpedia.org/resource/The_White_Tiger, http://dbpedia.org/resource/The_Wonders_of_Nature, http://dbpedia.org/resource/Things_Hidden_Since_the_Foundation_of_the_World, http://dbpedia.org/resource/Three_Essays_on_the_Theory_of_Sexuality, http://dbpedia.org/resource/Time's_Arrow_(novel), http://dbpedia.org/resource/Tongues_of_Serpents, http://dbpedia.org/resource/Too_Far_to_Go, http://dbpedia.org/resource/Trainspotting_(novel), http://dbpedia.org/resource/Treasure_Island, http://dbpedia.org/resource/Troubles_(novel), http://dbpedia.org/resource/True_History_of_the_Kelly_Gang, http://dbpedia.org/resource/Under_the_Sunset, http://dbpedia.org/resource/United_States_Code, http://dbpedia.org/resource/Vanity_Fair_(novel), http://dbpedia.org/resource/Venetia_(Disraeli_novel), http://dbpedia.org/resource/Venus_in_Furs, http://dbpedia.org/resource/Vernon_God_Little, http://dbpedia.org/resource/Vivian_Grey, http://dbpedia.org/resource/We_Were_the_Mulvaneys, http://dbpedia.org/resource/With_the_Old_Breed, http://dbpedia.org/resource/Wolf_Hall, http://dbpedia.org/resource/World_Almanac, http://dbpedia.org/resource/Yu-Gi-Oh!_Trading_Card_Game, http://dbpedia.org/resource/100_Greatest_African_Americans, http://dbpedia.org/resource/1066_and_All_That, http://dbpedia.org/resource/A_Dictionary_of_Modern_Slang,_Cant,_and_Vulgar_Words, http://dbpedia.org/resource/A_Passage_to_India, http://dbpedia.org/resource/A_Planet_Named_Shayol, http://dbpedia.org/resource/A_Short_Account_of_the_Destruction_of_the_Indies, http://dbpedia.org/resource/A_Thousand_Plateaus, http://dbpedia.org/resource/Advanced_Programming_in_the_Unix_Environment, http://dbpedia.org/resource/Alpha_Ralpha_Boulevard, http://dbpedia.org/resource/Ancient_Law, http://dbpedia.org/resource/And_Quiet_Flows_the_Don, http://dbpedia.org/resource/Anne_of_Green_Gables, http://dbpedia.org/resource/Aradia,_or_the_Gospel_of_the_Witches, http://dbpedia.org/resource/Beyond_the_Pleasure_Principle, http://dbpedia.org/resource/Brewster's_Millions, http://dbpedia.org/resource/Broca's_Brain, http://dbpedia.org/resource/Buck_Rogers:_A_Life_in_the_Future, http://dbpedia.org/resource/Burning_Chrome, http://dbpedia.org/resource/Candide,_Part_II, http://dbpedia.org/resource/Candy_(Southern_and_Hoffenberg_novel), http://dbpedia.org/resource/Captain_Stormfield's_Visit_to_Heaven, http://dbpedia.org/resource/Carnosaur_(novel), http://dbpedia.org/resource/Cartesian_Meditations, http://dbpedia.org/resource/Charlie_and_the_Chocolate_Factory, http://dbpedia.org/resource/Chocolat_(novel), http://dbpedia.org/resource/Chéri_(novel), http://dbpedia.org/resource/Civilization_and_Its_Discontents, http://dbpedia.org/resource/Classic_of_Filial_Piety, http://dbpedia.org/resource/Clear_and_Present_Danger, http://dbpedia.org/resource/Cleopatra_(Gardner_novel), http://dbpedia.org/resource/Conan_of_Cimmeria, http://dbpedia.org/resource/Conspiracies_(novel), http://dbpedia.org/resource/Crank_(novel), http://dbpedia.org/resource/Cuestión_moral:_si_el_chocolate_quebranta_el_ayuno_eclesiástico, http://dbpedia.org/resource/Danny_Dunn,_Time_Traveler, http://dbpedia.org/resource/Descent_into_Hell_(novel), http://dbpedia.org/resource/Dialectic_of_Enlightenment, http://dbpedia.org/resource/Discovering_History_in_China, http://dbpedia.org/resource/Dune_Messiah, http://dbpedia.org/resource/Duo_(novel), http://dbpedia.org/resource/Emma_Brown, http://dbpedia.org/resource/Encyclopedia_of_Australian_Rock_and_Pop, http://dbpedia.org/resource/Encyclopedia_of_Chicago, http://dbpedia.org/resource/Fablehaven, http://dbpedia.org/resource/Female_Chauvinist_Pigs, http://dbpedia.org/resource/Flora_Europaea, http://dbpedia.org/resource/For_the_Strength_of_Youth, http://dbpedia.org/resource/God_Emperor_of_Dune, http://dbpedia.org/resource/Gravitation_(book), http://dbpedia.org/resource/Gulag:_A_History, http://dbpedia.org/resource/Guoyu_(book), http://dbpedia.org/resource/Haywire_(book), http://dbpedia.org/resource/Heretics_of_Dune, http://dbpedia.org/resource/Higher:_A_Historic_Race_to_the_Sky_and_the_Making_of_a_City, http://dbpedia.org/resource/Hunters_of_Dune, http://dbpedia.org/resource/I_Am_a_Barbarian, http://dbpedia.org/resource/I_Love_You,_Beth_Cooper, http://dbpedia.org/resource/In_a_Glass_Darkly, http://dbpedia.org/resource/Infinity's_Shore, http://dbpedia.org/resource/Judaism_as_a_Civilization, http://dbpedia.org/resource/Junie_B._Jones, http://dbpedia.org/resource/Jurassic_Park_(novel), http://dbpedia.org/resource/Kalevala, http://dbpedia.org/resource/La_Chatte, http://dbpedia.org/resource/La_Cleopatra_(poem), http://dbpedia.org/resource/Like_Water_for_Chocolate, http://dbpedia.org/resource/Lions'_Commentary_on_UNIX_6th_Edition,_with_Source_Code, http://dbpedia.org/resource/Little_Men, http://dbpedia.org/resource/Lives_of_the_Most_Excellent_Painters,_Sculptors,_and_Architects, http://dbpedia.org/resource/London_Plan, http://dbpedia.org/resource/Malcolm_X:_A_Life_of_Reinvention, http://dbpedia.org/resource/Man_Plus, http://dbpedia.org/resource/Miscellaneous_Morsels_from_Youyang, http://dbpedia.org/resource/Moses_and_Monotheism, http://dbpedia.org/resource/Moving_Pictures_(novel), http://dbpedia.org/resource/Mrs_Beeton's_Book_of_Household_Management, http://dbpedia.org/resource/Mythologies_(book), http://dbpedia.org/resource/Negative_Dialectics, http://dbpedia.org/resource/New_History_of_the_Five_Dynasties, http://dbpedia.org/resource/No_Great_Mischief, http://dbpedia.org/resource/Norstrilia, http://dbpedia.org/resource/Nuclear_War_Survival_Skills, http://dbpedia.org/resource/Of_Grammatology, http://dbpedia.org/resource/On_Narcissism, http://dbpedia.org/resource/One-Dimensional_Man, http://dbpedia.org/resource/Oryx_and_Crake, http://dbpedia.org/resource/Pale_Blue_Dot_(book), http://dbpedia.org/resource/Pattern_Recognition_(novel), http://dbpedia.org/resource/Plum_Island_(novel), http://dbpedia.org/resource/Principles_of_Political_Economy, http://dbpedia.org/resource/Programming_Perl, http://dbpedia.org/resource/Rabbit_Redux, http://dbpedia.org/resource/Records_of_the_Three_Kingdoms, http://dbpedia.org/resource/Red_Nails, http://dbpedia.org/resource/Sandworms_of_Dune, http://dbpedia.org/resource/Scanners_Live_in_Vain, http://dbpedia.org/resource/Science_and_Civilisation_in_China, http://dbpedia.org/resource/Shirley_(novel), http://dbpedia.org/resource/Shulchan_Aruch, http://dbpedia.org/resource/Simulacron-3, http://dbpedia.org/resource/Something_Rotten, http://dbpedia.org/resource/Space_Lords_(short_story_collection), http://dbpedia.org/resource/Spock_Must_Die!, http://dbpedia.org/resource/Spring_and_Autumn_Annals_of_the_Sixteen_Kingdoms, http://dbpedia.org/resource/Syrup_(novel), http://dbpedia.org/resource/TCP/IP_Illustrated, http://dbpedia.org/resource/The_100_Most_Influential_Books_Ever_Written, http://dbpedia.org/resource/The_Andromeda_Strain, http://dbpedia.org/resource/The_Authoritarian_Personality, http://dbpedia.org/resource/The_Bicentennial_Man, http://dbpedia.org/resource/The_Bloody_Crown_of_Conan, http://dbpedia.org/resource/The_Blue_Equinox, http://dbpedia.org/resource/The_Book_of_the_New_Sun, http://dbpedia.org/resource/The_Boys_from_Brazil_(novel), http://dbpedia.org/resource/The_Bridges_of_Madison_County, http://dbpedia.org/resource/The_Bull_from_the_Sea, http://dbpedia.org/resource/The_Cambridge_History_of_Iran, http://dbpedia.org/resource/The_Captive_Mind, http://dbpedia.org/resource/The_Coming_of_Conan, http://dbpedia.org/resource/The_Coming_of_Conan_the_Cimmerian, http://dbpedia.org/resource/The_Complete_Robot, http://dbpedia.org/resource/The_Compound_(book), http://dbpedia.org/resource/The_Conquering_Sword_of_Conan, http://dbpedia.org/resource/The_Constant_Princess, http://dbpedia.org/resource/The_Destiny_of_The_Mother_Church, http://dbpedia.org/resource/The_Door_into_Summer, http://dbpedia.org/resource/The_Dragons_of_Eden, http://dbpedia.org/resource/The_Encyclopedia_of_Science_Fiction, http://dbpedia.org/resource/The_Forme_of_Cury, http://dbpedia.org/resource/The_Fortunes_of_Nigel, http://dbpedia.org/resource/The_Future_of_an_Illusion, http://dbpedia.org/resource/The_Gold-Bug, http://dbpedia.org/resource/The_Great_Illusion, http://dbpedia.org/resource/The_Hacker_Crackdown, http://dbpedia.org/resource/The_Hour_of_the_Dragon, http://dbpedia.org/resource/The_Interpretation_of_Dreams, http://dbpedia.org/resource/The_Last_Days_of_Pompeii, http://dbpedia.org/resource/The_Last_Supper_(novel), http://dbpedia.org/resource/The_Long_Journey, http://dbpedia.org/resource/The_Lord_(book), http://dbpedia.org/resource/The_Manchurian_Candidate, http://dbpedia.org/resource/The_Meadows_of_Gold, http://dbpedia.org/resource/The_Miernik_Dossier, http://dbpedia.org/resource/The_Moth_Diaries, http://dbpedia.org/resource/The_Mysterious_Caravan, http://dbpedia.org/resource/The_Myth_of_the_Vaginal_Orgasm, http://dbpedia.org/resource/The_Mythical_Man-Month, http://dbpedia.org/resource/The_Name_of_the_Rose, http://dbpedia.org/resource/The_Other_Boleyn_Girl, http://dbpedia.org/resource/The_People_of_the_Black_Circle, http://dbpedia.org/resource/The_Petrified_Planet, http://dbpedia.org/resource/The_Prime_of_Miss_Jean_Brodie_(novel), http://dbpedia.org/resource/The_Principle_of_Hope, http://dbpedia.org/resource/The_Rediscovery_of_Man, http://dbpedia.org/resource/The_Rise_and_Fall_of_the_Confederate_Government, http://dbpedia.org/resource/The_Scarlet_Citadel, http://dbpedia.org/resource/The_Secret_Lovers_(novel), http://dbpedia.org/resource/The_Ship_Who_Sang, http://dbpedia.org/resource/The_Shipping_News, http://dbpedia.org/resource/The_Siege_of_Shkodra_(book), http://dbpedia.org/resource/The_Star_Rover, http://dbpedia.org/resource/The_Stone_Diaries, http://dbpedia.org/resource/The_Structural_Transformation_of_the_Public_Sphere, http://dbpedia.org/resource/The_Tears_of_Autumn, http://dbpedia.org/resource/The_Theory_of_Communicative_Action, http://dbpedia.org/resource/The_Turn_of_the_Screw, http://dbpedia.org/resource/The_Vagabond_(novel), http://dbpedia.org/resource/The_Vampire_Prince, http://dbpedia.org/resource/The_War_Against_the_Jews, http://dbpedia.org/resource/The_World_Next_Door, http://dbpedia.org/resource/The_Wretched_of_the_Earth, http://dbpedia.org/resource/Thirteen_Days_(book), http://dbpedia.org/resource/Totem_and_Taboo, http://dbpedia.org/resource/Two_Solitudes_(novel), http://dbpedia.org/resource/U.S.A._(trilogy), http://dbpedia.org/resource/Ur_(novella), http://dbpedia.org/resource/Villette_(novel), http://dbpedia.org/resource/We_(novel), http://dbpedia.org/resource/When_the_People_Fell, http://dbpedia.org/resource/Writing_and_Difference, http://dbpedia.org/resource/Xunzi_(book), http://dbpedia.org/resource/You_Can't_Go_Home_Again, http://dbpedia.org/resource/Zadig, http://dbpedia.org/resource/A_Dynamical_Theory_of_the_Electromagnetic_Field, http://dbpedia.org/resource/A_Fighting_Man_of_Mars, http://dbpedia.org/resource/A_Nation_at_Risk, http://dbpedia.org/resource/A_Princess_of_Mars, http://dbpedia.org/resource/A_Theory_of_Justice, http://dbpedia.org/resource/A_Visit_from_the_Goon_Squad, http://dbpedia.org/resource/Abramowitz_and_Stegun, http://dbpedia.org/resource/Across_the_River_and_into_the_Trees, http://dbpedia.org/resource/Almost_Like_a_Whale, http://dbpedia.org/resource/An_Antarctic_Mystery, http://dbpedia.org/resource/Anarchism_and_Other_Essays, http://dbpedia.org/resource/Animals_in_Translation, http://dbpedia.org/resource/Anti-Oedipus, http://dbpedia.org/resource/Around_the_Moon, http://dbpedia.org/resource/Arundhati_(epic), http://dbpedia.org/resource/Ashtavakra_(epic), http://dbpedia.org/resource/Astronautilia, http://dbpedia.org/resource/At_the_Earth's_Core_(novel), http://dbpedia.org/resource/Auto-da-Fé_(novel), http://dbpedia.org/resource/Back_to_the_Stone_Age, http://dbpedia.org/resource/Banglapedia, http://dbpedia.org/resource/Beyond_Thirty, http://dbpedia.org/resource/Beyond_the_Farthest_Star_(novel), http://dbpedia.org/resource/Biographia_Literaria, http://dbpedia.org/resource/Bureaucracy_(book), http://dbpedia.org/resource/Bye_Bye_Birdie_(musical), http://dbpedia.org/resource/Canto_General, http://dbpedia.org/resource/Caramuru_(epic_poem), http://dbpedia.org/resource/Carrie_(novel), http://dbpedia.org/resource/Carson_of_Venus, http://dbpedia.org/resource/Charlotte's_Web, http://dbpedia.org/resource/Children_of_the_Lens_(novel), http://dbpedia.org/resource/Clarissa, http://dbpedia.org/resource/Crime_and_Punishment, http://dbpedia.org/resource/Critique_of_Dialectical_Reason, http://dbpedia.org/resource/Daddy-Long-Legs_(novel), http://dbpedia.org/resource/Death_in_the_Afternoon, http://dbpedia.org/resource/Democracy_and_Education, http://dbpedia.org/resource/Dictionary_of_American_Naval_Fighting_Ships, http://dbpedia.org/resource/Difference_and_Repetition, http://dbpedia.org/resource/Doc_Savage:_His_Apocalyptic_Life, http://dbpedia.org/resource/Doctor_Who:_A_Celebration, http://dbpedia.org/resource/Doctor_Who_Prom_(2010), http://dbpedia.org/resource/Doctor_Who_Prom_(2013), http://dbpedia.org/resource/Don't_Go_Near_the_Water_(novel), http://dbpedia.org/resource/Dr._Heidenhoff's_Process, http://dbpedia.org/resource/Dream_Story, http://dbpedia.org/resource/Duma_Key, http://dbpedia.org/resource/Dwellers_in_the_Mirage, http://dbpedia.org/resource/Earthlight, http://dbpedia.org/resource/Edison's_Conquest_of_Mars, http://dbpedia.org/resource/Edvard_Grieg_–_mennesket_og_kunstneren, http://dbpedia.org/resource/Elizabeth_Barrett_Browning:_A_Biography, http://dbpedia.org/resource/Elvis:_What_Happened%3F, http://dbpedia.org/resource/Emperor_Shaka_the_Great, http://dbpedia.org/resource/Enemies:_How_America's_Foes_Steal_Our_Vital_Secrets_–_And_How_We_Let_It_Happen, http://dbpedia.org/resource/Enlightenment_(Doctor_Who), http://dbpedia.org/resource/Enola;_or,_Her_fatal_mistake, http://dbpedia.org/resource/Equality_(novel), http://dbpedia.org/resource/Erewhon_Revisited, http://dbpedia.org/resource/Escape_on_Venus, http://dbpedia.org/resource/Essays_(Montaigne), http://dbpedia.org/resource/Evelina, http://dbpedia.org/resource/Evolution_and_the_Theory_of_Games, http://dbpedia.org/resource/Evolving_the_Alien, http://dbpedia.org/resource/Experience_and_Education_(book), http://dbpedia.org/resource/First_Lensman, http://dbpedia.org/resource/Flush:_A_Biography, http://dbpedia.org/resource/Following_the_Equator, http://dbpedia.org/resource/Forgotten_Tales_of_Love_and_Murder, http://dbpedia.org/resource/Fredy_Neptune, http://dbpedia.org/resource/Freud_and_Philosophy, http://dbpedia.org/resource/Friday_(novel), http://dbpedia.org/resource/Gadsby_(novel), http://dbpedia.org/resource/Galactic_Patrol_(novel), http://dbpedia.org/resource/Gitaramayanam, http://dbpedia.org/resource/Gondibert, http://dbpedia.org/resource/Gradshteyn_and_Ryzhik, http://dbpedia.org/resource/Gray_Lensman, http://dbpedia.org/resource/Great_Soviet_Encyclopedia, http://dbpedia.org/resource/Green_Hills_of_Africa, http://dbpedia.org/resource/Hacking:_The_Art_of_Exploitation, http://dbpedia.org/resource/Herzog_(novel), http://dbpedia.org/resource/I-Juca-Pirama, http://dbpedia.org/resource/If_I_Were_a_Boy_(novel), http://dbpedia.org/resource/If_Women_Counted, http://dbpedia.org/resource/Il_Conquisto_di_Granata, http://dbpedia.org/resource/Infinite_Worlds:_An_Illustrated_Voyage_to_Planets_Beyond_Our_Sun, http://dbpedia.org/resource/J_R, http://dbpedia.org/resource/James_and_the_Giant_Peach, http://dbpedia.org/resource/John_Carter_of_Mars_(collection), http://dbpedia.org/resource/Journey_to_the_End_of_the_Night, http://dbpedia.org/resource/Jungle_Girl_(novel), http://dbpedia.org/resource/Jungle_Tales_of_Tarzan, http://dbpedia.org/resource/Kidnapped_(novel), http://dbpedia.org/resource/La_Argentina_(poem), http://dbpedia.org/resource/La_Légende_des_siècles, http://dbpedia.org/resource/Lady_Susan, http://dbpedia.org/resource/Land_of_Terror, http://dbpedia.org/resource/Legacy_of_the_Daleks, http://dbpedia.org/resource/Lemony_Snicket:_The_Unauthorized_Autobiography, http://dbpedia.org/resource/Les_Liaisons_dangereuses, http://dbpedia.org/resource/Letters_of_a_Portuguese_Nun, http://dbpedia.org/resource/Life_of_Samuel_Johnson, http://dbpedia.org/resource/Ligeia, http://dbpedia.org/resource/Lincoln's_Gamble, http://dbpedia.org/resource/Living_My_Life, http://dbpedia.org/resource/Llana_of_Gathol, http://dbpedia.org/resource/Logical_Investigations_(Husserl), http://dbpedia.org/resource/Lost_on_Venus, http://dbpedia.org/resource/Love-Letters_Between_a_Nobleman_and_His_Sister, http://dbpedia.org/resource/Lunar_Park, http://dbpedia.org/resource/Mary_Had_a_Little_Lamb, http://dbpedia.org/resource/Master_of_Adventure:_The_Worlds_of_Edgar_Rice_Burroughs, http://dbpedia.org/resource/Men_Without_Women_(short_story_collection), http://dbpedia.org/resource/Men_of_Mathematics, http://dbpedia.org/resource/Methuselah's_Children, http://dbpedia.org/resource/Moose:_Chapters_from_My_Life, http://dbpedia.org/resource/Morella_(short_story), http://dbpedia.org/resource/My_Disillusionment_in_Russia, http://dbpedia.org/resource/My_Further_Disillusionment_in_Russia, http://dbpedia.org/resource/Nana_(novel), http://dbpedia.org/resource/Natural_Theology_or_Evidences_of_the_Existence_and_Attributes_of_the_Deity, http://dbpedia.org/resource/Nausea_(novel), http://dbpedia.org/resource/New_Watch_(novel), http://dbpedia.org/resource/News_from_Nowhere, http://dbpedia.org/resource/Notes_from_Underground, http://dbpedia.org/resource/Object-Oriented_Software_Construction, http://dbpedia.org/resource/Omnipotent_Government, http://dbpedia.org/resource/One_Flew_Over_the_Cuckoo's_Nest_(novel), http://dbpedia.org/resource/Our_Gang_(novel), http://dbpedia.org/resource/Pan_Tadeusz, http://dbpedia.org/resource/Pellucidar_(novel), http://dbpedia.org/resource/Phenomenology_of_Perception, http://dbpedia.org/resource/Philosophie_Zoologique, http://dbpedia.org/resource/Philosophy_of_Arithmetic, http://dbpedia.org/resource/Pirates_of_Venus, http://dbpedia.org/resource/Poor_Folk, http://dbpedia.org/resource/Pride_and_Prejudice, http://dbpedia.org/resource/Prime_Obsession, http://dbpedia.org/resource/Prisoner_of_the_Daleks, http://dbpedia.org/resource/Project_Management_Body_of_Knowledge, http://dbpedia.org/resource/Rashmirathi, http://dbpedia.org/resource/Rights_of_Man, http://dbpedia.org/resource/Rob_Roy_(novel), http://dbpedia.org/resource/Rommel:_The_Desert_Fox, http://dbpedia.org/resource/Salve,_Oh_Patria, http://dbpedia.org/resource/San_Manuel_Bueno,_Mártir, http://dbpedia.org/resource/Sasha_and_Emma, http://dbpedia.org/resource/Savage_Pellucidar, http://dbpedia.org/resource/Scepticism_and_Animal_Faith, http://dbpedia.org/resource/Scouting_for_Boys, http://dbpedia.org/resource/Second_Stage_Lensmen, http://dbpedia.org/resource/Sex_at_Dawn, http://dbpedia.org/resource/Skylark_DuQuesne, http://dbpedia.org/resource/Skylark_Three, http://dbpedia.org/resource/Skylark_of_Valeron, http://dbpedia.org/resource/Slaughterhouse-Five, http://dbpedia.org/resource/Socialism_(book), http://dbpedia.org/resource/Sociobiology:_The_New_Synthesis, http://dbpedia.org/resource/Some_of_Your_Blood, http://dbpedia.org/resource/Spacehounds_of_IPC, http://dbpedia.org/resource/Spycatcher, http://dbpedia.org/resource/Sribhargavaraghaviyam, http://dbpedia.org/resource/Star_Smashers_of_the_Galaxy_Rangers, http://dbpedia.org/resource/Strange_Case_of_Dr_Jekyll_and_Mr_Hyde, http://dbpedia.org/resource/Stuart_Little, http://dbpedia.org/resource/Subspace_Explorers, http://dbpedia.org/resource/Super_Sad_True_Love_Story, http://dbpedia.org/resource/Swords_of_Mars, http://dbpedia.org/resource/Synthetic_Men_of_Mars, http://dbpedia.org/resource/Tales_of_the_Grotesque_and_Arabesque, http://dbpedia.org/resource/Tamerlane_and_Other_Poems, http://dbpedia.org/resource/Tanar_of_Pellucidar, http://dbpedia.org/resource/Tarzan's_Quest, http://dbpedia.org/resource/Tarzan,_Lord_of_the_Jungle_(novel), http://dbpedia.org/resource/Tarzan_Triumphant, http://dbpedia.org/resource/Tarzan_and_the_Ant_Men, http://dbpedia.org/resource/Tarzan_and_the_Castaways, http://dbpedia.org/resource/Tarzan_and_the_City_of_Gold, http://dbpedia.org/resource/Tarzan_and_the_Forbidden_City, http://dbpedia.org/resource/Tarzan_and_the_Foreign_Legion, http://dbpedia.org/resource/Tarzan_and_the_Golden_Lion, http://dbpedia.org/resource/Tarzan_and_the_Jewels_of_Opar, http://dbpedia.org/resource/Tarzan_and_the_Leopard_Men, http://dbpedia.org/resource/Tarzan_and_the_Lion_Man, http://dbpedia.org/resource/Tarzan_and_the_Lost_Empire, http://dbpedia.org/resource/Tarzan_and_the_Madman, http://dbpedia.org/resource/Tarzan_and_the_Tarzan_Twins, http://dbpedia.org/resource/Tarzan_at_the_Earth's_Core, http://dbpedia.org/resource/Tarzan_of_the_Apes, http://dbpedia.org/resource/Tarzan_the_Invincible, http://dbpedia.org/resource/Tarzan_the_Magnificent_(novel), http://dbpedia.org/resource/Tarzan_the_Terrible, http://dbpedia.org/resource/Tarzan_the_Untamed, http://dbpedia.org/resource/The_Adapted_Mind, http://dbpedia.org/resource/The_Also_People, http://dbpedia.org/resource/The_Amazing_Adventures_of_Kavalier_&_Clay, http://dbpedia.org/resource/The_Anderson_Tapes_(novel), http://dbpedia.org/resource/The_Art_of_Unix_Programming, http://dbpedia.org/resource/The_Bandit_of_Hell's_Bend, http://dbpedia.org/resource/The_Banner_of_Joan, http://dbpedia.org/resource/The_Battlefield_Where_The_Moon_Says_I_Love_You, http://dbpedia.org/resource/The_Beasts_of_Tarzan, http://dbpedia.org/resource/The_Bell_Jar, http://dbpedia.org/resource/The_Black_Cat_(short_story), http://dbpedia.org/resource/The_Blank_Slate, http://dbpedia.org/resource/The_Blood_of_Others, http://dbpedia.org/resource/The_Cat_Who_Walks_Through_Walls, http://dbpedia.org/resource/The_Cave_Girl, http://dbpedia.org/resource/The_Changing_Light_at_Sandover, http://dbpedia.org/resource/The_Chessmen_of_Mars, http://dbpedia.org/resource/The_Color_Purple, http://dbpedia.org/resource/The_Confidential_Agent, http://dbpedia.org/resource/The_Constitution_of_Liberty, http://dbpedia.org/resource/The_Coquette, http://dbpedia.org/resource/The_Dalek_Factor, http://dbpedia.org/resource/The_Dalek_Generation, http://dbpedia.org/resource/The_Dangerous_Summer, http://dbpedia.org/resource/The_Descent_of_Man,_and_Selection_in_Relation_to_Sex, http://dbpedia.org/resource/The_Divine_Invasion, http://dbpedia.org/resource/The_Documents_in_the_Case, http://dbpedia.org/resource/The_Efficiency_Expert_(novel), http://dbpedia.org/resource/The_Electric_Kool-Aid_Acid_Test, http://dbpedia.org/resource/The_Elements_of_Style, http://dbpedia.org/resource/The_Enemy_(Bagley_novel), http://dbpedia.org/resource/The_Enemy_(Child_novel), http://dbpedia.org/resource/The_Enemy_(Higson_novel), http://dbpedia.org/resource/The_Eternal_Lover, http://dbpedia.org/resource/The_Evolution_of_Cooperation, http://dbpedia.org/resource/The_Expression_of_the_Emotions_in_Man_and_Animals, http://dbpedia.org/resource/The_Facts_in_the_Case_of_M._Valdemar, http://dbpedia.org/resource/The_Fall_of_Arthur, http://dbpedia.org/resource/The_Fifth_Column_and_the_First_Forty-Nine_Stories, http://dbpedia.org/resource/The_Fifth_Mountain, http://dbpedia.org/resource/The_Garden_of_Eden_(novel), http://dbpedia.org/resource/The_Girl_from_Hollywood, http://dbpedia.org/resource/The_Gods_of_Mars, http://dbpedia.org/resource/The_Guernsey_Literary_and_Potato_Peel_Pie_Society, http://dbpedia.org/resource/The_Haymarket_Tragedy, http://dbpedia.org/resource/The_Heroin_Diaries:_A_Year_in_the_Life_of_a_Shattered_Rock_Star, http://dbpedia.org/resource/The_House_of_Hades, http://dbpedia.org/resource/The_Jungle_Book, http://dbpedia.org/resource/The_Justice_of_God_in_the_Damnation_of_Sinners, http://dbpedia.org/resource/The_King_of_Elfland's_Daughter, http://dbpedia.org/resource/The_Lad_and_the_Lion, http://dbpedia.org/resource/The_Land_That_Time_Forgot_(novel), http://dbpedia.org/resource/The_Legend_of_Sigurd_and_Gudrún, http://dbpedia.org/resource/The_Mad_King, http://dbpedia.org/resource/The_Man-Eater, http://dbpedia.org/resource/The_Man_Who_Loved_Only_Numbers, http://dbpedia.org/resource/The_Master_Mind_of_Mars, http://dbpedia.org/resource/The_Monster_Men, http://dbpedia.org/resource/The_Moon_Maid, http://dbpedia.org/resource/The_Moon_Pool, http://dbpedia.org/resource/The_Moral_Landscape, http://dbpedia.org/resource/The_Mountain_Wreath, http://dbpedia.org/resource/The_Number_of_the_Beast_(novel), http://dbpedia.org/resource/The_Oakdale_Affair, http://dbpedia.org/resource/The_Outlaw_of_Torn, http://dbpedia.org/resource/The_People_That_Time_Forgot_(novel), http://dbpedia.org/resource/The_Pit_and_the_Pendulum, http://dbpedia.org/resource/The_Poe_Shadow, http://dbpedia.org/resource/The_Premature_Burial, http://dbpedia.org/resource/The_Private_Memoirs_and_Confessions_of_a_Justified_Sinner, http://dbpedia.org/resource/The_Protestant_Ethic_and_the_Spirit_of_Capitalism, http://dbpedia.org/resource/The_Puzzle_Palace, http://dbpedia.org/resource/The_Red_Queen:_Sex_and_the_Evolution_of_Human_Nature, http://dbpedia.org/resource/The_Return_of_Tarzan, http://dbpedia.org/resource/The_Rider, http://dbpedia.org/resource/The_Road_to_Reality, http://dbpedia.org/resource/The_Sands_of_Time_(Sheldon_novel), http://dbpedia.org/resource/The_Second_Sex, http://dbpedia.org/resource/The_Selfish_Gene, http://dbpedia.org/resource/The_Sexual_Life_of_Savages_in_North-Western_Melanesia, http://dbpedia.org/resource/The_Ship_of_Ishtar, http://dbpedia.org/resource/The_Sickness_Unto_Death, http://dbpedia.org/resource/The_Skylark_of_Space, http://dbpedia.org/resource/The_Son_of_Tarzan, http://dbpedia.org/resource/The_Sorrows_of_Young_Werther, http://dbpedia.org/resource/The_Story_of_Sigurd_the_Volsung_and_the_Fall_of_the_Niblungs, http://dbpedia.org/resource/The_Structure_of_Evolutionary_Theory, http://dbpedia.org/resource/The_Sword_in_the_Stone_(novel), http://dbpedia.org/resource/The_Symbolic_Species, http://dbpedia.org/resource/The_System_of_Doctor_Tarr_and_Professor_Fether, http://dbpedia.org/resource/The_Tale_of_Genji, http://dbpedia.org/resource/The_Tell-Tale_Heart, http://dbpedia.org/resource/The_Tenth_Muse_Lately_Sprung_Up_in_America, http://dbpedia.org/resource/The_Theory_of_Evolution, http://dbpedia.org/resource/The_Theory_of_Island_Biogeography, http://dbpedia.org/resource/The_True_Believer, http://dbpedia.org/resource/The_Trumpet_of_the_Swan, http://dbpedia.org/resource/The_Vortex_Blaster, http://dbpedia.org/resource/The_Wanderings_of_Oisin, http://dbpedia.org/resource/The_War_of_the_Worlds, http://dbpedia.org/resource/The_Warlord_of_Mars, http://dbpedia.org/resource/The_Wizard_of_Venus, http://dbpedia.org/resource/The_Yellow_Wallpaper, http://dbpedia.org/resource/Thuvia,_Maid_of_Mars, http://dbpedia.org/resource/Triplanetary_(novel), http://dbpedia.org/resource/Uncommon_Valour, http://dbpedia.org/resource/Unofficial_Football_World_Championships, http://dbpedia.org/resource/War_of_the_Daleks, http://dbpedia.org/resource/What_Engineers_Know_and_How_They_Know_It, http://dbpedia.org/resource/What_Is_Life%3F, http://dbpedia.org/resource/What_the_Dormouse_Said, http://dbpedia.org/resource/Why_We_Broke_Up, http://dbpedia.org/resource/World_Book_Encyclopedia, http://dbpedia.org/resource/World_War_Z, http://dbpedia.org/resource/A_Case_of_Need, http://dbpedia.org/resource/A_Gathering_of_Days, http://dbpedia.org/resource/A_Lie_of_Reinvention, http://dbpedia.org/resource/A_Perfect_Spy, http://dbpedia.org/resource/A_Universe_from_Nothing, http://dbpedia.org/resource/Adaptation_and_Natural_Selection, http://dbpedia.org/resource/Airframe_(novel), http://dbpedia.org/resource/Barometer_Rising, http://dbpedia.org/resource/Bored_of_the_Rings, http://dbpedia.org/resource/Brainwashing:_The_Science_of_Thought_Control, http://dbpedia.org/resource/Cetaganda, http://dbpedia.org/resource/Cults_in_Our_Midst, http://dbpedia.org/resource/Das_Dritte_Reich, http://dbpedia.org/resource/Dealing:_or_the_Berkeley-to-Boston_Forty-Brick_Lost-Bag_Blues, http://dbpedia.org/resource/Disclosure_(novel), http://dbpedia.org/resource/Discourses_(Meher_Baba), http://dbpedia.org/resource/Dragon_Teeth, http://dbpedia.org/resource/Driving_Like_Crazy, http://dbpedia.org/resource/Drug_of_Choice, http://dbpedia.org/resource/Dune:_The_Battle_of_Corrin, http://dbpedia.org/resource/Easy_Go, http://dbpedia.org/resource/Ecology:_From_Individuals_to_Ecosystems, http://dbpedia.org/resource/Electronic_Life, http://dbpedia.org/resource/Ever_Since_Darwin, http://dbpedia.org/resource/Five_Patients, http://dbpedia.org/resource/God_Speaks, http://dbpedia.org/resource/God_in_a_Pill%3F, http://dbpedia.org/resource/Grave_Descend, http://dbpedia.org/resource/Growing_Up_X, http://dbpedia.org/resource/Heart-Shaped_Box_(novel), http://dbpedia.org/resource/Historia_verdadera_de_la_conquista_de_la_Nueva_España, http://dbpedia.org/resource/Hitman:_Enemy_Within, http://dbpedia.org/resource/Ideas_Have_Consequences, http://dbpedia.org/resource/In_Search_of_Schrödinger's_Cat, http://dbpedia.org/resource/Inquisition:_The_Persecution_and_Prosecution_of_the_Reverend_Sun_Myung_Moon, http://dbpedia.org/resource/Jasper_Johns_(book), http://dbpedia.org/resource/Jean_Santeuil, http://dbpedia.org/resource/Komarr, http://dbpedia.org/resource/Kristin_Lavransdatter, http://dbpedia.org/resource/Liar's_Poker, http://dbpedia.org/resource/Look_Me_in_the_Eye, http://dbpedia.org/resource/Manual_of_the_Planes, http://dbpedia.org/resource/Max_Havelaar, http://dbpedia.org/resource/Meat_Loaf:_To_Hell_and_Back, http://dbpedia.org/resource/Memory_(Bujold_novel), http://dbpedia.org/resource/Micro_(novel), http://dbpedia.org/resource/More-with-Less_Cookbook, http://dbpedia.org/resource/My_Opposition, http://dbpedia.org/resource/My_Turn_(memoir), http://dbpedia.org/resource/My_Uncle_Napoleon, http://dbpedia.org/resource/Necessary_Illusions, http://dbpedia.org/resource/Next_(novel), http://dbpedia.org/resource/Nibbana-The_Mind_Stilled, http://dbpedia.org/resource/Odds_On, http://dbpedia.org/resource/On_the_Juche_Idea, http://dbpedia.org/resource/Peyton_Place_(novel), http://dbpedia.org/resource/Pirate_Latitudes, http://dbpedia.org/resource/Present_Danger, http://dbpedia.org/resource/Prey_(novel), http://dbpedia.org/resource/Propaganda:_The_Formation_of_Men's_Attitudes, http://dbpedia.org/resource/Rising_Sun_(novel), http://dbpedia.org/resource/Rough_Crossings, http://dbpedia.org/resource/SCUM_Manifesto, http://dbpedia.org/resource/Scratch_One, http://dbpedia.org/resource/Sister_Time, http://dbpedia.org/resource/Sphere_(novel), http://dbpedia.org/resource/Stages_on_Life's_Way, http://dbpedia.org/resource/Teachings_of_the_Prophet_Joseph_Smith_(book), http://dbpedia.org/resource/The_Autobiography_of_Malcolm_X, http://dbpedia.org/resource/The_Bishop's_Man, http://dbpedia.org/resource/The_Black_Mountain, http://dbpedia.org/resource/The_Book_of_Negroes, http://dbpedia.org/resource/The_Cleanest_Race, http://dbpedia.org/resource/The_Cluetrain_Manifesto, http://dbpedia.org/resource/The_Decline_and_Fall_of_Nokia, http://dbpedia.org/resource/The_Devil_and_Daniel_Webster, http://dbpedia.org/resource/The_Diary_of_Malcolm_X, http://dbpedia.org/resource/The_Gender_Knot:_Unraveling_our_Patriarchal_Legacy, http://dbpedia.org/resource/The_Gospel_According_to_Jesus_Christ, http://dbpedia.org/resource/The_Great_Train_Robbery_(novel), http://dbpedia.org/resource/The_Greening_of_America, http://dbpedia.org/resource/The_Hero_(novel), http://dbpedia.org/resource/The_Informant_(book), http://dbpedia.org/resource/The_King_Must_Die, http://dbpedia.org/resource/The_Lost_World_(Crichton_novel), http://dbpedia.org/resource/The_Man_Who_Mistook_His_Wife_for_a_Hat, http://dbpedia.org/resource/The_Manipulated_Man, http://dbpedia.org/resource/The_Myth_of_Male_Power, http://dbpedia.org/resource/The_Pleasantries_of_the_Incredible_Mulla_Nasrudin, http://dbpedia.org/resource/The_Predators'_Ball, http://dbpedia.org/resource/The_Prime_Ministers, http://dbpedia.org/resource/The_Principles_of_Quantum_Mechanics, http://dbpedia.org/resource/The_Punch_and_Judy_Murders, http://dbpedia.org/resource/The_Rise_of_the_Meritocracy, http://dbpedia.org/resource/The_Second_World_War_(book), http://dbpedia.org/resource/The_Secret_Magdalene, http://dbpedia.org/resource/The_Soul_of_a_Butterfly, http://dbpedia.org/resource/The_Souls_of_Black_Folk, http://dbpedia.org/resource/The_Steam_House, http://dbpedia.org/resource/The_Sword_of_Moses_(novel), http://dbpedia.org/resource/The_Templar_Revelation, http://dbpedia.org/resource/The_Terminal_Man, http://dbpedia.org/resource/The_Titan's_Curse, http://dbpedia.org/resource/The_Venom_Business, http://dbpedia.org/resource/The_Zombie_Survival_Guide, http://dbpedia.org/resource/Travels_(book), http://dbpedia.org/resource/Who_Speaks_for_the_Negro%3F, http://dbpedia.org/resource/Worldwar:_Striking_the_Balance, http://dbpedia.org/resource/Zero_Cool, http://dbpedia.org/resource/A_Door_into_Ocean, http://dbpedia.org/resource/A_Fringe_of_Leaves, http://dbpedia.org/resource/A_Greek–English_Lexicon, http://dbpedia.org/resource/A_Matter_of_Time_(novel), http://dbpedia.org/resource/A_Moral_Reckoning, http://dbpedia.org/resource/A_Quantum_Murder, http://dbpedia.org/resource/A_Random_Walk_Down_Wall_Street, http://dbpedia.org/resource/A_Second_Chance_at_Eden, http://dbpedia.org/resource/A_Short_History_of_Tractors_in_Ukrainian, http://dbpedia.org/resource/A_Sorrow_Beyond_Dreams, http://dbpedia.org/resource/A_Spot_of_Bother, http://dbpedia.org/resource/A_Wish_for_Wings_That_Work, http://dbpedia.org/resource/Alfred_Hitchcock_and_the_Making_of_Psycho, http://dbpedia.org/resource/Allan_Stein, http://dbpedia.org/resource/Ally_(novel), http://dbpedia.org/resource/Alongside_Night, http://dbpedia.org/resource/Amnesia_(Carey_novel), http://dbpedia.org/resource/Ancillary_Justice, http://dbpedia.org/resource/And_Tango_Makes_Three, http://dbpedia.org/resource/And_the_Band_Played_On, http://dbpedia.org/resource/Augustinus_(Jansenist_book), http://dbpedia.org/resource/Beyond_the_Barrier, http://dbpedia.org/resource/Biochemical_Predestination, http://dbpedia.org/resource/Bliss_(novel), http://dbpedia.org/resource/Bone_Dance, http://dbpedia.org/resource/Book_of_Longing, http://dbpedia.org/resource/Brown_Girl_in_the_Ring_(novel), http://dbpedia.org/resource/Caesar's_Women, http://dbpedia.org/resource/Cagebird_(novel), http://dbpedia.org/resource/Carnival_(Bear_novel), http://dbpedia.org/resource/Chainfire, http://dbpedia.org/resource/Chasing_Vermeer, http://dbpedia.org/resource/Chthon_(novel), http://dbpedia.org/resource/City_of_Pearl, http://dbpedia.org/resource/Clade_(novel), http://dbpedia.org/resource/Company_of_Liars, http://dbpedia.org/resource/Constantine's_Sword, http://dbpedia.org/resource/Countdown_City, http://dbpedia.org/resource/Courtship_Rite, http://dbpedia.org/resource/Cowl_(novel), http://dbpedia.org/resource/Crossing_the_Threshold_of_Hope, http://dbpedia.org/resource/Cuss_Control, http://dbpedia.org/resource/Dante's_Equation, http://dbpedia.org/resource/Darcy's_Story, http://dbpedia.org/resource/Dark_Rivers_of_the_Heart, http://dbpedia.org/resource/De_verborum_significatu, http://dbpedia.org/resource/Deadline_(Grant_novel), http://dbpedia.org/resource/Death_Comes_to_Pemberley, http://dbpedia.org/resource/Dover_Beach_(novel), http://dbpedia.org/resource/Earth_Abides, http://dbpedia.org/resource/East_(novel), http://dbpedia.org/resource/Elvissey, http://dbpedia.org/resource/Emergence_(novel), http://dbpedia.org/resource/Empire_of_Bones, http://dbpedia.org/resource/Everything_That_Rises_Must_Converge, http://dbpedia.org/resource/Evolution's_Darling, http://dbpedia.org/resource/Finity's_End, http://dbpedia.org/resource/Flaws_in_the_Glass, http://dbpedia.org/resource/Fool_the_World:_The_Oral_History_of_a_Band_Called_Pixies, http://dbpedia.org/resource/For_the_Win, http://dbpedia.org/resource/Four_Ways_to_Forgiveness, http://dbpedia.org/resource/From_the_Notebooks_of_Dr._Brain, http://dbpedia.org/resource/Fuck:_Word_Taboo_and_Protecting_Our_First_Amendment_Liberties, http://dbpedia.org/resource/Glasshouse_(novel), http://dbpedia.org/resource/Gradisil, http://dbpedia.org/resource/Ha'penny_(novel), http://dbpedia.org/resource/Half_a_Crown_(novel), http://dbpedia.org/resource/Happy_Valley_(novel), http://dbpedia.org/resource/Harbingers_(novel), http://dbpedia.org/resource/Hidden_Empire, http://dbpedia.org/resource/His_Illegal_Self, http://dbpedia.org/resource/Hitler's_Pope, http://dbpedia.org/resource/Homeland_(Cory_Doctorow_novel), http://dbpedia.org/resource/How_Are_We_to_Live%3F, http://dbpedia.org/resource/How_Ethical_is_Australia%3F, http://dbpedia.org/resource/Hrolf_Kraki's_Saga, http://dbpedia.org/resource/Icarus_at_the_Edge_of_Time, http://dbpedia.org/resource/Imperial_Earth, http://dbpedia.org/resource/In_Defence_of_Animals, http://dbpedia.org/resource/In_Search_of_the_Castaways, http://dbpedia.org/resource/In_the_Line_of_Fire:_A_Memoir, http://dbpedia.org/resource/In_the_Mouth_of_the_Whale, http://dbpedia.org/resource/Inventing_the_AIDS_Virus, http://dbpedia.org/resource/Irreducible_Mind, http://dbpedia.org/resource/It_Can't_Happen_Here, http://dbpedia.org/resource/Jack_Maggs, http://dbpedia.org/resource/Judge_(novel), http://dbpedia.org/resource/Kaspar_(play), http://dbpedia.org/resource/Kill_Decision, http://dbpedia.org/resource/Kings_of_the_High_Frontier, http://dbpedia.org/resource/Knight_Moves_(novel), http://dbpedia.org/resource/L'Œuvre, http://dbpedia.org/resource/Learning_Perl, http://dbpedia.org/resource/Leonard_Maltin's_Movie_Guide, http://dbpedia.org/resource/Life_After_Life_(book), http://dbpedia.org/resource/Life_Before_Life, http://dbpedia.org/resource/Little,_Big, http://dbpedia.org/resource/Little_Brother_(Doctorow_novel), http://dbpedia.org/resource/Lost_Everything, http://dbpedia.org/resource/Mahars_of_Pellucidar, http://dbpedia.org/resource/Makers_(novel), http://dbpedia.org/resource/Memoirs_of_Many_in_One, http://dbpedia.org/resource/Memory_of_Water, http://dbpedia.org/resource/Men_at_Arms, http://dbpedia.org/resource/Mental_Radio, http://dbpedia.org/resource/Mercier_and_Camier, http://dbpedia.org/resource/Midnight_Robber, http://dbpedia.org/resource/Millennium_(novel), http://dbpedia.org/resource/Mindplayers, http://dbpedia.org/resource/Mindstar_Rising, http://dbpedia.org/resource/Moondyne, http://dbpedia.org/resource/More_Than_Two, http://dbpedia.org/resource/Mr._Darcy's_Daughters, http://dbpedia.org/resource/Mr._Popper's_Penguins, http://dbpedia.org/resource/Muse_(novel), http://dbpedia.org/resource/My_Life_as_a_Fake, http://dbpedia.org/resource/Naked_Empire, http://dbpedia.org/resource/Nonzero:_The_Logic_of_Human_Destiny, http://dbpedia.org/resource/Old_Souls, http://dbpedia.org/resource/One_World:_The_Ethics_of_Globalisation, http://dbpedia.org/resource/Only_Forward, http://dbpedia.org/resource/Orion_Shall_Rise, http://dbpedia.org/resource/Palestine_(2011_book), http://dbpedia.org/resource/Palestine_(comics), http://dbpedia.org/resource/Parapsychology:_Frontier_Science_of_the_Mind, http://dbpedia.org/resource/Parrot_and_Olivier_in_America, http://dbpedia.org/resource/Penpal_(novel), http://dbpedia.org/resource/Phantom_(Kay_novel), http://dbpedia.org/resource/Phantom_(Nesbø_novel)]... [INFO] P: 2.6867275658248254E-5, R: 1.0, F: 5.373310765428118E-5 -[INFO] Cumulative metrics, sample: 147, P: 0.320, R: 0.411, macro F1: 0.327, macro F1*: 0.360 -[INFO] QALD Cumulative metrics, sample: 147, P: 0.537, R: 0.411, macro F1: 0.327, QALD macro F1: 0.466 -[INFO] Current time: 854 ms, average: 3362.46 ms/ques, entity detection: 740.36 ms/ques (linking tool: 652.54 ms/ques), relation detection: 1071.21 ms/ques (similarity: 620.39 ms/ques), EDG generation: 45.67 ms/ques +[INFO] Cumulative metrics, sample: 147, P: 0.320, R: 0.411, macro F1: 0.360 +[INFO] QALD Cumulative metrics, sample: 147, P: 0.537, R: 0.411, average F1: 0.327, QALD macro F1: 0.466 +[INFO] Current question consumed time: 771 ms, average: 3026.15 ms/ques, entity detection: 651.55 ms/ques (linking tool: 577.73 ms/ques), relation detection: 994.15 ms/ques (similarity: 619.99 ms/ques), EDG generation: 46.91 ms/ques [INFO] Entity Linking P: 0.29931972789115646, R: 0.29931972789115646, micro F1: 0.29931972789115646, macro F1: 0.29931972789115646 [INFO] Relation Linking P: 0.30612244897959184, R: 0.30612244897959184, micro F1: 0.30612244897959184, macro F1: 0.30612244897959184 [INFO] Type Linking P: 0.8707482993197279, R: 0.8707482993197279, micro F1: 0.8707482993197279, macro F1: 0.8707482993197279 @@ -2217,9 +2217,9 @@ Question 147: What is the largest state in the United States? [INFO] Golden Answer: [http://dbpedia.org/resource/Alaska] [INFO] Predicted Answer: [http://dbpedia.org/resource/Odyssey_–_The_Essential_School, http://dbpedia.org/resource/Wesley_College_(Mississippi), http://dbpedia.org/resource/Academy_of_Citizenship_and_Empowerment, http://dbpedia.org/resource/Global_Connections_High_School, http://dbpedia.org/resource/Cristo_Rey_New_York_High_School, http://dbpedia.org/resource/Pella_Dutch_dialect, http://dbpedia.org/resource/Patriot_Party_(1960s–1980s)] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 148, P: 0.317, R: 0.408, macro F1: 0.324, macro F1*: 0.357 -[INFO] QALD Cumulative metrics, sample: 148, P: 0.534, R: 0.408, macro F1: 0.324, QALD macro F1: 0.463 -[INFO] Current time: 52086 ms, average: 3691.67 ms/ques, entity detection: 861.88 ms/ques (linking tool: 773.86 ms/ques), relation detection: 1151.89 ms/ques (similarity: 632.32 ms/ques), EDG generation: 45.45 ms/ques +[INFO] Cumulative metrics, sample: 148, P: 0.317, R: 0.408, macro F1: 0.357 +[INFO] QALD Cumulative metrics, sample: 148, P: 0.534, R: 0.408, average F1: 0.324, QALD macro F1: 0.463 +[INFO] Current question consumed time: 45051 ms, average: 3310.10 ms/ques, entity detection: 754.96 ms/ques (linking tool: 681.04 ms/ques), relation detection: 1066.07 ms/ques (similarity: 631.81 ms/ques), EDG generation: 46.69 ms/ques [INFO] Entity Linking P: 0.2972972972972973, R: 0.2972972972972973, micro F1: 0.2972972972972973, macro F1: 0.2972972972972973 [INFO] Relation Linking P: 0.30405405405405406, R: 0.30405405405405406, micro F1: 0.30405405405405406, macro F1: 0.30405405405405406 [INFO] Type Linking P: 0.8716216216216216, R: 0.8716216216216216, micro F1: 0.8716216216216216, macro F1: 0.8716216216216216 @@ -2232,9 +2232,9 @@ Question 148: Give me the websites of companies with more than 500000 employees. [INFO] Golden Answer: [https://www.usps.com/, http://www.baytexenergy.com/, http://www.petrochina.com.cn/ptr/, http://www.koelnmesse.com, http://www.umw.com.my, http://www.molmed.com/, http://en.mvd.ru, http://www.sgcc.com.cn/ywlm/default.shtml, http://www.moe.gov.my/, http://volkswagenag.com, http://www.tcs.com/, http://www.umw.com.my/, http://www.oxy.com, http://www.peopleready.com, http://www.issworld.com/, http://www.foxconn.com, http://www.defense.gov, http://www.yum.com/, http://www.carverbank.com, http://www.chinapost.com.cn, http://www.molmed.com/financial-reports?language=en, http://www.marykay.com/, http://www.cpgroupglobal.com, http://www.errt.org/, http://www.mod.gov.pk/, http://www.baytexenergy.com, http://lindenlab.com/, http://www.tesco.com, http://www.tcs.com, http://www.permasteelisagroup.com/, http://rostec.ru/en/, http://en.mvd.ru/, http://www.bundespost.de/, http://www.cnpc.com.cn/en/, http://www.bahriatown.com/, http://www.tata.com/, http://www.indianrailways.gov.in, http://www.avic.com/en, http://www.nuffnang.com, http://www.telent.com/, http://www.moiegypt.gov.eg/english, http://www.indianrailways.gov.in/, http://g4s.com] [INFO] Predicted Answer: [] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 149, P: 0.315, R: 0.406, macro F1: 0.322, macro F1*: 0.355 -[INFO] QALD Cumulative metrics, sample: 149, P: 0.537, R: 0.406, macro F1: 0.322, QALD macro F1: 0.462 -[INFO] Current time: 641 ms, average: 3671.19 ms/ques, entity detection: 856.09 ms/ques (linking tool: 768.67 ms/ques), relation detection: 1145.36 ms/ques (similarity: 628.94 ms/ques), EDG generation: 45.79 ms/ques +[INFO] Cumulative metrics, sample: 149, P: 0.315, R: 0.406, macro F1: 0.355 +[INFO] QALD Cumulative metrics, sample: 149, P: 0.537, R: 0.406, average F1: 0.322, QALD macro F1: 0.462 +[INFO] Current question consumed time: 552 ms, average: 3291.59 ms/ques, entity detection: 749.90 ms/ques (linking tool: 676.48 ms/ques), relation detection: 1060.01 ms/ques (similarity: 628.40 ms/ques), EDG generation: 47.08 ms/ques [INFO] Entity Linking P: 0.30201342281879195, R: 0.30201342281879195, micro F1: 0.30201342281879195, macro F1: 0.30201342281879195 [INFO] Relation Linking P: 0.3087248322147651, R: 0.3087248322147651, micro F1: 0.3087248322147651, macro F1: 0.3087248322147651 [INFO] Type Linking P: 0.87248322147651, R: 0.87248322147651, micro F1: 0.87248322147651, macro F1: 0.87248322147651 @@ -2247,12 +2247,12 @@ Question 149: What were the names of the three ships by Columbus? [INFO] Golden Answer: [http://dbpedia.org/resource/Niña, http://dbpedia.org/resource/Pinta_(ship), http://dbpedia.org/resource/Santa_María_(ship)] [INFO] Predicted Answer: [] [INFO] P: 0.0, R: 0.0, F: 0.0 -[INFO] Cumulative metrics, sample: 150, P: 0.313, R: 0.403, macro F1: 0.320, macro F1*: 0.352 -[INFO] QALD Cumulative metrics, sample: 150, P: 0.540, R: 0.403, macro F1: 0.320, QALD macro F1: 0.461 -[INFO] Current time: 997 ms, average: 3653.37 ms/ques, entity detection: 852.55 ms/ques (linking tool: 765.14 ms/ques), relation detection: 1138.67 ms/ques (similarity: 625.33 ms/ques), EDG generation: 46.11 ms/ques +[INFO] Cumulative metrics, sample: 150, P: 0.313, R: 0.403, macro F1: 0.352 +[INFO] QALD Cumulative metrics, sample: 150, P: 0.540, R: 0.403, average F1: 0.320, QALD macro F1: 0.461 +[INFO] Current question consumed time: 861 ms, average: 3275.39 ms/ques, entity detection: 746.73 ms/ques (linking tool: 673.35 ms/ques), relation detection: 1053.81 ms/ques (similarity: 624.79 ms/ques), EDG generation: 47.39 ms/ques [INFO] Entity Linking P: 0.30666666666666664, R: 0.30666666666666664, micro F1: 0.30666666666666664, macro F1: 0.30666666666666664 [INFO] Relation Linking P: 0.31333333333333335, R: 0.31333333333333335, micro F1: 0.31333333333333335, macro F1: 0.31333333333333335 [INFO] Type Linking P: 0.8733333333333333, R: 0.8733333333333333, micro F1: 0.8733333333333333, macro F1: 0.8733333333333333 =============================================== -[INFO] QuestionSolver total time: 548005 ms +[INFO] QuestionSolver total time: 491309 ms [INFO] query generation finished diff --git a/src/main/java/cn/edu/nju/ws/edgqa/domain/edg/EDG.java b/src/main/java/cn/edu/nju/ws/edgqa/domain/edg/EDG.java index 3379ecc..938cf69 100644 --- a/src/main/java/cn/edu/nju/ws/edgqa/domain/edg/EDG.java +++ b/src/main/java/cn/edu/nju/ws/edgqa/domain/edg/EDG.java @@ -525,7 +525,7 @@ private void construct(String question) { // generate syntax treeNode of taggedQuestion this.syntaxTreeText = NLPUtil.getSyntaxTree(taggedQuestion); this.syntaxTreeNode = createTree(this.syntaxTreeText); - System.out.println(NLPUtil.TransferParentheses(this.syntaxTreeText)); + System.out.println(NLPUtil.transferParentheses(this.syntaxTreeText)); // parse the syntaxTree, generate nodes and edges Sent(this.syntaxTreeNode); @@ -4023,7 +4023,7 @@ private void Sent(TreeNode root) { secondNode = matcher1.group(2); } else { firstNode = tempQuestion; - String syntaxTree = NLPUtil.TransferParentheses(NLPUtil.getSyntaxTree(tempQuestion)); + String syntaxTree = NLPUtil.transferParentheses(NLPUtil.getSyntaxTree(tempQuestion)); System.out.println(syntaxTree); TreeNode rootNode = createTree(syntaxTree); LinkedList toSearch = new LinkedList<>(); diff --git a/src/main/java/cn/edu/nju/ws/edgqa/main/EDGQA.java b/src/main/java/cn/edu/nju/ws/edgqa/main/EDGQA.java index 485add5..a951262 100644 --- a/src/main/java/cn/edu/nju/ws/edgqa/main/EDGQA.java +++ b/src/main/java/cn/edu/nju/ws/edgqa/main/EDGQA.java @@ -12,6 +12,7 @@ import cn.edu.nju.ws.edgqa.handler.QuestionSolver; import cn.edu.nju.ws.edgqa.utils.FileUtil; import cn.edu.nju.ws.edgqa.utils.LogUtil; + import cn.edu.nju.ws.edgqa.utils.Timer; import cn.edu.nju.ws.edgqa.utils.enumerates.DatasetEnum; import cn.edu.nju.ws.edgqa.utils.enumerates.GoldenMode; diff --git a/src/main/java/cn/edu/nju/ws/edgqa/utils/NLPUtil.java b/src/main/java/cn/edu/nju/ws/edgqa/utils/NLPUtil.java index 52790f4..8ac3f46 100755 --- a/src/main/java/cn/edu/nju/ws/edgqa/utils/NLPUtil.java +++ b/src/main/java/cn/edu/nju/ws/edgqa/utils/NLPUtil.java @@ -1,86 +1,82 @@ package cn.edu.nju.ws.edgqa.utils; import cn.edu.nju.ws.edgqa.domain.beans.TreeNode; -import cn.edu.nju.ws.edgqa.domain.edg.EDG; import cn.edu.nju.ws.edgqa.utils.linking.LinkingTool; import edu.stanford.nlp.ling.CoreAnnotations; import edu.stanford.nlp.ling.CoreLabel; import edu.stanford.nlp.pipeline.*; -import edu.stanford.nlp.time.TimeAnnotations; -import edu.stanford.nlp.time.TimeAnnotator; import edu.stanford.nlp.trees.TreeCoreAnnotations; import edu.stanford.nlp.util.CoreMap; -import java.io.File; -import java.io.FileNotFoundException; -import java.io.FileWriter; import java.io.IOException; -import java.text.SimpleDateFormat; import java.util.*; public class NLPUtil { private static final HashSet entityTypes = new HashSet<>(Arrays.asList("PERSON", "LOCATION", "ORGANIZATION", "MISC")); - /** - * Get all the n_gram in the question, desc order by length - * - * @param question the natural language question - * @return - */ - public static Vector getNgram(String question) { - - //deal word by word - String[] tokens = question.toLowerCase().replaceAll("([\u4e00-\u9fa5])", " $1 ") - .replaceAll("[ ]{2,}", " ").split(" "); // remove the redundant space and split it by space - - Vector ngram = new Vector(); - for (int i = 0; i < tokens.length; i++) { - for (int j = 0; j <= i; j++) { - if (i - j <= 8) { // the maximum token length is set to 8 here - StringBuilder sb = new StringBuilder(); - for (int k = j; k <= i; k++) { - if (k == i) - sb.append(tokens[k]); - else - sb.append(tokens[k]); - } - if (!ngram.contains(sb.toString())) - ngram.add(sb.toString()); - } - } - } - //sort result by the length of element - ngram.sort((left, right) -> (right.length() - left.length())); - return ngram; + + private static final StanfordCoreNLP parsePipeline; + private static final StanfordCoreNLP lemmatizePipeline; + private static final StanfordCoreNLP nerPipeline; + private static final StanfordCoreNLP tokenizePipeline; + private static final StanfordCoreNLP posPipeline; + + static { + + // properties for constituency parsing + Properties parseProps = new Properties(); + parseProps.setProperty("annotators", "tokenize, ssplit, pos, parse"); + //props.setProperty("parse.model", "edu/stanford/nlp/models/srparser/englishSR.ser.gz"); + parsePipeline = new StanfordCoreNLP(parseProps); + + // properties for lemmatization + Properties lemmaProps = new Properties(); + lemmaProps.setProperty("annotators", "tokenize,ssplit,pos,lemma"); + lemmatizePipeline = new StanfordCoreNLP(lemmaProps); + + // properties for NER + Properties nerProps = new Properties(); + nerProps.setProperty("annotators", "tokenize,ssplit,pos,lemma,ner"); + nerProps.setProperty("ner.applyFineGrained", "false"); + nerPipeline = new StanfordCoreNLP(nerProps); + + // properties for tokenization + Properties tokenizeProps = new Properties(); + tokenizeProps.setProperty("annotators", "tokenize"); + tokenizePipeline = new StanfordCoreNLP(tokenizeProps); + + // properties for POS tagging + Properties props = new Properties(); + props.setProperty("annotators", "tokenize,ssplit,pos"); + posPipeline = new StanfordCoreNLP(props); + } + + /** - * return the lemmazation of a word + * return the lemmatization of a word * * @param word raw word - * @return lemmazatio of a word + * @return lemmatization of a word */ public static String getLemma(String word) { - // do not lemmatize for 'born' - if (word.toLowerCase().equals("born")) { + if (word.equalsIgnoreCase("born")) { return word; } - Properties props = new Properties(); - props.setProperty("annotators", "tokenize,ssplit,pos,lemma"); - StanfordCoreNLP pipeline = new StanfordCoreNLP(props); Annotation doc = new Annotation(word); - pipeline.annotate(doc); + lemmatizePipeline.annotate(doc); return doc.get(CoreAnnotations.TokensAnnotation.class).get(0).lemma(); } /** - * return the lemmazation of a sentence + * return the lemmatization of a sentence * * @param sent raw sentence - * @return lemmazation of the sentence + * @return lemmatization of the sentence */ public static String getLemmaSent(String sent) { ArrayList tokens = getTokens(sent); @@ -91,67 +87,30 @@ public static String getLemmaSent(String sent) { return sb.toString().trim(); } - public static String tagNumNTime(String sentence) { - - StringBuilder sb = new StringBuilder(sentence); - HashSet timeNNumber = new HashSet<>(Arrays.asList("MONEY", "NUMBER", "ORDINAL", "PERCENT", "DATE", "TIME", "DURATION", "SET")); - //ArrayList sent = new ArrayList(Arrays.asList(sentence.toCharArray())); - Properties props = new Properties(); - props.setProperty("annotators", "tokenize,ssplit,pos,lemma,ner"); - StanfordCoreNLP pipeline = new StanfordCoreNLP(props); - CoreDocument document = new CoreDocument(sentence); - pipeline.annotate(document); - List indexes = new ArrayList<>(); - for (CoreEntityMention mention : document.entityMentions()) { - if (timeNNumber.contains(mention.entityType().trim())) { - int start = mention.charOffsets().first; - int end = mention.charOffsets().second; - indexes.add(start); - indexes.add(end); - /*sb.insert(start,'<'); - //sb.insert(end,(" "+mention.entityType()).toCharArray()); - sb.insert(end,'>');*/ - } - } - for (int i = 0; i < indexes.size(); i++) { - if ((i & 1) == 0) { // i is even - sb.insert(indexes.get(i) + i, '<'); - } else { - sb.insert(indexes.get(i) + i, '>'); - } - } - - //System.out.println(document.entityMentions()); - //System.out.println(document.quotes()); - - //return document.entityMentions(); - - return sb.toString(); - } - + /** + * return all the entity mentions spotted by coreNLP ner module in a sentence + * @param sentence natural language sentence + * @return all the entity mentions spotted + */ public static List coreNLPEntityMentions(String sentence) { List res = new LinkedList<>(); - Properties props = new Properties(); - props.setProperty("annotators", "tokenize,ssplit,pos,lemma,ner"); - props.setProperty("ner.applyFineGrained", "false"); - StanfordCoreNLP pipeline = new StanfordCoreNLP(props); CoreDocument document = new CoreDocument(sentence); - pipeline.annotate(document); - List indexes = new ArrayList<>(); + nerPipeline.annotate(document); for (CoreEntityMention mention : document.entityMentions()) { if (entityTypes.contains(mention.entityType())) { res.add(mention.text()); } - /*System.out.println("Entity: " + mention.entity()); - System.out.println("Mention Type: " + mention.entityType()); - System.out.println("Text: " + mention.text()); - System.out.println("Tokens: " + mention.tokens());*/ } return res; } + /** + * judge whether a phase is an Entity by dexter and coreNLP + * @param phase a natural language phase + * @return if it is an entity, return true, else return false + */ public static boolean judgeIfEntity(String phase) { System.out.println("[DEBUG] judgeIfEntity:" + phase); @@ -170,189 +129,29 @@ public static boolean judgeIfEntity(String phase) { return true; } - // need set proxy in EntityLinking script，not recommend in large scale(band ip warning) - /*if(EntityLinking.isDBpediaEntity(phase,e)){ - return true; - }*/ - for (String mention : coreNLPEntityMentions(phase)) { if ((double) mention.length() / phase.length() >= e) { return true; } } - //judge if it is instance after concatenate - /*String[] tokens = phase.split(" "); - StringBuilder sb = new StringBuilder("http://dbpedia.org/resource/"); - for (String token : tokens) { - sb.append(token).append("_"); - } - sb.deleteCharAt(sb.length() - 1); - System.out.println(sb); - - if (KBUtil.isAnInstance(sb.toString())) { - return true; - }*/ - - return false; - } - - public static boolean judgeIfCompare(String phase) { - HashSet comparePOS = new HashSet<>(Arrays.asList("JJR", "JJS", "RBR", "RBS")); - List poses = getPOS(phase); - for (String pos : poses) { - if (comparePOS.contains(pos)) { - return true; - } - } - return false; - } - - public static boolean judgeIfTime(String phase) { - Properties props = new Properties(); - AnnotationPipeline pipeline = new AnnotationPipeline(); - pipeline.addAnnotator(new TokenizerAnnotator(false)); - pipeline.addAnnotator(new WordsToSentencesAnnotator(false)); - pipeline.addAnnotator(new POSTaggerAnnotator(false)); - pipeline.addAnnotator(new TimeAnnotator("sutime", props)); - - Annotation annotation = new Annotation(phase); - SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd"); - String today = sdf.format(new Date()); - annotation.set(CoreAnnotations.DocDateAnnotation.class, today); - pipeline.annotate(annotation); - //System.out.println(annotation.get(CoreAnnotations.TextAnnotation.class)); - List timexAnnsAll = annotation.get(TimeAnnotations.TimexAnnotations.class); - List all_tokens = annotation.get(CoreAnnotations.TokensAnnotation.class); - int length = all_tokens.size(); - - if (timexAnnsAll.size() != 1) { - return false; - } else { - CoreMap timexAnn = timexAnnsAll.get(0); - List tokens = timexAnn.get(CoreAnnotations.TokensAnnotation.class); - int start = tokens.get(0).index(); - int end = tokens.get(tokens.size() - 1).index(); - if (start == 1 && end == length) { - return true; - } - } return false; } /** - * input a sentence, return entity in it(use Stanford CoreNLP NER) - * - * @param sentence - * @return entity in sentence + * get a list of the tokens in a sentence by coreNLP tokenizer + * @param sentence natural language sentence + * @return a list of the tokens in the sentence */ - public static List getEntities(String sentence) { - //HashSet notEntity = new HashSet<>(Arrays.asList("MISC", "MONEY", "NUMBER", "ORDINAL", "PERCENT", "DATE", "TIME", "DURATION", "SET")); - HashSet entitySet = new HashSet<>(Arrays.asList("PERSON", "LOCATION", "ORGANIZATION")); - List entities = new LinkedList<>(); - Properties properties = new Properties(); - properties.setProperty("annotators", "tokenize,ssplit,pos,lemma,ner"); - properties.setProperty("ner.applyFineGrained", "false"); - StanfordCoreNLP pipeline = new StanfordCoreNLP(properties); - CoreDocument document = new CoreDocument(sentence); - pipeline.annotate(document); - - if (document.entityMentions() != null) { - //System.out.println(document.entityMentions()); - document.entityMentions().forEach(o -> { - if (entitySet.contains(o.entityType())) { - entities.add(o.text()); - } - }); - } - - return entities; - } - - public static List getTimeEntities(String sentence) { - Properties properties = new Properties(); - properties.setProperty("annotators", "tokenize,ssplit,pos,lemma,ner"); - StanfordCoreNLP pipeline = new StanfordCoreNLP(properties); - Annotation document = new Annotation(sentence); - pipeline.annotate(document); - HashSet timeEntity = new HashSet<>(Arrays.asList("DATE", "TIME", "DURATION", "SET")); - List sents = document.get(CoreAnnotations.SentencesAnnotation.class); - List entities = new LinkedList<>(); - for (CoreMap sent : sents) { - String currentType = null; - String entity = null; - for (CoreLabel token : sent.get(CoreAnnotations.TokensAnnotation.class)) { - String word = token.get(CoreAnnotations.TextAnnotation.class); - String ner = token.get(CoreAnnotations.NamedEntityTagAnnotation.class); - if (!ner.equals("O") && (timeEntity.contains(ner))) { - if (currentType == null) { - currentType = ner; - entity = word; - } else if (currentType.equals(ner)) { - entity += " " + word; - } else { - if (entity != null) { - entities.add(entity.trim()); - } - currentType = ner; - entity = word; - } - } else { - if (entity != null) { - entities.add(entity.trim()); - entity = null; - currentType = null; - } - } - } - if (entity != null) { - entities.add(entity.trim()); - } - } - return entities; - } - - public static List generateSimpleEDGFromTXT(String inputFilePath) { - - File inputFile = new File(inputFilePath); - List EDGS = new LinkedList<>(); - try { - Scanner scanner = new Scanner(inputFile); - String line = null; - while (scanner.hasNextLine()) { - - line = scanner.nextLine(); - System.out.println("Processing: " + line); - try { - EDGS.add(new EDG(line)); - } catch (RuntimeException e) { - System.out.println("Processing Error:" + line + "\n"); - } - } - - scanner.close(); - - } catch (FileNotFoundException e) { - - e.printStackTrace(); - } - return EDGS; - - } - public static ArrayList getTokens(String sentence) { - //null str check + //null string check if (sentence == null || sentence.trim().equals("")) { return new ArrayList<>(); } - Properties props = new Properties(); - props.setProperty("annotators", "tokenize"); - StanfordCoreNLP pipeline = new StanfordCoreNLP(props); Annotation annotation = new Annotation(sentence); - pipeline.annotate(annotation); - + tokenizePipeline.annotate(annotation); ArrayList result = new ArrayList<>(); List coreLabels = annotation.get(CoreAnnotations.TokensAnnotation.class); @@ -364,66 +163,15 @@ public static ArrayList getTokens(String sentence) { return result; } - //return token number of a sentence - public static int getTokenNum(String sentence) { - return getTokens(sentence).size(); - } - - public static void addPosTag(String inputFilePath, String outputFilePath) { - File inputFile = new File(inputFilePath); - File outputFile = new File(outputFilePath); - try { - if (!inputFile.exists()) { - System.out.println("Input File '" + inputFilePath + "' does not exist! Please check it."); - } else { - - Properties props = new Properties(); - props.setProperty("annotators", "tokenize"); - StanfordCoreNLP pipeline = new StanfordCoreNLP(props); - Scanner scanner = new Scanner(inputFile); - if (!outputFile.exists()) { - outputFile.createNewFile(); - } - FileWriter fileWriter = new FileWriter(outputFile); - - String line = null; - while (scanner.hasNextLine()) { - line = scanner.nextLine(); - //create a parser, input text - Annotation document = new Annotation(line); - //parse - pipeline.annotate(document); - - List tokens = document.get(CoreAnnotations.TokensAnnotation.class); - int count = 0; - for (CoreLabel token : tokens) { - - String word = token.toString(); - fileWriter.write(word + "[" + count + "] "); - System.out.print(word + "[" + count + "] "); - count++; - } - System.out.println(); - fileWriter.write("\n"); - } - scanner.close(); - - - } - } catch (IOException e) { - e.printStackTrace(); - } - - } - + /** + * get the constituency parsing of a sentence by coreNLP + * @param sentence natural language sentence + * @return constituency parsing result + */ public static String getSyntaxTree(String sentence) { - Properties props = new Properties(); - props.setProperty("annotators", "tokenize, ssplit, pos, parse"); - //props.setProperty("parse.model", "edu/stanford/nlp/models/srparser/englishSR.ser.gz"); - StanfordCoreNLP pipeline = new StanfordCoreNLP(props); Annotation document = new Annotation(sentence); - pipeline.annotate(document); + parsePipeline.annotate(document); List sentences = document.get(CoreAnnotations.SentencesAnnotation.class); StringBuilder sb = new StringBuilder(); for (CoreMap sent : sentences) { @@ -432,34 +180,15 @@ public static String getSyntaxTree(String sentence) { return sb.toString(); } - public static String getPosNum(String sentence) { - Properties props = new Properties(); - props.setProperty("annotators", "tokenize"); - StanfordCoreNLP pipeline = new StanfordCoreNLP(props); - Annotation document = new Annotation(sentence); - pipeline.annotate(document); - List tokens = document.get(CoreAnnotations.TokensAnnotation.class); - StringBuilder sb = new StringBuilder(); - - int count = 0; - for (CoreLabel token : tokens) { - - String word = token.toString(); - //fileWriter.write(word + "[" + count + "] "); - //System.out.print(word + "[" + count + "] "); - sb.append(word).append("[").append(count).append("] "); - count++; - } - return sb.toString(); - - } - + /** + * get the POS tags of the tokens in a sentence + * @param sentence natural language sentence + * @return the POS tags of the tokens + */ public static List getPOS(String sentence) { - Properties props = new Properties(); - props.setProperty("annotators", "tokenize,ssplit,pos"); - StanfordCoreNLP pipeline = new StanfordCoreNLP(props); + Annotation annotation = new Annotation(sentence); - pipeline.annotate(annotation); + posPipeline.annotate(annotation); LinkedList poses = new LinkedList<>(); for (CoreLabel token : annotation.get(CoreAnnotations.TokensAnnotation.class)) { //poses.add(token.get(CoreAnnotations.PartOfSpeechAnnotation.class)) @@ -469,6 +198,12 @@ public static List getPOS(String sentence) { } //Judge whether a phrase is a verb phrase, return true if yes, false if not + + /** + * judge whether a span of the sentence is a verb-phrase + * @param span a span of the sentence in the form of String + * @return if the sentence is a verb-phrase, return true; else return false + */ public static boolean judgeIfVP(String span) { HashSet relationTags = new HashSet<>( Arrays.asList("VB", "VBD", "VBG", "VBN", "VBP", "VBZ", "VP", "S", "SQ", "ADJP", "ADVP")); @@ -487,6 +222,11 @@ public static boolean judgeIfVP(String span) { } + /** + * remove the redundant head and tail in a sentence for question preprocess + * @param str sentence + * @return a sentence with redundant head and tail trimmed + */ public static String removeRedundantHeaderAndTailer(String str) { String originalStr = str; int len = str.length(); @@ -509,7 +249,7 @@ public static String removeRedundantHeaderAndTailer(String str) { return str; } - public static String TransferParentheses(String s) { + public static String transferParentheses(String s) { return s.replace("(", "[").replace(")", "]"); } @@ -517,7 +257,7 @@ public static void main(String[] args) throws IOException { //String question = "How many different teams have the players debuted in Houston Astros played for?"; //String syntaxTree = NLPUtil.getSyntaxTree(question); - //System.out.println(TransferParentheses(syntaxTree)); + //System.out.println(transferParentheses(syntaxTree)); }