demo branch

func/collocation/collocation.py

@@ -11,6 +11,7 @@ from db.db_config import get_db
 
 def run_collocation_on_text(page):
     datasetid = page.split('><p>')[0].replace('<div id=', '').replace('"', '').strip()
+    print('dataset id in run_collocation_on_text: ',datasetid)
     collocations = []
 
     nlp = spacy.load('zh_core_web_sm')
@@ -22,6 +23,7 @@ def run_collocation_on_text(page):
     data = []
 
     for row in res:
+        print('row in collocation from db: ',row)
 
         docid = row[0]
 

func/concordance/concordance.py

@@ -8,12 +8,13 @@ from nltk.metrics import BigramAssocMeasures
 
 
 
-def collocations():
+def collocations(datasetid):
     collocations = []
 
     nlp = spacy.load('zh_core_web_sm')
     conn, cursor = get_db()
-    cursor.execute('SELECT * from news;')
+    #cursor.execute('SELECT * from news;')
+    cursor.execute('SELECT * from files where dataset_id = "' + datasetid + '";')
     res = cursor.fetchall()
 
     data = []
@@ -65,11 +66,13 @@ def collocations():
 
 
 def run_concordance_on_text(page):
-    #print('page: ',page)
-    page = page+'部'
+    datasetid = page.replace("<p>Collocations for the word '部' (department) for ",'').replace('</p>','').strip()
+    print('datasetid inside run_concordance_on_text: ',datasetid)
+    #page = page+'部'
     nlp = spacy.load('zh_core_web_sm')
     conn, cursor = get_db()
-    cursor.execute('SELECT * from news;')
+    #cursor.execute('SELECT * from news;')
+    cursor.execute('SELECT * from files where dataset_id = "' + datasetid + '";')
     res = cursor.fetchall()
     data = []
     for row in res:
@@ -78,7 +81,7 @@ def run_concordance_on_text(page):
         data.append([docid, content])
 
     concordances = []
-    terms = collocations()
+    terms = collocations(datasetid)
 
     #terms = [page]
     for i in range(0, len(data)):

func/neroverall/neroverall.py

@@ -9,7 +9,7 @@ def run_neroverall_on_text(page):
     print('neroverall page old: ',page)
     datasetid = page.split('><p>')[0].replace('<div id=','').replace('"','').strip()
     #output_json = html_to_json.convert_tables(page)
-    print('neroverall page: ',page)
+    print('neroverall datasetid: ',datasetid)
 
     ner_driver = CkipNerChunker(model="bert-base")
     conn, cursor = get_db()

func/usas/usas.py

@@ -9,6 +9,7 @@ from db.db_config import get_db
 def run_usas_on_text(page):
     print('usasoverall page old: ', page)
     datasetid = page.split('><p>')[0].replace('<div id=', '').replace('"', '').strip()
+    print('usas overall datasetid: ', datasetid)
     d = {}
     with open('func/usas/usas_overall.txt') as f:
         for line in f: