From: Shane Jaroch Date: Tue, 4 Aug 2020 15:55:42 +0000 (-0400) Subject: remove res folder X-Git-Tag: 0.0.1~9 X-Git-Url: https://git.nutra.tk/v2?a=commitdiff_plain;h=f2e8796d442ca7be962e8c9dd92916f5436eb1c8;p=nutratech%2Fusda-sqlite.git remove res folder --- diff --git a/data/dep-graph.yml b/data/dep-graph.yml new file mode 100644 index 0000000..71e12aa --- /dev/null +++ b/data/dep-graph.yml @@ -0,0 +1,12 @@ +- NUTR_DEF +- FD_GROUP + - FOOD_DES + - SRC_CD + - DERIV_CD + - NUT_DATA + - LANGDESC + - LANGUAL + - DATA_SRC + - DATASRCLN + - WEIGHT + - FOOTNOTE diff --git a/data/format-sql.sh b/data/format-sql.sh new file mode 100755 index 0000000..2c499b6 --- /dev/null +++ b/data/format-sql.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +cd "$(dirname "$0")" + +pg_format -s 2 tables.sql -o tables.sql +# pg_format -s 2 functions.sql -o functions.sql diff --git a/data/import.sql b/data/import.sql index 4db1f79..20a0581 100644 --- a/data/import.sql +++ b/data/import.sql @@ -1,6 +1,8 @@ .mode csv .import '| tail -n +2 nt/nutr_def.csv' nutr_def +.import '| tail -n +2 nt/fdgrp.csv' fdgrp +.import '| tail -n +2 nt/food_des.csv' food_des .header on .mode column diff --git a/data/process.py b/data/process.py index ff26a23..215d630 100644 --- a/data/process.py +++ b/data/process.py @@ -1,5 +1,5 @@ -# nutra-db, a database for nutratracker clients -# Copyright (C) 2020 Nutra, LLC. [Shane & Kyle] +# nt-sqlite, an sqlite3 database for nutratracker clients +# Copyright (C) 2020 Shane Jaroch # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -16,10 +16,12 @@ import csv import os +import shutil import sys # change to script's dir os.chdir(os.path.dirname(os.path.abspath(__file__))) +shutil.rmtree("nt", True) os.makedirs("nt", 0o755, True) @@ -28,6 +30,13 @@ os.makedirs("nt", 0o755, True) # -------------------- output_files = { "SR-Leg_DB/FD_GROUP.csv": "nt/fdgrp.csv", + "SR-Leg_DB/SRC_CD.csv": "nt/src_cd.csv", + "SR-Leg_DB/DERIV_CD.csv": "nt/deriv_cd.csv", + "SR-Leg_DB/LANGDESC.csv": "nt/lang_desc.csv", + "SR-Leg_DB/LANGUAL.csv": "nt/langual.csv", + "SR-Leg_DB/DATA_SRC.csv": "nt/data_src.csv", + "SR-Leg_DB/DATSRCLN.csv": "nt/datsrcln.csv", + "SR-Leg_DB/FOOTNOTE.csv": "nt/footnote.csv", "SR-Leg_DB/WEIGHT.csv": None, } @@ -62,7 +71,7 @@ def main(args): # ----------------- print("==> Process CSV") process_nutr_def() - process_nut_data() + # process_nut_data() process_food_des() for fname in output_files: @@ -71,7 +80,7 @@ def main(args): with open(fname) as file: reader = csv.reader(file) rows = list(reader) - # + ######################### # Process and write out if fname == "SR-Leg_DB/WEIGHT.csv": process_weight(rows, fname) @@ -211,7 +220,7 @@ def process_nut_data(): for row in rows[1:]: result.append(row[:3]) - # + ######################### # Write out result with open("nt/nut_data.csv", "w+") as file: writer = csv.writer(file, lineterminator="\n") @@ -234,16 +243,10 @@ def process_food_des(): reader = csv.reader(file) rows = list(reader) # Add to final solution - for i, _row in enumerate(rows): + for i, row in enumerate(rows): if i > 0: - food_ids.add(int(_row[0])) - row = _row[:10] - del row[6] - row.insert(2, 1) # Data src - row.append(None) # user_id - row.append(True) # is_shared + food_ids.add(int(row[0])) result.append(row) - # result.append(row[:3]) # Special interests DB for dir in special_interests_dirs: @@ -261,17 +264,15 @@ def process_food_des(): print(f"new food: {food_id} {_row[2]}") food_ids.add(food_id) # Set new row - row = [None] * 12 + row = [None] * 14 row[0] = food_id row[1] = _row[1] # Food group - row[2] = 2 # Data src - row[3] = _row[2] # Long Desc + row[2] = _row[2] # Long Desc if len(_row) > 3: - row[5] = _row[3] # Sci name - row[11] = True # is_shared + row[4] = _row[3] # Comm/sci name result.append(row) - # + ######################### # Write out result with open("nt/food_des.csv", "w+") as file: writer = csv.writer(file, lineterminator="\n") @@ -288,8 +289,8 @@ def process_weight(rows, fname): servings_set = set() # CSV rows - serving_id = [["id", "msre_desc"]] - servings = [["food_id", "msre_id", "grams"]] + serv_desc = [["id", "msre_desc"]] + serving = [["food_id", "msre_id", "grams", "num_data_pts", "std_dev"]] # # Main logic @@ -303,10 +304,12 @@ def process_weight(rows, fname): msre_desc = row[3] grams = float(row[4]) grams /= amount + num_data_pts = row[5] + std_dev = row[6] # Get key if used previously if msre_desc not in msre_ids: - serving_id.append([id, msre_desc]) + serv_desc.append([id, msre_desc]) msre_ids[msre_desc] = id id += 1 msre_id = msre_ids[msre_desc] @@ -316,17 +319,17 @@ def process_weight(rows, fname): # DETAIL: Key (food_id, msre_id)=(1036, 3) already exists. prim_key = (food_id, msre_id) if prim_key not in servings_set: - servings.append([food_id, msre_id, grams]) + serving.append([food_id, msre_id, grams, num_data_pts, std_dev]) servings_set.add(prim_key) - # - # Write serving_id and servings tables - with open("nt/serving_id.csv", "w+") as file: + ################################################## + # Write serv_desc and serving tables + with open("nt/serv_desc.csv", "w+") as file: writer = csv.writer(file, lineterminator="\n") - writer.writerows(serving_id) - with open("nt/servings.csv", "w+") as file: + writer.writerows(serv_desc) + with open("nt/serving.csv", "w+") as file: writer = csv.writer(file, lineterminator="\n") - writer.writerows(servings) + writer.writerows(serving) # diff --git a/data/res/dep-graph.yml b/data/res/dep-graph.yml deleted file mode 100644 index 75af85d..0000000 --- a/data/res/dep-graph.yml +++ /dev/null @@ -1,11 +0,0 @@ -- NUTR_DEF - - FD_GROUP - - FOOD_DES - - SRC_CD && DERIV_CD - - NUT_DATA - - LANGDESC - - LANGUAL - - DATA_SRC - - DATASRCLN - - WEIGHT - - FOOTNOTE diff --git a/data/res/flav/tables.sql b/data/res/flav/tables.sql deleted file mode 100644 index 50226fe..0000000 --- a/data/res/flav/tables.sql +++ /dev/null @@ -1,64 +0,0 @@ -CREATE TABLE DATA_SRC ( - DataSrc_ID TEXT, - Authors TEXT, - Title TEXT, - Jorunal TEXT, - Year TEXT, - Volume TEXT, - Issue TEXT, - Start_Page TEXT, - Emd_Page TEXT, -) -CREATE TABLE DATSRCLN ( - NDB_No TEXT, - Nutr_no TEXT, - DataSrc_ID TEXT, -) -CREATE TABLE FD_GROUP ( - FdGrp_CD TEXT, - FdGrp_Desc TEXT, -) -CREATE TABLE FLAV_DAT ( - NDB_No TEXT, - Nutr_no TEXT, - Flav_Val DOUBLE, - SE DOUBLE, - n INT, - Min DOUBLE, - Max DOUBLE, - CC TEXT, -) -CREATE TABLE FLAV_IND ( - NDB_No TEXT, - DataSrc ID TEXT, - Food No TEXT, - FoodIndiv_Desc TEXT, - Method TEXT, - Cmpd_Name TEXT, - Rptd CmpdVal DOUBLE, - Rptd_Std Dev TEXT, - Num_Data_Pts DOUBLE, - LT TEXT, - Rptd_Units TEXT, - Fresh Dry_Wt TEXT, - Quant_Std TEXT, - Conv_Factor_G DOUBLE, - Conv_Factor_M TEXT, - Conv_Factor_SpGr DOUBLE, - Cmpd_Val DOUBLE, - Cmpt_StdDev TEXT, -) -CREATE TABLE FOOD_DES ( - NDB_No TEXT, - FdGrp_Cd TEXT, - Long_Desc TEXT, - SciName TEXT, -) -CREATE TABLE NUTR_DEF ( - Nutr_no TEXT, - Nutrient name TEXT, - Flav_Class TEXT, - Tagname TEXT, - Unit TEXT, -) - diff --git a/data/res/import.sql b/data/res/import.sql deleted file mode 100644 index bbaeacf..0000000 --- a/data/res/import.sql +++ /dev/null @@ -1,20 +0,0 @@ - -.read tables.sql - -.mode csv - -.import DATA_SRC.csv DATA_SRC -.import DERIV_CD.csv DERIV_CD -.import FOOD_DES.csv FOOD_DES -.import LANGDESC.csv LANGDESC -.import NUT_DATA.csv NUT_DATA -.import SRC_CD.csv SRC_CD -.import DATSRCLN.csv DATASRCLN -.import FD_GROUP.csv FD_GROUP -.import FOOTNOTE.csv FOOTNOTE -.import LANGUAL.csv LANGUAL -.import NUTR_DEF.csv NUTR_DEF -.import WEIGHT.csv WEIGHT - -.headers on -.mode column diff --git a/data/res/isoflav/tables.sql b/data/res/isoflav/tables.sql deleted file mode 100644 index 4bd8984..0000000 --- a/data/res/isoflav/tables.sql +++ /dev/null @@ -1,52 +0,0 @@ -CREATE TABLE DATA_SRC ( - Authors TEXT, - Title TEXT, - Year DOUBLE, - Journal TEXT, - Vol TEXT, - Start_Page TEXT, - End_Page TEXT, - DataSrc_ID TEXT, -) -CREATE TABLE DATSRCLN ( - NDB_No TEXT, - Nutr_No TEXT, - DataSrc_ID TEXT, -) -CREATE TABLE FOOD_DES ( - NDB_No TEXT, - FdGrp_Cd TEXT, - Long_Desc TEXT, -) -CREATE TABLE ISFL_DAT ( - NDB_No TEXT, - Nutr_No TEXT, - Isfl_Val DOUBLE, - SD DOUBLE, - n DOUBLE, - Min DOUBLE, - Max DOUBLE, - CC TEXT, - DataSrc_ID TEXT, -) -CREATE TABLE NUTR_DEF ( - Nutr_no TEXT, - NutrDesc TEXT, - Unit TEXT, -) -CREATE TABLE SYBN_DTL ( - NDB_No TEXT, - Nutr_No TEXT, - DataSrc_ID DOUBLE, - FoodNo TEXT, - Food_Detail_Desc TEXT, - Nutr_Val DOUBLE, - Std_Dev DOUBLE, - Num_Data_Pts DOUBLE, - Sam Hand_Rtg DOUBLE, - AnalMeth_Rtg DOUBLE, - SampPlan_Rtg DOUBLE, - AnalQC_Rtg DOUBLE, - NumSamp_QC DOUBLE, - CC TEXT, -) diff --git a/data/res/proanth/tables.sql b/data/res/proanth/tables.sql deleted file mode 100644 index fe62259..0000000 --- a/data/res/proanth/tables.sql +++ /dev/null @@ -1,45 +0,0 @@ -CREATE TABLE DATA_SRC ( - DataSrc_ID TEXT, - Authors TEXT, - Title TEXT, - Year DOUBLE, - Jounral TEXT, - Vol DOUBLE, - Issue DOUBLE, - Start_Page TEXT, - End_Page TEXT, -) -CREATE TABLE DATSRCLN ( - NDB_No TEXT, - Nutr_No TEXT, - DataSrc_ID TEXT, -) -CREATE TABLE FD_GROUP ( - FdGrp_CD TEXT, - FdGrp_Desc TEXT, -) -CREATE TABLE FOOD_DES ( - NDB No TEXT, - FdGrp_Cd TEXT, - Long_Desc TEXT, - SciName TEXT, -) -CREATE TABLE NUTR_DEF ( - Nutr_No TEXT, - Units TEXT, - Tagname TEXT, - NutrDesc TEXT, - Num_Dec TEXT, - SR_Order INT, -) -CREATE TABLE PA_DAT ( - NDB No TEXT, - Nutr_No TEXT, - Flav_Val DOUBLE, - SD DOUBLE, - N DOUBLE, - Min DOUBLE, - Max DOUBLE, - CC TEXT, -) - diff --git a/data/res/tables.sql b/data/res/tables.sql deleted file mode 100644 index d4d86f5..0000000 --- a/data/res/tables.sql +++ /dev/null @@ -1,96 +0,0 @@ -CREATE TABLE DATA_SRC ( - DataSrc_ID TEXT, - Authors TEXT, - Title TEXT, - Year TEXT, - Journal TEXT, - Vol_City TEXT, - Issue_State TEXT, - Start_Page TEXT, - End_Page TEXT -); -CREATE TABLE DATSRCLN ( - NDB_No TEXT, - Nutr_No TEXT, - DataSrc_ID TEXT -); -CREATE TABLE DERIV_CD ( - Deriv_Cd TEXT, - Deriv_Desc TEXT -); -CREATE TABLE FD_GROUP ( - FdGrp_Cd TEXT, - FdGrp_desc TEXT -); -CREATE TABLE FOOD_DES ( - NDB_No TEXT, - FdGrp_Cd TEXT, - Long_Desc TEXT, - Shrt_Desc TEXT, - Com_Name TEXT, - ManufacName TEXT, - Survey TEXT, - Ref_Desc TEXT, - Refuse INT, - Sci_Name TEXT, - N_FActor FLOAT, - Pro_Factor_ FLOAT, - Fat_Factor_ FLOAT, - CHO_Factor FLOAT -); -CREATE TABLE FOOTNOTE ( - NDB_No TEXT, - Footnt_No TEXT, - Footnt_Typ TEXT, - Nutr_No TEXT, - Footnt_Txt TEXT -); -CREATE TABLE LANGDESC ( - Factor TEXT, - Description TEXT -); -CREATE TABLE LANGUAL ( - NDB_No TEXT, - Factor TEXT -); -CREATE TABLE NUT_DATA ( - NDB_No TEXT, - Nutr_No TEXT, - Nutr_Val DOUBLE, - Num_Data_Pts INT, - Std_Error DOUBLE, - Src_Cd TEXT, - Deriv_Cd TEXT, - Ref_NDB_No TEXT, - Add_Nutr_Mark TEXT, - Num_Studies LONG, - Min DOUBLE, - Max DOUBLE, - DF LONG, - Low_EB DOUBLE, - Up_EB DOUBLE, - Stat_Cmt TEXT, - AddMod_Date TEXT -); -CREATE TABLE NUTR_DEF ( - Nutr_no TEXT, - Units TEXT, - Tagname TEXT, - NutrDesc TEXT, - Num_Dec TEXT, - SR_Order LONG -); -CREATE TABLE SRC_CD ( - Src_Cd TEXT, - SrcCd_Desc TEXT -); -CREATE TABLE WEIGHT ( - NDB_No TEXT, - Seq TEXT, - Amount FLOAT, - Msre_Desc TEXT, - Gm_Wgt DOUBLE, - Num_Data_pts LONG, - Std_Dev DOUBLE -); - diff --git a/data/setup.sh b/data/setup.sh index 9932fb8..987b120 100755 --- a/data/setup.sh +++ b/data/setup.sh @@ -1,7 +1,7 @@ #!/bin/bash -e -# nutra-db, a database for nutratracker clients -# Copyright (C) 2020 Nutra, LLC. [Shane & Kyle] +# nt-sqlite, an sqlite3 database for nutratracker clients +# Copyright (C) 2020 Shane Jaroch # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -83,7 +83,7 @@ cd ../isoflav # rm DATA_SRC.csv # rm DATSRCLN.csv # rm SYBN_DTL.csv -# mv ISFL_DAT.csv NUT_DATA.csv +mv ISFL_DAT.csv NUT_DATA.csv cd ../proanth # rm DATA_SRC.csv diff --git a/data/tables.sql b/data/tables.sql index b869da8..556ad7e 100644 --- a/data/tables.sql +++ b/data/tables.sql @@ -15,13 +15,37 @@ -- along with this program. If not, see . CREATE TABLE nutr_def ( - id INTEGER PRIMARY KEY AUTOINCREMENT, + id integer PRIMARY KEY AUTOINCREMENT, rda float, - unit TEXT NOT NULL, - tagname TEXT, - nutr_desc TEXT, - anti_nutrient BOOLEAN, - num_dec INT, - sr_order INT, - flav_class TEXT + unit text NOT NULL, + tagname text, + nutr_desc text, + anti_nutrient boolean, + num_dec int, + sr_order int, + flav_class text ); + +CREATE TABLE fdgrp ( + id integer PRIMARY KEY AUTOINCREMENT, + fdgrp_desc text NOT NULL +); + +CREATE TABLE food_des ( + id integer PRIMARY KEY AUTOINCREMENT, + fdgrp_id int, + long_desc text, + shrt_desc text, + com_name text, + manufac_name text, + survey text, + ref_desc text, + refuse int, + sci_name text, + n_factor FLOAT, + pro_factor_ FLOAT, + fat_factor_ FLOAT, + cho_factor FLOAT, + FOREIGN KEY (fdgrp_id) REFERENCES fdgrp (id) +); +