remove res folder

diff --git a/data/dep-graph.yml b/data/dep-graph.yml
new file mode 100644 (file)
index 0000000..71e12aa
--- /dev/null
+++ b/data/dep-graph.yml
@@ -0,0 +1,12 @@
+- NUTR_DEF
+- FD_GROUP
+  - FOOD_DES
+    - SRC_CD
+    - DERIV_CD
+      - NUT_DATA
+    - LANGDESC
+      - LANGUAL
+    - DATA_SRC
+      - DATASRCLN
+    - WEIGHT
+    - FOOTNOTE

diff --git a/data/format-sql.sh b/data/format-sql.sh
new file mode 100755 (executable)
index 0000000..2c499b6
--- /dev/null
+++ b/data/format-sql.sh
@@ -0,0 +1,6 @@
+#!/bin/bash
+
+cd "$(dirname "$0")"
+
+pg_format -s 2 tables.sql -o tables.sql
+# pg_format -s 2 functions.sql -o functions.sql

diff --git a/data/import.sql b/data/import.sql
index 4db1f7939c517e85b633fe9f74a484b1b1dd73e9..20a058146e3ee2a6f0ba4e3e9152a6dc015f0f6e 100644 (file)
--- a/data/import.sql
+++ b/data/import.sql
@@ -1,6 +1,8 @@
 .mode csv
 
 .import '| tail -n +2 nt/nutr_def.csv' nutr_def
+.import '| tail -n +2 nt/fdgrp.csv' fdgrp
+.import '| tail -n +2 nt/food_des.csv' food_des
 
 .header on
 .mode column

diff --git a/data/process.py b/data/process.py
index ff26a23352e37069cfc6baa7223afc031982d517..215d6309a9d30af2c24448493f650a8234cd4185 100644 (file)
--- a/data/process.py
+++ b/data/process.py
@@ -1,5 +1,5 @@
-# nutra-db, a database for nutratracker clients
-# Copyright (C) 2020  Nutra, LLC. [Shane & Kyle] <nutratracker@gmail.com>
+# nt-sqlite, an sqlite3 database for nutratracker clients
+# Copyright (C) 2020  Shane Jaroch <mathmuncher11@gmail.com>
 
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -16,10 +16,12 @@
 
 import csv
 import os
+import shutil
 import sys
 
 # change to script's dir
 os.chdir(os.path.dirname(os.path.abspath(__file__)))
+shutil.rmtree("nt", True)
 os.makedirs("nt", 0o755, True)
 
 
@@ -28,6 +30,13 @@ os.makedirs("nt", 0o755, True)
 # --------------------
 output_files = {
     "SR-Leg_DB/FD_GROUP.csv": "nt/fdgrp.csv",
+    "SR-Leg_DB/SRC_CD.csv": "nt/src_cd.csv",
+    "SR-Leg_DB/DERIV_CD.csv": "nt/deriv_cd.csv",
+    "SR-Leg_DB/LANGDESC.csv": "nt/lang_desc.csv",
+    "SR-Leg_DB/LANGUAL.csv": "nt/langual.csv",
+    "SR-Leg_DB/DATA_SRC.csv": "nt/data_src.csv",
+    "SR-Leg_DB/DATSRCLN.csv": "nt/datsrcln.csv",
+    "SR-Leg_DB/FOOTNOTE.csv": "nt/footnote.csv",
     "SR-Leg_DB/WEIGHT.csv": None,
 }
 
@@ -62,7 +71,7 @@ def main(args):
     # -----------------
     print("==> Process CSV")
     process_nutr_def()
-    process_nut_data()
+    # process_nut_data()
     process_food_des()
 
     for fname in output_files:
@@ -71,7 +80,7 @@ def main(args):
         with open(fname) as file:
             reader = csv.reader(file)
             rows = list(reader)
-        #
+        #########################
         # Process and write out
         if fname == "SR-Leg_DB/WEIGHT.csv":
             process_weight(rows, fname)
@@ -211,7 +220,7 @@ def process_nut_data():
             for row in rows[1:]:
                 result.append(row[:3])
 
-    #
+    #########################
     # Write out result
     with open("nt/nut_data.csv", "w+") as file:
         writer = csv.writer(file, lineterminator="\n")
@@ -234,16 +243,10 @@ def process_food_des():
         reader = csv.reader(file)
         rows = list(reader)
         # Add to final solution
-        for i, _row in enumerate(rows):
+        for i, row in enumerate(rows):
             if i > 0:
-                food_ids.add(int(_row[0]))
-            row = _row[:10]
-            del row[6]
-            row.insert(2, 1)  # Data src
-            row.append(None)  # user_id
-            row.append(True)  # is_shared
+                food_ids.add(int(row[0]))
             result.append(row)
-            # result.append(row[:3])
 
     # Special interests DB
     for dir in special_interests_dirs:
@@ -261,17 +264,15 @@ def process_food_des():
                     print(f"new food: {food_id} {_row[2]}")
                     food_ids.add(food_id)
                     # Set new row
-                    row = [None] * 12
+                    row = [None] * 14
                     row[0] = food_id
                     row[1] = _row[1]  # Food group
-                    row[2] = 2  # Data src
-                    row[3] = _row[2]  # Long Desc
+                    row[2] = _row[2]  # Long Desc
                     if len(_row) > 3:
-                        row[5] = _row[3]  # Sci name
-                    row[11] = True  # is_shared
+                        row[4] = _row[3]  # Comm/sci name
                     result.append(row)
 
-    #
+    #########################
     # Write out result
     with open("nt/food_des.csv", "w+") as file:
         writer = csv.writer(file, lineterminator="\n")
@@ -288,8 +289,8 @@ def process_weight(rows, fname):
     servings_set = set()
 
     # CSV rows
-    serving_id = [["id", "msre_desc"]]
-    servings = [["food_id", "msre_id", "grams"]]
+    serv_desc = [["id", "msre_desc"]]
+    serving = [["food_id", "msre_id", "grams", "num_data_pts", "std_dev"]]
 
     #
     # Main logic
@@ -303,10 +304,12 @@ def process_weight(rows, fname):
         msre_desc = row[3]
         grams = float(row[4])
         grams /= amount
+        num_data_pts = row[5]
+        std_dev = row[6]
 
         # Get key if used previously
         if msre_desc not in msre_ids:
-            serving_id.append([id, msre_desc])
+            serv_desc.append([id, msre_desc])
             msre_ids[msre_desc] = id
             id += 1
         msre_id = msre_ids[msre_desc]
@@ -316,17 +319,17 @@ def process_weight(rows, fname):
         # DETAIL:  Key (food_id, msre_id)=(1036, 3) already exists.
         prim_key = (food_id, msre_id)
         if prim_key not in servings_set:
-            servings.append([food_id, msre_id, grams])
+            serving.append([food_id, msre_id, grams, num_data_pts, std_dev])
             servings_set.add(prim_key)
 
-    #
-    # Write serving_id and servings tables
-    with open("nt/serving_id.csv", "w+") as file:
+    ##################################################
+    # Write serv_desc and serving tables
+    with open("nt/serv_desc.csv", "w+") as file:
         writer = csv.writer(file, lineterminator="\n")
-        writer.writerows(serving_id)
-    with open("nt/servings.csv", "w+") as file:
+        writer.writerows(serv_desc)
+    with open("nt/serving.csv", "w+") as file:
         writer = csv.writer(file, lineterminator="\n")
-        writer.writerows(servings)
+        writer.writerows(serving)
 
 
 #

diff --git a/data/res/dep-graph.yml b/data/res/dep-graph.yml
deleted file mode 100644 (file)
index 75af85d..0000000
--- a/data/res/dep-graph.yml
+++ /dev/null
@@ -1,11 +0,0 @@
-- NUTR_DEF
-  - FD_GROUP
-    - FOOD_DES
-      - SRC_CD && DERIV_CD
-        - NUT_DATA
-      - LANGDESC
-        - LANGUAL
-      - DATA_SRC
-        - DATASRCLN
-      - WEIGHT
-      - FOOTNOTE

diff --git a/data/res/flav/tables.sql b/data/res/flav/tables.sql
deleted file mode 100644 (file)
index 50226fe..0000000
--- a/data/res/flav/tables.sql
+++ /dev/null
@@ -1,64 +0,0 @@
-CREATE TABLE DATA_SRC (
-  DataSrc_ID TEXT,
-  Authors TEXT,
-  Title TEXT,
-  Jorunal TEXT,
-  Year TEXT,
-  Volume TEXT,
-  Issue TEXT,
-  Start_Page TEXT,
-  Emd_Page TEXT,
-)
-CREATE TABLE DATSRCLN (
-  NDB_No TEXT,
-  Nutr_no TEXT,
-  DataSrc_ID TEXT,
-)
-CREATE TABLE FD_GROUP (
-  FdGrp_CD TEXT,
-  FdGrp_Desc TEXT,
-)
-CREATE TABLE FLAV_DAT (
-  NDB_No TEXT,
-  Nutr_no TEXT,
-  Flav_Val DOUBLE,
-  SE DOUBLE,
-  n INT,
-  Min DOUBLE,
-  Max DOUBLE,
-  CC TEXT,
-)
-CREATE TABLE FLAV_IND (
-  NDB_No TEXT,
-  DataSrc ID TEXT,
-  Food No TEXT,
-  FoodIndiv_Desc TEXT,
-  Method TEXT,
-  Cmpd_Name TEXT,
-  Rptd CmpdVal DOUBLE,
-  Rptd_Std Dev TEXT,
-  Num_Data_Pts DOUBLE,
-  LT TEXT,
-  Rptd_Units TEXT,
-  Fresh Dry_Wt TEXT,
-  Quant_Std TEXT,
-  Conv_Factor_G DOUBLE,
-  Conv_Factor_M TEXT,
-  Conv_Factor_SpGr DOUBLE,
-  Cmpd_Val DOUBLE,
-  Cmpt_StdDev TEXT,
-)
-CREATE TABLE FOOD_DES (
-  NDB_No TEXT,
-  FdGrp_Cd TEXT,
-  Long_Desc TEXT,
-  SciName TEXT,
-)
-CREATE TABLE NUTR_DEF (
-  Nutr_no TEXT,
-  Nutrient name TEXT,
-  Flav_Class TEXT,
-  Tagname TEXT,
-  Unit TEXT,
-)
-

diff --git a/data/res/import.sql b/data/res/import.sql
deleted file mode 100644 (file)
index bbaeacf..0000000
--- a/data/res/import.sql
+++ /dev/null
@@ -1,20 +0,0 @@
-
-.read tables.sql
-
-.mode csv
-
-.import DATA_SRC.csv DATA_SRC
-.import DERIV_CD.csv DERIV_CD
-.import FOOD_DES.csv FOOD_DES
-.import LANGDESC.csv LANGDESC
-.import NUT_DATA.csv NUT_DATA
-.import SRC_CD.csv SRC_CD
-.import DATSRCLN.csv DATASRCLN
-.import FD_GROUP.csv FD_GROUP
-.import FOOTNOTE.csv FOOTNOTE
-.import LANGUAL.csv LANGUAL
-.import NUTR_DEF.csv NUTR_DEF
-.import WEIGHT.csv WEIGHT
-
-.headers on
-.mode column

diff --git a/data/res/isoflav/tables.sql b/data/res/isoflav/tables.sql
deleted file mode 100644 (file)
index 4bd8984..0000000
--- a/data/res/isoflav/tables.sql
+++ /dev/null
@@ -1,52 +0,0 @@
-CREATE TABLE DATA_SRC (
-  Authors TEXT,
-  Title TEXT,
-  Year DOUBLE,
-  Journal TEXT,
-  Vol TEXT,
-  Start_Page TEXT,
-  End_Page TEXT,
-  DataSrc_ID TEXT,
-)
-CREATE TABLE DATSRCLN (
-  NDB_No TEXT,
-  Nutr_No TEXT,
-  DataSrc_ID TEXT,
-)
-CREATE TABLE FOOD_DES (
-  NDB_No TEXT,
-  FdGrp_Cd TEXT,
-  Long_Desc TEXT,
-)
-CREATE TABLE ISFL_DAT (
-  NDB_No TEXT,
-  Nutr_No TEXT,
-  Isfl_Val DOUBLE,
-  SD DOUBLE,
-  n DOUBLE,
-  Min DOUBLE,
-  Max DOUBLE,
-  CC TEXT,
-  DataSrc_ID TEXT,
-)
-CREATE TABLE NUTR_DEF (
-  Nutr_no TEXT,
-  NutrDesc TEXT,
-  Unit TEXT,
-)
-CREATE TABLE SYBN_DTL (
-  NDB_No TEXT,
-  Nutr_No TEXT,
-  DataSrc_ID DOUBLE,
-  FoodNo TEXT,
-  Food_Detail_Desc TEXT,
-  Nutr_Val DOUBLE,
-  Std_Dev DOUBLE,
-  Num_Data_Pts DOUBLE,
-  Sam Hand_Rtg DOUBLE,
-  AnalMeth_Rtg DOUBLE,
-  SampPlan_Rtg DOUBLE,
-  AnalQC_Rtg DOUBLE,
-  NumSamp_QC DOUBLE,
-  CC TEXT,
-)

diff --git a/data/res/proanth/tables.sql b/data/res/proanth/tables.sql
deleted file mode 100644 (file)
index fe62259..0000000
--- a/data/res/proanth/tables.sql
+++ /dev/null
@@ -1,45 +0,0 @@
-CREATE TABLE DATA_SRC (
-  DataSrc_ID TEXT,
-  Authors TEXT,
-  Title TEXT,
-  Year DOUBLE,
-  Jounral TEXT,
-  Vol DOUBLE,
-  Issue DOUBLE,
-  Start_Page TEXT,
-  End_Page TEXT,
-)
-CREATE TABLE DATSRCLN (
-  NDB_No TEXT,
-  Nutr_No TEXT,
-  DataSrc_ID TEXT,
-)
-CREATE TABLE FD_GROUP (
-  FdGrp_CD TEXT,
-  FdGrp_Desc TEXT,
-)
-CREATE TABLE FOOD_DES (
-  NDB No TEXT,
-  FdGrp_Cd TEXT,
-  Long_Desc TEXT,
-  SciName TEXT,
-)
-CREATE TABLE NUTR_DEF (
-  Nutr_No TEXT,
-  Units TEXT,
-  Tagname TEXT,
-  NutrDesc TEXT,
-  Num_Dec TEXT,
-  SR_Order INT,
-)
-CREATE TABLE PA_DAT (
-  NDB No TEXT,
-  Nutr_No TEXT,
-  Flav_Val DOUBLE,
-  SD DOUBLE,
-  N DOUBLE,
-  Min DOUBLE,
-  Max DOUBLE,
-  CC TEXT,
-)
-

diff --git a/data/res/tables.sql b/data/res/tables.sql
deleted file mode 100644 (file)
index d4d86f5..0000000
--- a/data/res/tables.sql
+++ /dev/null
@@ -1,96 +0,0 @@
-CREATE TABLE DATA_SRC (
-  DataSrc_ID TEXT,
-  Authors TEXT,
-  Title TEXT,
-  Year TEXT,
-  Journal TEXT,
-  Vol_City TEXT,
-  Issue_State TEXT,
-  Start_Page TEXT,
-  End_Page TEXT
-);
-CREATE TABLE DATSRCLN (
-  NDB_No TEXT,
-  Nutr_No TEXT,
-  DataSrc_ID TEXT
-);
-CREATE TABLE DERIV_CD (
-  Deriv_Cd TEXT,
-  Deriv_Desc TEXT
-);
-CREATE TABLE FD_GROUP (
-  FdGrp_Cd TEXT,
-  FdGrp_desc TEXT
-);
-CREATE TABLE FOOD_DES (
-  NDB_No TEXT,
-  FdGrp_Cd TEXT,
-  Long_Desc TEXT,
-  Shrt_Desc TEXT,
-  Com_Name TEXT,
-  ManufacName TEXT,
-  Survey TEXT,
-  Ref_Desc TEXT,
-  Refuse INT,
-  Sci_Name TEXT,
-  N_FActor FLOAT,
-  Pro_Factor_ FLOAT,
-  Fat_Factor_ FLOAT,
-  CHO_Factor FLOAT
-);
-CREATE TABLE FOOTNOTE (
-  NDB_No TEXT,
-  Footnt_No TEXT,
-  Footnt_Typ TEXT,
-  Nutr_No TEXT,
-  Footnt_Txt TEXT
-);
-CREATE TABLE LANGDESC (
-  Factor TEXT,
-  Description TEXT
-);
-CREATE TABLE LANGUAL (
-  NDB_No TEXT,
-  Factor TEXT
-);
-CREATE TABLE NUT_DATA (
-  NDB_No TEXT,
-  Nutr_No TEXT,
-  Nutr_Val DOUBLE,
-  Num_Data_Pts INT,
-  Std_Error DOUBLE,
-  Src_Cd TEXT,
-  Deriv_Cd TEXT,
-  Ref_NDB_No TEXT,
-  Add_Nutr_Mark TEXT,
-  Num_Studies LONG,
-  Min DOUBLE,
-  Max DOUBLE,
-  DF LONG,
-  Low_EB DOUBLE,
-  Up_EB DOUBLE,
-  Stat_Cmt TEXT,
-  AddMod_Date TEXT
-);
-CREATE TABLE NUTR_DEF (
-  Nutr_no TEXT,
-  Units TEXT,
-  Tagname TEXT,
-  NutrDesc TEXT,
-  Num_Dec TEXT,
-  SR_Order LONG
-);
-CREATE TABLE SRC_CD (
-  Src_Cd TEXT,
-  SrcCd_Desc TEXT
-);
-CREATE TABLE WEIGHT (
-  NDB_No TEXT,
-  Seq TEXT,
-  Amount FLOAT,
-  Msre_Desc TEXT,
-  Gm_Wgt DOUBLE,
-  Num_Data_pts LONG,
-  Std_Dev DOUBLE
-);
-

diff --git a/data/setup.sh b/data/setup.sh
index 9932fb8b4f451bffa2e959f01714268242399692..987b120a6f15cdc5c6d3522e973c5e36fae496bd 100755 (executable)
--- a/data/setup.sh
+++ b/data/setup.sh
@@ -1,7 +1,7 @@
 #!/bin/bash -e
 
-# nutra-db, a database for nutratracker clients
-# Copyright (C) 2020  Nutra, LLC. [Shane & Kyle] <nutratracker@gmail.com>
+# nt-sqlite, an sqlite3 database for nutratracker clients
+# Copyright (C) 2020  Shane Jaroch <mathmuncher11@gmail.com>
 
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -83,7 +83,7 @@ cd ../isoflav
 # rm DATA_SRC.csv
 # rm DATSRCLN.csv
 # rm SYBN_DTL.csv
-# mv ISFL_DAT.csv NUT_DATA.csv
+mv ISFL_DAT.csv NUT_DATA.csv
 
 cd ../proanth
 # rm DATA_SRC.csv

diff --git a/data/tables.sql b/data/tables.sql
index b869da87fca0cbd0a362a9f14ba015d16cd7e139..556ad7e90cdfd7f7fb0c680368b0c1cad6e9ebb5 100644 (file)
--- a/data/tables.sql
+++ b/data/tables.sql
@@ -15,13 +15,37 @@
 -- along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
 CREATE TABLE nutr_def (
-  id INTEGER PRIMARY KEY AUTOINCREMENT,
+  id integer PRIMARY KEY AUTOINCREMENT,
   rda float,
-  unit TEXT NOT NULL,
-  tagname TEXT,
-  nutr_desc TEXT,
-  anti_nutrient BOOLEAN,
-  num_dec INT,
-  sr_order INT,
-  flav_class TEXT
+  unit text NOT NULL,
+  tagname text,
+  nutr_desc text,
+  anti_nutrient boolean,
+  num_dec int,
+  sr_order int,
+  flav_class text
 );
+
+CREATE TABLE fdgrp (
+  id integer PRIMARY KEY AUTOINCREMENT,
+  fdgrp_desc text NOT NULL
+);
+
+CREATE TABLE food_des (
+  id integer PRIMARY KEY AUTOINCREMENT,
+  fdgrp_id int,
+  long_desc text,
+  shrt_desc text,
+  com_name text,
+  manufac_name text,
+  survey text,
+  ref_desc text,
+  refuse int,
+  sci_name text,
+  n_factor FLOAT,
+  pro_factor_ FLOAT,
+  fat_factor_ FLOAT,
+  cho_factor FLOAT,
+  FOREIGN KEY (fdgrp_id) REFERENCES fdgrp (id)
+);
+