From: Benoît Fontaine Date: Sun, 17 Dec 2017 22:46:52 +0000 (+0100) Subject: add custom event and factorize facts X-Git-Url: https://git.nutra.tk/v1?a=commitdiff_plain;h=9bb18770c789b71410965d7b6fcd1127d2d8fe77;p=gamesguru%2Fgetmyancestors.git add custom event and factorize facts --- diff --git a/getmyancestors.py b/getmyancestors.py index e199be4..4cf6e83 100755 --- a/getmyancestors.py +++ b/getmyancestors.py @@ -32,9 +32,24 @@ try: import requests except ImportError: sys.stderr.write('You need to install the requests module first\n') - sys.stderr.write('(run this in your terminal: "python3 -m pip install requests" or "python3 -m pip install --user requests")\n') + sys.stderr.write( + '(run this in your terminal: "python3 -m pip install requests" or "python3 -m pip install --user requests")\n') exit(2) +FACT_TAGS = { + 'http://gedcomx.org/Birth': 'BIRT', + 'http://gedcomx.org/Christening': 'CHR', + 'http://gedcomx.org/Death': 'DEAT', + 'http://gedcomx.org/Burial': 'BURI', + 'http://gedcomx.org/PhysicalDescription': 'DSCR', + 'http://gedcomx.org/Occupation': 'OCCU', + 'http://gedcomx.org/MilitaryService': '_MILT', + 'http://gedcomx.org/Marriage': 'MARR', + 'http://gedcomx.org/Divorce': 'DIV', + 'http://gedcomx.org/Annulment': 'ANUL', + 'http://gedcomx.org/CommonLawMarriage': '_COML' +} + # FamilySearch session class class Session: @@ -53,11 +68,14 @@ class Session: try: url = 'https://www.familysearch.org/auth/familysearch/login' if self.verbose: - self.logfile.write('[' + time.strftime("%Y-%m-%d %H:%M:%S") + ']: Downloading: ' + url + '\n') - r = requests.get(url, params={'ldsauth': False}, allow_redirects=False) + self.logfile.write( + '[' + time.strftime("%Y-%m-%d %H:%M:%S") + ']: Downloading: ' + url + '\n') + r = requests.get( + url, params={'ldsauth': False}, allow_redirects=False) url = r.headers['Location'] if self.verbose: - self.logfile.write('[' + time.strftime("%Y-%m-%d %H:%M:%S") + ']: Downloading: ' + url + '\n') + self.logfile.write( + '[' + time.strftime("%Y-%m-%d %H:%M:%S") + ']: Downloading: ' + url + '\n') r = requests.get(url, allow_redirects=False) idx = r.text.index('name="params" value="') span = r.text[idx + 21:].index('"') @@ -65,51 +83,62 @@ class Session: url = 'https://ident.familysearch.org/cis-web/oauth2/v3/authorization' if self.verbose: - self.logfile.write('[' + time.strftime("%Y-%m-%d %H:%M:%S") + ']: Downloading: ' + url + '\n') - r = requests.post(url, data={'params': params, 'userName': self.username, 'password': self.password}, allow_redirects=False) + self.logfile.write( + '[' + time.strftime("%Y-%m-%d %H:%M:%S") + ']: Downloading: ' + url + '\n') + r = requests.post(url, data={ + 'params': params, 'userName': self.username, 'password': self.password}, allow_redirects=False) if 'The username or password was incorrect' in r.text: if self.verbose: - self.logfile.write('[' + time.strftime("%Y-%m-%d %H:%M:%S") + ']: The username or password was incorrect\n') + self.logfile.write( + '[' + time.strftime("%Y-%m-%d %H:%M:%S") + ']: The username or password was incorrect\n') exit() if 'Invalid Oauth2 Request' in r.text: if self.verbose: - self.logfile.write('[' + time.strftime("%Y-%m-%d %H:%M:%S") + ']: Invalid Oauth2 Request\n') + self.logfile.write( + '[' + time.strftime("%Y-%m-%d %H:%M:%S") + ']: Invalid Oauth2 Request\n') time.sleep(self.timeout) continue url = r.headers['Location'] if self.verbose: - self.logfile.write('[' + time.strftime("%Y-%m-%d %H:%M:%S") + ']: Downloading: ' + url + '\n') + self.logfile.write( + '[' + time.strftime("%Y-%m-%d %H:%M:%S") + ']: Downloading: ' + url + '\n') r = requests.get(url, allow_redirects=False) self.fssessionid = r.cookies['fssessionid'] except requests.exceptions.ReadTimeout: if self.verbose: - self.logfile.write('[' + time.strftime("%Y-%m-%d %H:%M:%S") + ']: Read timed out\n') + self.logfile.write( + '[' + time.strftime("%Y-%m-%d %H:%M:%S") + ']: Read timed out\n') continue except requests.exceptions.ConnectionError: if self.verbose: - self.logfile.write('[' + time.strftime("%Y-%m-%d %H:%M:%S") + ']: Connection aborted\n') + self.logfile.write( + '[' + time.strftime("%Y-%m-%d %H:%M:%S") + ']: Connection aborted\n') time.sleep(self.timeout) continue except requests.exceptions.HTTPError: if self.verbose: - self.logfile.write('[' + time.strftime("%Y-%m-%d %H:%M:%S") + ']: HTTPError\n') + self.logfile.write( + '[' + time.strftime("%Y-%m-%d %H:%M:%S") + ']: HTTPError\n') time.sleep(self.timeout) continue except KeyError: if self.verbose: - self.logfile.write('[' + time.strftime("%Y-%m-%d %H:%M:%S") + ']: KeyError\n') + self.logfile.write( + '[' + time.strftime("%Y-%m-%d %H:%M:%S") + ']: KeyError\n') time.sleep(self.timeout) continue except ValueError: if self.verbose: - self.logfile.write('[' + time.strftime("%Y-%m-%d %H:%M:%S") + ']: ValueError\n') + self.logfile.write( + '[' + time.strftime("%Y-%m-%d %H:%M:%S") + ']: ValueError\n') time.sleep(self.timeout) continue if self.verbose: - self.logfile.write('[' + time.strftime("%Y-%m-%d %H:%M:%S") + ']: FamilySearch session id: ' + self.fssessionid + '\n') + self.logfile.write('[' + time.strftime("%Y-%m-%d %H:%M:%S") + + ']: FamilySearch session id: ' + self.fssessionid + '\n') return # retrieve FamilySearch developer key (wget -O- --max-redirect 0 https://familysearch.org/auth/familysearch/login?ldsauth=false) @@ -117,54 +146,66 @@ class Session: url = 'https://familysearch.org/auth/familysearch/login' while True: if self.verbose: - self.logfile.write('[' + time.strftime("%Y-%m-%d %H:%M:%S") + ']: Downloading: ' + url + '\n') + self.logfile.write( + '[' + time.strftime("%Y-%m-%d %H:%M:%S") + ']: Downloading: ' + url + '\n') try: - r = requests.get(url, params={'ldsauth': False}, allow_redirects=False, timeout=self.timeout) + r = requests.get( + url, params={'ldsauth': False}, allow_redirects=False, timeout=self.timeout) location = r.headers['Location'] idx = location.index('client_id=') key = location[idx + 10:idx + 49] except ValueError: if self.verbose: - self.logfile.write('[' + time.strftime("%Y-%m-%d %H:%M:%S") + ']: FamilySearch developer key not found\n') + self.logfile.write( + '[' + time.strftime("%Y-%m-%d %H:%M:%S") + ']: FamilySearch developer key not found\n') time.sleep(self.timeout) continue if self.verbose: - self.logfile.write('[' + time.strftime("%Y-%m-%d %H:%M:%S") + ']: FamilySearch developer key: ' + key + '\n') + self.logfile.write( + '[' + time.strftime("%Y-%m-%d %H:%M:%S") + ']: FamilySearch developer key: ' + key + '\n') return key # retrieve FamilySearch session ID (https://familysearch.org/developers/docs/guides/oauth1/login) def old_login(self, oldmethod=False): url = 'https://api.familysearch.org/identity/v2/login' - data = {'key': self.key, 'username': self.username, 'password': self.password} + data = {'key': self.key, 'username': self.username, + 'password': self.password} while True: if self.verbose: - self.logfile.write('[' + time.strftime("%Y-%m-%d %H:%M:%S") + ']: Downloading: ' + url + '\n') + self.logfile.write( + '[' + time.strftime("%Y-%m-%d %H:%M:%S") + ']: Downloading: ' + url + '\n') try: r = requests.post(url, data, timeout=self.timeout) except requests.exceptions.ReadTimeout: if self.verbose: - self.logfile.write('[' + time.strftime("%Y-%m-%d %H:%M:%S") + ']: Read timed out\n') + self.logfile.write( + '[' + time.strftime("%Y-%m-%d %H:%M:%S") + ']: Read timed out\n') continue except requests.exceptions.ConnectionError: if self.verbose: - self.logfile.write('[' + time.strftime("%Y-%m-%d %H:%M:%S") + ']: Connection aborted\n') + self.logfile.write( + '[' + time.strftime("%Y-%m-%d %H:%M:%S") + ']: Connection aborted\n') time.sleep(self.timeout) continue if self.verbose: - self.logfile.write('[' + time.strftime("%Y-%m-%d %H:%M:%S") + ']: Status code: ' + str(r.status_code) + '\n') + self.logfile.write( + '[' + time.strftime("%Y-%m-%d %H:%M:%S") + ']: Status code: ' + str(r.status_code) + '\n') if r.status_code == 401: - self.logfile.write('[' + time.strftime("%Y-%m-%d %H:%M:%S") + ']: Login failure\n') + self.logfile.write( + '[' + time.strftime("%Y-%m-%d %H:%M:%S") + ']: Login failure\n') raise Exception('Login failure') try: r.raise_for_status() except requests.exceptions.HTTPError: if self.verbose: - self.logfile.write('[' + time.strftime("%Y-%m-%d %H:%M:%S") + ']: HTTPError\n') + self.logfile.write( + '[' + time.strftime("%Y-%m-%d %H:%M:%S") + ']: HTTPError\n') time.sleep(self.timeout) continue self.fssessionid = r.cookies['fssessionid'] if self.verbose: - self.logfile.write('[' + time.strftime("%Y-%m-%d %H:%M:%S") + ']: FamilySearch session id: ' + self.fssessionid + '\n') + self.logfile.write('[' + time.strftime("%Y-%m-%d %H:%M:%S") + + ']: FamilySearch session id: ' + self.fssessionid + '\n') return # retrieve JSON structure from FamilySearch URL @@ -172,20 +213,25 @@ class Session: while True: try: if self.verbose: - self.logfile.write('[' + time.strftime("%Y-%m-%d %H:%M:%S") + ']: Downloading: ' + url + '\n') + self.logfile.write( + '[' + time.strftime("%Y-%m-%d %H:%M:%S") + ']: Downloading: ' + url + '\n') # r = requests.get(url, cookies = { 's_vi': self.s_vi, 'fssessionid' : self.fssessionid }, timeout = self.timeout) - r = requests.get(url, cookies={'fssessionid': self.fssessionid}, timeout=self.timeout) + r = requests.get( + url, cookies={'fssessionid': self.fssessionid}, timeout=self.timeout) except requests.exceptions.ReadTimeout: if self.verbose: - self.logfile.write('[' + time.strftime("%Y-%m-%d %H:%M:%S") + ']: Read timed out\n') + self.logfile.write( + '[' + time.strftime("%Y-%m-%d %H:%M:%S") + ']: Read timed out\n') continue except requests.exceptions.ConnectionError: if self.verbose: - self.logfile.write('[' + time.strftime("%Y-%m-%d %H:%M:%S") + ']: Connection aborted\n') + self.logfile.write( + '[' + time.strftime("%Y-%m-%d %H:%M:%S") + ']: Connection aborted\n') time.sleep(self.timeout) continue if self.verbose: - self.logfile.write('[' + time.strftime("%Y-%m-%d %H:%M:%S") + ']: Status code: ' + str(r.status_code) + '\n') + self.logfile.write( + '[' + time.strftime("%Y-%m-%d %H:%M:%S") + ']: Status code: ' + str(r.status_code) + '\n') if r.status_code == 204 or r.status_code == 410: return None if r.status_code == 401: @@ -195,13 +241,16 @@ class Session: r.raise_for_status() except requests.exceptions.HTTPError: if self.verbose: - self.logfile.write('[' + time.strftime("%Y-%m-%d %H:%M:%S") + ']: HTTPError\n') + self.logfile.write( + '[' + time.strftime("%Y-%m-%d %H:%M:%S") + ']: HTTPError\n') if r.status_code == 403: if 'message' in r.json()['errors'][0] and r.json()['errors'][0]['message'] == u'Unable to get ordinances.': - self.logfile.write('Unable to get ordinances. Try with an LDS account or without option -c.\n') + self.logfile.write( + 'Unable to get ordinances. Try with an LDS account or without option -c.\n') exit() else: - self.logfile.write('Warning code 403 link: ' + url + ' ' + r.json()['errors'][0]['message'] or '') + self.logfile.write( + 'Warning code 403 link: ' + url + ' ' + r.json()['errors'][0]['message'] or '') return None time.sleep(self.timeout) continue @@ -233,7 +282,8 @@ class Note: self.text = text.strip() def print(self, file=sys.stdout): - file.write('0 @N' + str(self.num) + '@ NOTE ' + self.text.replace('\n', '\n1 CONT ') + '\n') + file.write('0 @N' + str(self.num) + '@ NOTE ' + + self.text.replace('\n', '\n1 CONT ') + '\n') def link(self, file=sys.stdout, level=1): file.write(str(level) + ' NOTE @N' + str(self.num) + '@\n') @@ -258,7 +308,8 @@ class Source: if data: self.fid = data['id'] if 'about' in data: - self.url = data['about'].replace('familysearch.org/platform/memories/memories', 'www.familysearch.org/photos/artifacts') + self.url = data['about'].replace( + 'familysearch.org/platform/memories/memories', 'www.familysearch.org/photos/artifacts') if 'citations' in data: self.citation = data['citations'][0]['value'] if data['titles']: @@ -273,7 +324,8 @@ class Source: if self.title: file.write('1 TITL ' + self.title.replace('\n', '\n2 CONT ') + '\n') if self.citation: - file.write('1 AUTH ' + self.citation.replace('\n', '\n2 CONT ') + '\n') + file.write( + '1 AUTH ' + self.citation.replace('\n', '\n2 CONT ') + '\n') if self.url: file.write('1 PUBL ' + self.url.replace('\n', '\n2 CONT ') + '\n') for n in self.notes: @@ -299,19 +351,27 @@ class Fact: self.place = data['place']['original'] if 'changeMessage' in data['attribution']: self.note = Note(data['attribution']['changeMessage'], tree) + if self.type == 'http://gedcomx.org/Death' and not any([self.date, self.place]): + self.value = 'Y' def print(self, file=sys.stdout, key=None): - if key: - file.write('1 ' + key) - if self.value: - file.write(' ' + self.value) - file.write('\n') - if self.date: - file.write('2 DATE ' + self.date + '\n') - if self.place: - file.write('2 PLAC ' + self.place + '\n') - if self.note: - self.note.link(file, 2) + if self.type in FACT_TAGS: + file.write('1 ' + FACT_TAGS[self.type]) + elif self.type[:6] == u'data:,': + file.write('1 EVEN\n2 TYPE ' + + self.type[6:] + '\n2 NOTE Description:') + else: + return + if self.value: + file.write(' ' + self.value) + file.write('\n') + if self.date: + file.write('2 DATE ' + self.date + '\n') + if self.place: + file.write('2 PLAC ' + self.place + '\n') + if self.note: + self.note.link(file, 2) + class Memorie: @@ -322,16 +382,18 @@ class Memorie: if 'titles' in data: self.description = data['titles'][0]['value'] if 'descriptions' in data: - self.description = ('' if not self.description else self.description + '\n') + data['descriptions'][0]['value'] - # self.description = (self.description or '') + '\n' + data['descriptions'][0]['value'] + self.description = ( + '' if not self.description else self.description + '\n') + data['descriptions'][0]['value'] def print(self, file=sys.stdout): file.write('1 OBJE\n2 FORM URL\n') if self.description: - file.write('2 TITL ' + self.description.replace('\n', '\n2 CONT ') + '\n') + file.write( + '2 TITL ' + self.description.replace('\n', '\n2 CONT ') + '\n') if self.url: file.write('2 FILE ' + self.url + '\n') + class Name: def __init__(self, data=None, tree=None): @@ -408,19 +470,15 @@ class Indi: self.num = Indi.counter self.fid = fid self.tree = tree - self.living = True self.famc_fid = set() self.fams_fid = set() self.famc_num = set() self.fams_num = set() self.name = None - self.gender = self.birtdate = self.birtplac = self.deatdate = self.deatplac = None - self.chrdate = self.chrplac = self.buridate = self.buriplac = None + self.gender = None self.baptism = self.confirmation = self.endowment = self.sealing_child = None - self.physical_descriptions = set() self.nicknames = set() - self.occupations = set() - self.military = set() + self.facts = set() self.birthnames = set() self.married = set() self.aka = set() @@ -432,7 +490,6 @@ class Indi: data = tree.fs.get_url(url) if data: x = data['persons'][0] - self.living = x['living'] if x['names']: for y in x['names']: if y['preferred']: @@ -454,32 +511,18 @@ class Indi: elif x['gender']['type'] == 'http://gedcomx.org/Unknown': self.gender = 'U' for y in x['facts']: - if y['type'] == u'http://gedcomx.org/Birth': - self.birtdate = y['date']['original'] if 'date' in y and 'original' in y['date'] else None - self.birtplac = y['place']['original'] if 'place' in y and 'original' in y['place'] else None - if y['type'] == u'http://gedcomx.org/Christening': - self.chrdate = y['date']['original'] if 'date' in y and 'original' in y['date'] else None - self.chrplac = y['place']['original'] if 'place' in y and 'original' in y['place'] else None - if y['type'] == u'http://gedcomx.org/Death': - self.deatdate = y['date']['original'] if 'date' in y and 'original' in y['date'] else None - self.deatplac = y['place']['original'] if 'place' in y and 'original' in y['place'] else None - if y['type'] == u'http://gedcomx.org/Burial': - self.buridate = y['date']['original'] if 'date' in y and 'original' in y['date'] else None - self.buriplac = y['place']['original'] if 'place' in y and 'original' in y['place'] else None - if y['type'] == u'http://gedcomx.org/PhysicalDescription': - self.physical_descriptions.add(Fact(y, self.tree)) - if y['type'] == u'http://gedcomx.org/Occupation': - self.occupations.add(Fact(y, self.tree)) - if y['type'] == u'http://gedcomx.org/MilitaryService': - self.military.add(Fact(y, self.tree)) if y['type'] == u'http://familysearch.org/v1/LifeSketch': - self.notes.add(Note('=== Life Sketch ===\n' + y['value'], self.tree)) + self.notes.add( + Note('=== Life Sketch ===\n' + y['value'], self.tree)) + else: + self.facts.add(Fact(y, self.tree)) if 'sources' in x: for y in x['sources']: source = self.tree.add_source(y['descriptionId']) if source: if 'changeMessage' in y['attribution']: - self.sources.add((source, y['attribution']['changeMessage'])) + self.sources.add( + (source, y['attribution']['changeMessage'])) else: self.sources.add((source,)) if 'evidence' in x: @@ -533,15 +576,18 @@ class Indi: url = 'https://familysearch.org/platform/tree/persons/%s/spouses.json' % self.fid data = self.tree.fs.get_url(url) if data and 'relationships' in data: - self.spouses = [(x['person1']['resourceId'], x['person2']['resourceId'], x['id']) for x in data['relationships']] + self.spouses = [(x['person1']['resourceId'], x['person2'] + ['resourceId'], x['id']) for x in data['relationships']] return self.spouses # retrieve individual notes def get_notes(self): - notes = self.tree.fs.get_url('https://familysearch.org/platform/tree/persons/%s/notes.json' % self.fid) + notes = self.tree.fs.get_url( + 'https://familysearch.org/platform/tree/persons/%s/notes.json' % self.fid) if notes: for n in notes['persons'][0]['notes']: - text_note = '===' + n['subject'] + '===\n' if 'subject' in n else '' + text_note = '===' + n['subject'] + \ + '===\n' if 'subject' in n else '' text_note += n['text'] + '\n' if 'text' in n else '' self.notes.add(Note(text_note, self.tree)) @@ -562,7 +608,8 @@ class Indi: if o['type'] == u'http://lds.org/SealingChildToParents': self.sealing_child = Ordinance(o) if 'father' in o and 'mother' in o: - famc = (o['father']['resourceId'], o['mother']['resourceId']) + famc = (o['father']['resourceId'], + o['mother']['resourceId']) if o['type'] == u'http://lds.org/SealingToSpouse': res.append(o) return res, famc @@ -570,7 +617,8 @@ class Indi: # retrieve contributors def get_contributors(self): temp = set() - data = self.tree.fs.get_url('https://familysearch.org/platform/tree/persons/%s/changes.json' % self.fid) + data = self.tree.fs.get_url( + 'https://familysearch.org/platform/tree/persons/%s/changes.json' % self.fid) for entries in data['entries']: for contributors in entries['contributors']: temp.add(contributors['name']) @@ -597,34 +645,10 @@ class Indi: o.print(file, 'married') if self.gender: file.write('1 SEX ' + self.gender + '\n') - if self.birtdate or self.birtplac: - file.write('1 BIRT\n') - if self.birtdate: - file.write('2 DATE ' + self.birtdate + '\n') - if self.birtplac: - file.write('2 PLAC ' + self.birtplac + '\n') - if self.chrdate or self.chrplac: - file.write('1 CHR\n') - if self.chrdate: - file.write('2 DATE ' + self.chrdate + '\n') - if self.chrplac: - file.write('2 PLAC ' + self.chrplac + '\n') - if self.deatdate or self.deatplac: - file.write('1 DEAT\n') - if self.deatdate: - file.write('2 DATE ' + self.deatdate + '\n') - if self.deatplac: - file.write('2 PLAC ' + self.deatplac + '\n') - elif not self.living: - file.write('1 DEAT Y\n') - if self.buridate or self.buriplac: - file.write('1 BURI\n') - if self.buridate: - file.write('2 DATE ' + self.buridate + '\n') - if self.buriplac: - file.write('2 PLAC ' + self.buriplac + '\n') - for o in self.physical_descriptions: - o.print(file, 'DSCR') + for o in self.facts: + o.print(file) + for o in self.memories: + o.print(file) if self.baptism: file.write('1 BAPL\n') self.baptism.print(file) @@ -641,12 +665,6 @@ class Indi: file.write('1 FAMS @F' + str(num) + '@\n') for num in self.famc_num: file.write('1 FAMC @F' + str(num) + '@\n') - for o in self.memories: - o.print(file) - for o in self.occupations: - o.print(file, 'OCCU') - for o in self.military: - o.print(file, '_MILT') file.write('1 _FSFTID ' + self.fid + '\n') for o in self.notes: o.link(file) @@ -671,7 +689,7 @@ class Fam: self.wife_fid = wife if wife else None self.tree = tree self.husb_num = self.wife_num = self.fid = None - self.marriage_facts = set() + self.facts = set() self.sealing_spouse = None self.chil_fid = set() self.chil_num = set() @@ -691,23 +709,26 @@ class Fam: data = self.tree.fs.get_url(url) if data and 'facts' in data['relationships'][0]: for x in data['relationships'][0]['facts']: - self.marriage_facts.add(Fact(x, self.tree)) + self.facts.add(Fact(x, self.tree)) if data and 'sources' in data['relationships'][0]: for y in data['relationships'][0]['sources']: source = self.tree.add_source(y['descriptionId']) if source: if 'changeMessage' in y['attribution']: - self.sources.add((source, y['attribution']['changeMessage'])) + self.sources.add( + (source, y['attribution']['changeMessage'])) else: self.sources.add((source,)) # retrieve marriage notes def get_notes(self): if self.fid: - notes = self.tree.fs.get_url('https://familysearch.org/platform/tree/couple-relationships/%s/notes.json' % self.fid) + notes = self.tree.fs.get_url( + 'https://familysearch.org/platform/tree/couple-relationships/%s/notes.json' % self.fid) if notes: for n in notes['relationships'][0]['notes']: - text_note = '===' + n['subject'] + '===\n' if 'subject' in n else '' + text_note = '===' + n['subject'] + \ + '===\n' if 'subject' in n else '' text_note += n['text'] + '\n' if 'text' in n else '' self.notes.add(Note(text_note, self.tree)) @@ -715,7 +736,8 @@ class Fam: def get_contributors(self): if self.fid: temp = set() - data = self.tree.fs.get_url('https://familysearch.org/platform/tree/couple-relationships/%s/changes.json' % self.fid) + data = self.tree.fs.get_url( + 'https://familysearch.org/platform/tree/couple-relationships/%s/changes.json' % self.fid) for entries in data['entries']: for contributors in entries['contributors']: temp.add(contributors['name']) @@ -736,18 +758,8 @@ class Fam: file.write('1 WIFE @I' + str(self.wife_num) + '@\n') for num in self.chil_num: file.write('1 CHIL @I' + str(num) + '@\n') - for o in self.marriage_facts: - key = '' - if o.type == u'http://gedcomx.org/Marriage': - key = 'MARR' - if o.type == u'http://gedcomx.org/Divorce': - key = 'DIV' - if o.type == u'http://gedcomx.org/Annulment': - key = 'ANUL' - if o.type == u'http://gedcomx.org/CommonLawMarriage': - key = '_COML' - if key: - o.print(file, key) + for o in self.facts: + o.print(file) if self.sealing_spouse: file.write('1 SLGS\n') self.sealing_spouse.print(file) @@ -829,16 +841,19 @@ class Tree: self.indi[fid].sealing_child.famc = self.fam[famc] for o in ret: if (fid, o['spouse']['resourceId']) in self.fam: - self.fam[(fid, o['spouse']['resourceId'])].sealing_spouse = Ordinance(o) + self.fam[(fid, o['spouse']['resourceId']) + ].sealing_spouse = Ordinance(o) elif (o['spouse']['resourceId'], fid) in self.fam: - self.fam[(o['spouse']['resourceId'], fid)].sealing_spouse = Ordinance(o) + self.fam[(o['spouse']['resourceId'], fid) + ].sealing_spouse = Ordinance(o) # Find source by fid def add_source(self, fid=None): if fid: if fid in self.sources: return self.sources[fid] - data = self.fs.get_url('https://familysearch.org/platform/sources/descriptions/%s.json' % fid) + data = self.fs.get_url( + 'https://familysearch.org/platform/sources/descriptions/%s.json' % fid) if data: return Source(data['sourceDescriptions'][0], self) return False @@ -847,10 +862,13 @@ class Tree: for husb, wife in self.fam: self.fam[(husb, wife)].husb_num = self.indi[husb].num if husb else None self.fam[(husb, wife)].wife_num = self.indi[wife].num if wife else None - self.fam[(husb, wife)].chil_num = set([self.indi[chil].num for chil in self.fam[(husb, wife)].chil_fid]) + self.fam[(husb, wife)].chil_num = set( + [self.indi[chil].num for chil in self.fam[(husb, wife)].chil_fid]) for fid in self.indi: - self.indi[fid].famc_num = set([self.fam[(husb, wife)].num for husb, wife in self.indi[fid].famc_fid]) - self.indi[fid].fams_num = set([self.fam[(husb, wife)].num for husb, wife in self.indi[fid].fams_fid]) + self.indi[fid].famc_num = set( + [self.fam[(husb, wife)].num for husb, wife in self.indi[fid].famc_fid]) + self.indi[fid].fams_num = set( + [self.fam[(husb, wife)].num for husb, wife in self.indi[fid].fams_fid]) # print GEDCOM file def print(self, file=sys.stdout): @@ -876,23 +894,37 @@ class Tree: if __name__ == '__main__': - parser = argparse.ArgumentParser(description='Retrieve GEDCOM data from FamilySearch Tree (4 Jul 2016)', add_help=False, usage='getmyancestors.py -u username -p password [options]') - parser.add_argument('-u', metavar='', type=str, help='FamilySearch username') - parser.add_argument('-p', metavar='', type=str, help='FamilySearch password') - parser.add_argument('-i', metavar='', nargs='+', type=str, help='List of individual FamilySearch IDs for whom to retrieve ancestors') - parser.add_argument('-a', metavar='', type=int, default=4, help='Number of generations to ascend [4]') - parser.add_argument('-d', metavar='', type=int, default=0, help='Number of generations to descend [0]') - parser.add_argument('-m', action="store_true", default=False, help='Add spouses and couples information [False]') - parser.add_argument('-r', action="store_true", default=False, help='Add list of contributors in notes [False]') - parser.add_argument('-c', action="store_true", default=False, help='Add LDS ordinances (need LDS account) [False]') - parser.add_argument("-v", action="store_true", default=False, help="Increase output verbosity [False]") - parser.add_argument('-t', metavar='', type=int, default=60, help='Timeout in seconds [60]') + parser = argparse.ArgumentParser(description='Retrieve GEDCOM data from FamilySearch Tree (4 Jul 2016)', + add_help=False, usage='getmyancestors.py -u username -p password [options]') + parser.add_argument('-u', metavar='', type=str, + help='FamilySearch username') + parser.add_argument('-p', metavar='', type=str, + help='FamilySearch password') + parser.add_argument('-i', metavar='', nargs='+', type=str, + help='List of individual FamilySearch IDs for whom to retrieve ancestors') + parser.add_argument('-a', metavar='', type=int, + default=4, help='Number of generations to ascend [4]') + parser.add_argument('-d', metavar='', type=int, + default=0, help='Number of generations to descend [0]') + parser.add_argument('-m', action="store_true", default=False, + help='Add spouses and couples information [False]') + parser.add_argument('-r', action="store_true", default=False, + help='Add list of contributors in notes [False]') + parser.add_argument('-c', action="store_true", default=False, + help='Add LDS ordinances (need LDS account) [False]') + parser.add_argument("-v", action="store_true", default=False, + help="Increase output verbosity [False]") + parser.add_argument('-t', metavar='', type=int, + default=60, help='Timeout in seconds [60]') try: - parser.add_argument('-o', metavar='', type=argparse.FileType('w', encoding='UTF-8'), default=sys.stdout, help='output GEDCOM file [stdout]') - parser.add_argument('-l', metavar='', type=argparse.FileType('w', encoding='UTF-8'), default=sys.stderr, help='output log file [stderr]') + parser.add_argument('-o', metavar='', type=argparse.FileType( + 'w', encoding='UTF-8'), default=sys.stdout, help='output GEDCOM file [stdout]') + parser.add_argument('-l', metavar='', type=argparse.FileType( + 'w', encoding='UTF-8'), default=sys.stderr, help='output log file [stderr]') except TypeError: sys.stderr.write('Python >= 3.4 is required to run this script\n') - sys.stderr.write('(see https://docs.python.org/3/whatsnew/3.4.html#argparse)\n') + sys.stderr.write( + '(see https://docs.python.org/3/whatsnew/3.4.html#argparse)\n') exit(2) # extract arguments from the command line @@ -904,7 +936,8 @@ if __name__ == '__main__': exit(2) username = args.u if args.u else input("Enter FamilySearch username: ") - password = args.p if args.p else getpass.getpass("Enter FamilySearch password: ") + password = args.p if args.p else getpass.getpass( + "Enter FamilySearch password: ") # initialize a FamilySearch session and a family tree object fs = Session(username, password, args.v, args.l, args.t) @@ -912,7 +945,8 @@ if __name__ == '__main__': # check LDS account if args.c: - fs.get_url('https://familysearch.org/platform/tree/persons/%s/ordinances.json' % fs.get_userid()) + fs.get_url( + 'https://familysearch.org/platform/tree/persons/%s/ordinances.json' % fs.get_userid()) loop = asyncio.get_event_loop() @@ -935,11 +969,13 @@ if __name__ == '__main__': todo = next_todo - done # download ancestors - loop.run_until_complete(download_tree(tree.add_parents, todo, args.a, loop)) + loop.run_until_complete(download_tree( + tree.add_parents, todo, args.a, loop)) # download descendants todo = set(tree.indi.keys()) - loop.run_until_complete(download_tree(tree.add_children, todo, args.d, loop)) + loop.run_until_complete(download_tree( + tree.add_children, todo, args.d, loop)) # download spouses async def download_spouses(loop): @@ -958,7 +994,8 @@ if __name__ == '__main__': futures = set() for fid, indi in tree.indi.items(): if args.c: - futures.add(loop.run_in_executor(None, tree.add_ordinances, fid)) + futures.add(loop.run_in_executor( + None, tree.add_ordinances, fid)) futures.add(loop.run_in_executor(None, indi.get_notes)) if args.r: futures.add(loop.run_in_executor(None, indi.get_contributors)) @@ -974,3 +1011,5 @@ if __name__ == '__main__': # compute number for family relationships and print GEDCOM file tree.reset_num() tree.print(args.o) + + import pdb; pdb.set_trace() \ No newline at end of file diff --git a/mergemyancestors.py b/mergemyancestors.py index 85d9ac5..e66fa2e 100755 --- a/mergemyancestors.py +++ b/mergemyancestors.py @@ -29,10 +29,15 @@ import sys import argparse # local import -from getmyancestors import Indi, Fam, Tree, Name, Note, Fact, Source, Ordinance, Memorie +from getmyancestors import FACT_TAGS, Indi, Fam, Tree, Name, Note, Fact, Source, Ordinance, Memorie sys.path.append(os.path.dirname(sys.argv[0])) +FACT_TYPES = dict() + +for key, value in FACT_TAGS.items(): + FACT_TYPES[value] = key + class Gedcom: @@ -102,16 +107,8 @@ class Gedcom: self.__get_name() elif self.tag == 'SEX': self.indi[self.num].gender = self.data - elif self.tag == 'BIRT': - self.__get_birt() - elif self.tag == 'CHR': - self.__get_chr() - elif self.tag == 'DEAT': - self.__get_deat() - elif self.tag == 'BURI': - self.__get_buri() - elif self.tag == 'DSCR' or self.tag == 'OCCU' or self.tag == '_MILT': - self.__get_fact() + elif self.tag in FACT_TYPES or self.tag == 'EVEN': + self.indi[self.num].facts.add(self.__get_fact()) elif self.tag == 'BAPL': self.indi[self.num].baptism = self.__get_ordinance() elif self.tag == 'CONL': @@ -121,9 +118,11 @@ class Gedcom: elif self.tag == 'SLGC': self.indi[self.num].sealing_child = self.__get_ordinance() elif self.tag == 'FAMS': - self.indi[self.num].fams_num.add(int(self.data[2:len(self.data) - 1])) + self.indi[self.num].fams_num.add( + int(self.data[2:len(self.data) - 1])) elif self.tag == 'FAMC': - self.indi[self.num].famc_num.add(int(self.data[2:len(self.data) - 1])) + self.indi[self.num].famc_num.add( + int(self.data[2:len(self.data) - 1])) elif self.tag == '_FSFTID': self.indi[self.num].fid = self.data elif self.tag == 'NOTE': @@ -140,13 +139,16 @@ class Gedcom: def __get_fam(self): while self.__get_line() and self.level > 0: if self.tag == 'HUSB': - self.fam[self.num].husb_num = int(self.data[2:len(self.data) - 1]) + self.fam[self.num].husb_num = int( + self.data[2:len(self.data) - 1]) elif self.tag == 'WIFE': - self.fam[self.num].wife_num = int(self.data[2:len(self.data) - 1]) + self.fam[self.num].wife_num = int( + self.data[2:len(self.data) - 1]) elif self.tag == 'CHIL': - self.fam[self.num].chil_num.add(int(self.data[2:len(self.data) - 1])) - elif self.tag in ('MARR', 'DIV', 'ANUL', '_COML'): - self.fam[self.num].marriage_facts.add(self.__get_marr()) + self.fam[self.num].chil_num.add( + int(self.data[2:len(self.data) - 1])) + elif self.tag in FACT_TYPES: + self.fam[self.num].facts.add(self.__get_fact()) elif self.tag == 'SLGS': self.fam[self.num].sealing_spouse = self.__get_ordinance() elif self.tag == '_FSFTID': @@ -211,7 +213,6 @@ class Gedcom: self.flag = True def __get_deat(self): - self.indi[self.num].living = False while self.__get_line() and self.level > 1: if self.tag == 'DATE': self.indi[self.num].deatdate = self.data @@ -227,38 +228,16 @@ class Gedcom: self.indi[self.num].buriplac = self.data self.flag = True - def __get_marr(self): - fact = Fact() - if self.tag == 'MARR': - fact.type = 'http://gedcomx.org/Marriage' - elif self.tag == 'DIV': - fact.type = 'http://gedcomx.org/Divorce' - elif self.tag == 'ANUL': - fact.type = 'http://gedcomx.org/Annulment' - elif self.tag == '_COML': - fact.type = 'http://gedcomx.org/CommonLawMarriage' - while self.__get_line() and self.level > 1: - if self.tag == 'DATE': - fact.date = self.data - elif self.tag == 'PLAC': - fact.place = self.data - elif self.tag == 'NOTE': - num = int(self.data[2:len(self.data) - 1]) - if num not in self.note: - self.note[num] = Note(tree=self.tree, num=num) - fact.note = self.note[num] - self.flag = True - return fact - def __get_fact(self): fact = Fact() - fact.value = self.data - if self.tag == 'DSCR': - self.indi[self.num].physical_descriptions.add(fact) - elif self.tag == 'OCCU': - self.indi[self.num].occupations.add(fact) - elif self.tag == '_MILT': - self.indi[self.num].military.add(fact) + if self.tag == 'EVEN': + self.__get_line() + fact.type = 'data:,' + self.data + self.__get_line() + fact.value = self.data[12:] + else: + fact.type = FACT_TYPES[self.tag] + fact.value = self.data while self.__get_line() and self.level > 1: if self.tag == 'DATE': fact.date = self.data @@ -270,6 +249,7 @@ class Gedcom: self.note[num] = Note(tree=self.tree, num=num) fact.note = self.note[num] self.flag = True + return fact def __get_text(self): text = self.data @@ -318,7 +298,6 @@ class Gedcom: def __get_memorie(self): memorie = Memorie() - pdb = False while self.__get_line() and self.level > 1: if self.tag == 'TITL': memorie.description = self.__get_text() @@ -358,19 +337,25 @@ class Gedcom: self.fam[num].chil_fid.add(self.indi[chil].fid) for num in self.indi: for famc in self.indi[num].famc_num: - self.indi[num].famc_fid.add((self.fam[famc].husb_fid, self.fam[famc].wife_fid)) + self.indi[num].famc_fid.add( + (self.fam[famc].husb_fid, self.fam[famc].wife_fid)) for fams in self.indi[num].fams_num: - self.indi[num].fams_fid.add((self.fam[fams].husb_fid, self.fam[fams].wife_fid)) + self.indi[num].fams_fid.add( + (self.fam[fams].husb_fid, self.fam[fams].wife_fid)) if __name__ == '__main__': - parser = argparse.ArgumentParser(description='Merge GEDCOM data from FamilySearch Tree (4 Jul 2016)', add_help=False, usage='mergemyancestors.py -i input1.ged input2.ged ... [options]') + parser = argparse.ArgumentParser(description='Merge GEDCOM data from FamilySearch Tree (4 Jul 2016)', + add_help=False, usage='mergemyancestors.py -i input1.ged input2.ged ... [options]') try: - parser.add_argument('-i', metavar='', nargs='+', type=argparse.FileType('r', encoding='UTF-8'), default=sys.stdin, help='input GEDCOM files [stdin]') - parser.add_argument('-o', metavar='', nargs='?', type=argparse.FileType('w', encoding='UTF-8'), default=sys.stdout, help='output GEDCOM files [stdout]') + parser.add_argument('-i', metavar='', nargs='+', type=argparse.FileType( + 'r', encoding='UTF-8'), default=sys.stdin, help='input GEDCOM files [stdin]') + parser.add_argument('-o', metavar='', nargs='?', type=argparse.FileType( + 'w', encoding='UTF-8'), default=sys.stdout, help='output GEDCOM files [stdout]') except TypeError: sys.stderr.write('Python >= 3.4 is required to run this script\n') - sys.stderr.write('(see https://docs.python.org/3/whatsnew/3.4.html#argparse)\n') + sys.stderr.write( + '(see https://docs.python.org/3/whatsnew/3.4.html#argparse)\n') exit(2) # extract arguments from the command line @@ -400,7 +385,6 @@ if __name__ == '__main__': tree.indi[fid] = Indi(tree=tree, num=indi_counter) tree.indi[fid].tree = tree tree.indi[fid].fid = ged.indi[num].fid - tree.indi[fid].living = ged.indi[num].living tree.indi[fid].fams_fid |= ged.indi[num].fams_fid tree.indi[fid].famc_fid |= ged.indi[num].famc_fid tree.indi[fid].name = ged.indi[num].name @@ -409,17 +393,7 @@ if __name__ == '__main__': tree.indi[fid].aka = ged.indi[num].aka tree.indi[fid].married = ged.indi[num].married tree.indi[fid].gender = ged.indi[num].gender - tree.indi[fid].birtdate = ged.indi[num].birtdate - tree.indi[fid].birtplac = ged.indi[num].birtplac - tree.indi[fid].chrdate = ged.indi[num].chrdate - tree.indi[fid].chrplac = ged.indi[num].chrplac - tree.indi[fid].deatdate = ged.indi[num].deatdate - tree.indi[fid].deatplac = ged.indi[num].deatplac - tree.indi[fid].buridate = ged.indi[num].buridate - tree.indi[fid].buriplac = ged.indi[num].buriplac - tree.indi[fid].physical_descriptions = ged.indi[num].physical_descriptions - tree.indi[fid].occupations = ged.indi[num].occupations - tree.indi[fid].military = ged.indi[num].military + tree.indi[fid].facts = ged.indi[num].facts tree.indi[fid].notes = ged.indi[num].notes tree.indi[fid].sources = ged.indi[num].sources tree.indi[fid].memories = ged.indi[num].memories @@ -438,7 +412,7 @@ if __name__ == '__main__': tree.fam[(husb, wife)].tree = tree tree.fam[(husb, wife)].chil_fid |= ged.fam[num].chil_fid tree.fam[(husb, wife)].fid = ged.fam[num].fid - tree.fam[(husb, wife)].marriage_facts = ged.fam[num].marriage_facts + tree.fam[(husb, wife)].facts = ged.fam[num].facts tree.fam[(husb, wife)].notes = ged.fam[num].notes tree.fam[(husb, wife)].sources = ged.fam[num].sources tree.fam[(husb, wife)].sealing_spouse = ged.fam[num].sealing_spouse