Re: Will this work?
[email protected] (Mikkel Eide Eriksen) Thu, 25 Feb 2010 17:59:11 +0100
| Newsgroups | perl.gedcom |
|---|---|
| Message-ID | <[email protected]> |
Hi Bob, John, Here's the script. I've attached it as it is rather small, hopefully it won't clog up this high-traffic list too bad :-) It was never meant for publication, so parts of it make assumptions. I've tried to add some comments where I could remember that special consideration was needed. One thing is it assumes that the entire file that it's working on is one single census transcript. The main part is line 80 and onwards. If I remember correctly, the script works rather well at creating the individuals themselves, but the created families need extra scrutiny. For instance, it may add resident servants as children of the house they're serving in. Or if a father & daughter live in the same house and are both widowed, they may end up married. Hopefully you can use it, and good luck with your respective projects! I'll try to answer any questions you may have as I can, but I am leaving on vacation for a couple weeks so email access may or may not be sporadic. Regards, Mikkel
kip2ged.20091110.pl
(text/x-perl-script, 7.2 KB)
#!/usr/bin/perl
use warnings;
use strict;
use diagnostics;
use Convert::Translit;
use Text::CSV::Simple;
use Gedcom;
use Data::Dumper;
/*
The following is just for getting the CSV:
*/
my $kiprequest = shift;
my $kipdata = "/Volumes/terra/Genealogy/FT + KB/kipdata.txt";
my $kippath = "/Volumes/terra/Genealogy/FT + KB/KIP/";
open (KIPDATA, $kipdata) || die "$!: $kipdata";
my @kipdata;
while (my $a = <KIPDATA>) {
chomp $a;
push @kipdata, $a;
}
close(KIPDATA);
my @kipmatches = grep { $_ =~ $kiprequest } @kipdata;
if (@kipmatches != 1) {
print @kipmatches." matches for '$kiprequest' found:\n";
foreach (@kipmatches) {
print " $_\n";
}
print "More accuracy needed.\n\n";
exit(0);
}
my ($kipamt, $kipherred, $kipsogn, $kipyear, $kipfile);
($kipamt, $kipherred, $kipsogn, $kipyear, $kipfile) = split(";", $kipmatches[0]);
my $file = $kippath.substr($kipfile,0,1)."/".$kipfile.".csv";
my $translator = new Convert::Translit("latin1", "mac");
open(HEADERS, "$file") || die "$!: $file";
my $headers = <HEADERS>;
close(HEADERS);
$headers =~ s/[\r\n]+//ig;
my @fields = split(";",$translator->transliterate($headers));
#print Dumper \@fields;
#$/ = "\r\n";
my $csv = Text::CSV::Simple->new({
sep_char => ";",
binary => 1,
eol => "\r\n"
});
$csv->field_map(@fields);
my @data = $csv->read_file($file);
#fix character map
for (my $i = 0; $i < @data; $i++) {
foreach my $key (keys %{$data[$i]}) {
$data[$i]->{$key} = $translator->transliterate($data[$i]->{$key});
}
}
shift @data;
#print Dumper \@data;
/*
Actual parsing starts here:
*/
my $ged = Gedcom->new;
my %fam;
my %houses;
#ADD PERSONS
foreach my $person (@data) {
my $indi = $ged->add_individual;
print "Adding... ".$person->{"Kildenavn"}."\n";
#name
my $gedname = $person->{"Kildenavn"};
$gedname =~ s|(\s)([^\s]+)$|$1/$2/|; #put // around surname as per gedcom standard
$indi->add("name", $gedname);
#gender (parsing is based on patronyms if no gender is in CSV)
my $gedgender = "U";
if ($person->{"Køn"} eq "M"
|| $person->{"Kildenavn"} =~ /(sen|son|søn)$/i
|| $person->{"Stilling i husstanden"} =~ /bonde/i) {
$gedgender = "M";
}
if ($person->{"Køn"} eq "F"
|| $person->{"Køn"} eq "K"
|| $person->{"Kildenavn"} =~ /datter$/i
|| $person->{"Kildenavn"} =~ /enke$/i
|| $person->{"Stilling i husstanden"} =~ /moder/i
|| $person->{"Stilling i husstanden"} =~ /pige/i
|| $person->{"Stilling i husstanden"} =~ /kone/i
|| $person->{"Stilling i husstanden"} =~ /datter/i
|| $person->{"Stilling i husstanden"} =~ /faster/i
|| $person->{"Stilling i husstanden"} =~ /moster/i
|| $person->{"Stilling i husstanden"} =~ /ske$/i) {
$gedgender = "F";
}
$indi->add("sex", $gedgender);
#birth
my $gedbirthdate;
if ($person->{"Født kildedato"}) {
$gedbirthdate = $person->{"Født kildedato"}; #TODO: convert date
} elsif ($person->{"Fødeår"}) {
$gedbirthdate = $person->{"Fødeår"};
} elsif ($person->{"Alder Kbhv."}) {
$gedbirthdate = "ABT ".($kipyear - $person->{"Alder Kbhv."}); #guess year from age
} elsif ($person->{"Alder"}) {
$gedbirthdate = "ABT ".($kipyear - $person->{"Alder"}); #guess year from age
} elsif ($person->{"Kildealder"}) {
$gedbirthdate = "ABT ".($kipyear - $person->{"Kildealder"}); #guess year from age
} else {
$gedbirthdate = "ABT $kipyear";
}
$indi->add("birth date", $gedbirthdate);
if ($person->{"Kildefødested"}) {
$indi->add("birth place", $person->{"Kildefødested"});
} elsif ($person->{"Fødested kun Frb. og Kbhv."}) {
$indi->add("birth place", $person->{"Fødested kun Frb. og Kbhv."});
}
#census record
$indi->add("census date", $kipyear);
if ($person->{"Kildestednavn"}) {
$indi->add("census place", "Husstand ".$person->{'Husstands/familienr.'}.", ".$person->{"Kildestednavn"}.", $kipsogn sogn, $kipherred herred, $kipamt amt, Denmark");
} else {
$indi->add("census place", "Husstand ".$person->{'Husstands/familienr.'}.", $kipsogn sogn, $kipherred herred, $kipamt amt, Denmark");
}
if ($person->{"Alder"}) {
$indi->add("census age", $person->{"Alder"});
} elsif ($person->{"Kildealder"}) {
$indi->add("census age", $person->{"Kildealder"});
}
#family key
my $famnum = $person->{'Kildestednavn'}."-".$person->{'Husstands/familienr.'};
my %p = (
'xref' => $indi->xref,
'marital' => $person->{"Civilstand"},
'position' => $person->{"Stilling i husstanden"},
'sex' => $gedgender
);
push @{$houses{$famnum}}, \%p;
#occupation
if ($person->{"Kildeerhverv"}) {
$indi->add("occupation", $person->{"Kildeerhverv"});
} elsif ($person->{"Stilling i husstanden"}) {
$indi->add("occupation", $person->{"Stilling i husstanden"});
}
}
/*
In the above, I've pushed the individuals into an array,
keyed by a "family key" (see line 157).
I run through that and create families after all the individuals
have been created.
*/
foreach my $h (keys %houses) {
# Works on the assumption that families are ordered (father first, then mother, then children)
my $fam = $ged->add_family;
#TODO: mandens faster, konens moder, moster faster etc.
#TODO: foster children
foreach my $i (@{$houses{$h}}) {
if ($i->{'marital'} =~ /(^Gift|enke)/i) {
if ($i->{'sex'} eq "M") {
if ($fam->husband) { $fam = $ged->add_family; }
$fam->add_husband($ged->resolve_xref($i->{'xref'}));
} elsif ($i->{'sex'} eq "F") {
if ($fam->wife) { $fam = $ged->add_family; }
$fam->add_wife($ged->resolve_xref($i->{'xref'}));
}
} elsif ($i->{'position'} =~ /(b[aø]rn|datter|søn)/i) {
$fam->add_child($ged->resolve_xref($i->{'xref'}));
}
}
}
foreach my $family ($ged->families) {
print "Verifying family: ".$family->xref."\n";
my @husbands = $family->husband;
my @wives = $family->wife;
if (@husbands > 1) {
print " too many husbands (".@husbands.")\n";
}
if (@wives > 1) {
print " too many wives (".@wives.")\n";
}
}
#print Dumper \%houses;
#WRITE OUTPUT
print "\nWriting output...";
$ged->order($ged->sort_sub);
$ged->write(substr($kipfile,0,5).".ged", 0);
print "Done.\n";
__END__
__DATA__
Some notes:
The colums that were in the original CSV:
'KIPnr' # census transciption ID
'Løbenr i indtastning' # serial
'Husstands/familienr.' # household number
'Antal familier/hus' # number of families per household
* 'Kildenavn' # name of individual
* 'Køn' # gender
* 'Fødeår' # birth year
* 'Født kildedato' # birth date
* 'Alder Kbhv.' # age (copenhagen census)
* 'Alder' # age
* 'Kildefødested' # birth place
* 'Fødested kun Frb. og Kbhv.' # birth place (copenhagen census)
* 'Kildestednavn' # place
'Adresse 1916'
'Adresse'
'Flyttet til kommunen'
'Matr.nr./Adresse'
'Matrikel 1916'
'Matrikel'
'Sidste bopæl/oph.sted'
'Midlertidig opholdssted'
'Midlertidigt nærværende'
* 'Civilstand' # civil position (married, widow, etc)
'Antal ægteskab'
'Hvornår æf. død'
'Hvornår gift'
'Nr. ægteskab'
'Stilling i husstanden'
'Antal levende børn'
'Antal døde børn'
'Kilde erhvervskommune'
'Kilde erhvervssted'
'Kildeerhverv'
'Arbejdsplads'
'Arbejdssted'
'Indkomst'
'Formue'
'Stats skat'
'Kommune skat'
'Trossamfund'
'Kommentar'
'Handicaps'
'Kildehenvisning'
'Transportmiddel'
'Boligtælling'