Re: Will this work?

[email protected] (Mikkel Eide Eriksen) Thu, 25 Feb 2010 17:59:11 +0100
Newsgroups perl.gedcom
Message-ID <[email protected]>
Hi Bob, John,

Here's the script. I've attached it as it is rather small, hopefully  
it won't clog up this high-traffic list too bad :-)

It was never meant for publication, so parts of it make assumptions.   
I've tried to add some comments where I could remember that special  
consideration was needed. One thing is it assumes that the entire file  
that it's working on is one single census transcript.

The main part is line 80 and onwards.

If I remember correctly, the script works rather well at creating the  
individuals themselves, but the created families need extra scrutiny.  
For instance, it may add resident servants as children of the house  
they're serving in. Or if a father & daughter live in the same house  
and are both widowed, they may end up married.

Hopefully you can use it, and good luck with your respective projects!

I'll try to answer any questions you may have as I can, but I am  
leaving on vacation for a couple weeks so email access may or may not  
be sporadic.

Regards,

Mikkel
kip2ged.20091110.pl (text/x-perl-script, 7.2 KB)
#!/usr/bin/perl

use warnings;
use strict;
use diagnostics;

use Convert::Translit;
use Text::CSV::Simple;

use Gedcom;

use Data::Dumper;

/*
	The following is just for getting the CSV:
*/

my $kiprequest = shift;

my $kipdata = "/Volumes/terra/Genealogy/FT + KB/kipdata.txt";
my $kippath = "/Volumes/terra/Genealogy/FT + KB/KIP/";

open (KIPDATA, $kipdata) || die "$!: $kipdata";
my @kipdata;
while (my $a = <KIPDATA>) {
	chomp $a;
	push @kipdata, $a;
}
close(KIPDATA);

my @kipmatches = grep { $_ =~ $kiprequest } @kipdata;

if (@kipmatches != 1) {
	print @kipmatches." matches for '$kiprequest' found:\n";
	foreach (@kipmatches) {
		print "  $_\n";
	}
	print "More accuracy needed.\n\n";
	exit(0);
}

my ($kipamt, $kipherred, $kipsogn, $kipyear, $kipfile);

($kipamt, $kipherred, $kipsogn, $kipyear, $kipfile) = split(";", $kipmatches[0]);
my $file = $kippath.substr($kipfile,0,1)."/".$kipfile.".csv";

my $translator = new Convert::Translit("latin1", "mac");

open(HEADERS, "$file") || die "$!: $file";
my $headers = <HEADERS>;
close(HEADERS);
$headers =~ s/[\r\n]+//ig;
my @fields = split(";",$translator->transliterate($headers));

#print Dumper \@fields;

#$/ = "\r\n";

my $csv = Text::CSV::Simple->new({
                                  sep_char => ";", 
                                  binary => 1,
                                  eol => "\r\n"
                                 });
$csv->field_map(@fields);
my @data = $csv->read_file($file);

#fix character map
for (my $i = 0; $i < @data; $i++) {
	foreach my $key (keys %{$data[$i]}) {
		$data[$i]->{$key} = $translator->transliterate($data[$i]->{$key});
	}
}

shift @data;

#print Dumper \@data;

/*
	Actual parsing starts here:
*/

my $ged = Gedcom->new;

my %fam;
my %houses;

#ADD PERSONS

foreach my $person (@data) {
	my $indi = $ged->add_individual;
	
	print "Adding... ".$person->{"Kildenavn"}."\n";
	
	#name
	my $gedname = $person->{"Kildenavn"};
	$gedname =~ s|(\s)([^\s]+)$|$1/$2/|; #put // around surname as per gedcom standard
	$indi->add("name", $gedname);
	
	#gender (parsing is based on patronyms if no gender is in CSV)
	my $gedgender = "U";
	if ($person->{"Køn"} eq "M"
	 || $person->{"Kildenavn"} =~ /(sen|son|søn)$/i
	 || $person->{"Stilling i husstanden"} =~ /bonde/i) {
		$gedgender = "M";
	} 
	if ($person->{"Køn"} eq "F"
	 || $person->{"Køn"} eq "K"
	 || $person->{"Kildenavn"} =~ /datter$/i
	 || $person->{"Kildenavn"} =~ /enke$/i
	 || $person->{"Stilling i husstanden"} =~ /moder/i
	 || $person->{"Stilling i husstanden"} =~ /pige/i
	 || $person->{"Stilling i husstanden"} =~ /kone/i
	 || $person->{"Stilling i husstanden"} =~ /datter/i
	 || $person->{"Stilling i husstanden"} =~ /faster/i
	 || $person->{"Stilling i husstanden"} =~ /moster/i
	 || $person->{"Stilling i husstanden"} =~ /ske$/i) {
		$gedgender = "F";
	}
	$indi->add("sex", $gedgender);
	
	#birth
	my $gedbirthdate;
	if ($person->{"Født kildedato"}) {
		$gedbirthdate = $person->{"Født kildedato"}; #TODO: convert date
	} elsif ($person->{"Fødeår"}) {
		$gedbirthdate = $person->{"Fødeår"};
	} elsif ($person->{"Alder Kbhv."}) {
		$gedbirthdate = "ABT ".($kipyear - $person->{"Alder Kbhv."}); #guess year from age
	} elsif ($person->{"Alder"}) {
		$gedbirthdate = "ABT ".($kipyear - $person->{"Alder"}); #guess year from age
	} elsif ($person->{"Kildealder"}) {
		$gedbirthdate = "ABT ".($kipyear - $person->{"Kildealder"}); #guess year from age
	} else {
		$gedbirthdate = "ABT $kipyear";
	}
	$indi->add("birth date", $gedbirthdate);
	if ($person->{"Kildefødested"}) {
		$indi->add("birth place", $person->{"Kildefødested"});
	} elsif ($person->{"Fødested kun Frb. og Kbhv."}) {
		$indi->add("birth place", $person->{"Fødested kun Frb. og Kbhv."});
	}
	
	#census record
	$indi->add("census date", $kipyear);
	if ($person->{"Kildestednavn"}) {
		$indi->add("census place", "Husstand ".$person->{'Husstands/familienr.'}.", ".$person->{"Kildestednavn"}.", $kipsogn sogn, $kipherred herred, $kipamt amt, Denmark");
	} else {
		$indi->add("census place", "Husstand ".$person->{'Husstands/familienr.'}.", $kipsogn sogn, $kipherred herred, $kipamt amt, Denmark");
	}
	if ($person->{"Alder"}) {
		$indi->add("census age", $person->{"Alder"});
	} elsif ($person->{"Kildealder"}) {
		$indi->add("census age", $person->{"Kildealder"});
	}
	
	#family key
	my $famnum = $person->{'Kildestednavn'}."-".$person->{'Husstands/familienr.'};
	my %p = (
				'xref' => $indi->xref,
				'marital' => $person->{"Civilstand"},
				'position' => $person->{"Stilling i husstanden"},
				'sex' => $gedgender
			);
	push @{$houses{$famnum}}, \%p;
	
	#occupation
	if ($person->{"Kildeerhverv"}) {
		$indi->add("occupation", $person->{"Kildeerhverv"});
	} elsif ($person->{"Stilling i husstanden"}) {
		$indi->add("occupation", $person->{"Stilling i husstanden"});
	}
}

/*
	In the above, I've pushed the individuals into an array,
	keyed by a "family key" (see line 157).
	
	I run through that and create families after all the individuals
	have been created.
*/

foreach my $h (keys %houses) {
	# Works on the assumption that families are ordered (father first, then mother, then children)
	
	my $fam = $ged->add_family;
	
	#TODO: mandens faster, konens moder, moster faster etc.
	#TODO: foster children
	
	foreach my $i (@{$houses{$h}}) {
		if ($i->{'marital'} =~ /(^Gift|enke)/i) {
			if ($i->{'sex'} eq "M") {
				if ($fam->husband) { $fam = $ged->add_family; }
				$fam->add_husband($ged->resolve_xref($i->{'xref'}));
			} elsif ($i->{'sex'} eq "F") {
				if ($fam->wife) { $fam = $ged->add_family; }
				$fam->add_wife($ged->resolve_xref($i->{'xref'}));
			}
		} elsif ($i->{'position'} =~ /(b[aø]rn|datter|søn)/i) {
			$fam->add_child($ged->resolve_xref($i->{'xref'}));
		}
	}
}

foreach my $family ($ged->families) {
	print "Verifying family: ".$family->xref."\n";
	
	my @husbands = $family->husband;
	my @wives = $family->wife;
	
	if (@husbands > 1) {
		print "  too many husbands (".@husbands.")\n";
		
	}
	if (@wives > 1) {
		print "  too many wives (".@wives.")\n";
	}
}

#print Dumper \%houses;

#WRITE OUTPUT

print "\nWriting output...";

$ged->order($ged->sort_sub);

$ged->write(substr($kipfile,0,5).".ged", 0);

print "Done.\n";

__END__

__DATA__

Some notes:

The colums that were in the original CSV:

	'KIPnr'							# census transciption ID
	'Løbenr i indtastning'			# serial
	'Husstands/familienr.'			# household number
	'Antal familier/hus'			# number of families per household

*	'Kildenavn'						# name of individual

*	'Køn'							# gender

*	'Fødeår'						# birth year
*	'Født kildedato'				# birth date
*	'Alder Kbhv.'					# age (copenhagen census)
*	'Alder'							# age
*	'Kildefødested'					# birth place
*	'Fødested kun Frb. og Kbhv.'	# birth place (copenhagen census)

*	'Kildestednavn'					# place
	'Adresse 1916'
	'Adresse'
	'Flyttet til kommunen'
	'Matr.nr./Adresse'
	'Matrikel 1916'
	'Matrikel'
	'Sidste bopæl/oph.sted'
	'Midlertidig opholdssted'
	'Midlertidigt nærværende'

*	'Civilstand'					# civil position (married, widow, etc)
	'Antal ægteskab'
	'Hvornår æf. død'
	'Hvornår gift'
	'Nr. ægteskab'
	'Stilling i husstanden'
	'Antal levende børn'
	'Antal døde børn'

	'Kilde erhvervskommune'
	'Kilde erhvervssted'
	'Kildeerhverv'
	'Arbejdsplads'
	'Arbejdssted'
	'Indkomst'
	'Formue'
	'Stats skat'
	'Kommune skat'

	'Trossamfund'

	'Kommentar'

	'Handicaps'
	'Kildehenvisning'
	'Transportmiddel'
	'Boligtælling'