Processing large data files.

Perl crashes when using the included script for large data files. How 
can I prevent this from happening?

Kindest Regards,

--
Bill Stephenson


Sample data source is a Delimited text file with the following fields:

FMT:STORE:TIKEY:TITLE:QTY:DC:COMPANY:ZONE

DV:043712:20882903:DVD-ALL THE KING'S M('06/WS/2D:8.00000:ECDC:Hlyw:4
DV:043712:20883071:DVD-ALTERED (2006/WS):8.00000:ECDC:Hlyw:4
DV:043712:20883070:DVD-AMERICAN PIE 5-NAKE(UR/WS:32.00000:ECDC:Hlyw:4
DV:043712:20883253:DVD-BROOKLYN LOBSTER (WS):2.00000:ECDC:Hlyw:4
DV:043704:20882903:DVD-ALL THE KING'S M('06/WS/2D:16.00000:WCDC:Hlyw:4
DV:043704:20883071:DVD-ALTERED (2006/WS):8.00000:WCDC:Hlyw:4
DV:043693:20882903:DVD-ALL THE KING'S M('06/WS/2D:24.00000:ECDC:Hlyw:4
DV:043693:20883071:DVD-ALTERED (2006/WS):8.00000:ECDC:Hlyw:4
DV:043693:20883070:DVD-AMERICAN PIE 5-NAKE(UR/WS:24.00000:ECDC:Hlyw:4
DV:043693:20883092:DVD-ZOMBIE NATION (2006/WS):4.00000:ECDC:Hlyw:4
DV:043670:20882903:DVD-ALL THE KING'S M('06/WS/2D:8.00000:ECDC:Hlyw:4

<code>

#!/usr/bin/perl

use strict;
use warnings;

# set-up our variable....

my $source = "dataSource.txt";
my $destination = "testOUTPUT.txt";

my $fmt = "";
my $storeno = "";
my $co = "";
my $dc = "";
my $zone = "";
my $store = "";
my $tikey = "";
my $title = "";
my $qty = "";
my $text1 = "";
my $text2 = "";
my $output = "";

my @stores = "";


# start the work...

open my $IN, $source or die "Can't read source file $source: $!\n";

# put all the stores into an array (@stores)...
my @lines = <$IN>;

foreach my $line(@lines) {

	($fmt, $store, $tikey, $title, $qty, $dc, $co, $zone) = 
split(/:/,$line);
		
	push @stores, $store;
	
}


# remove duplicate stores from the list...
my %seen = ();
my @uniq = ();

foreach my $item (@stores) {

     unless ($seen{$item}) {
         # if we get here, we have not seen it before
         $seen{$item} = 1;
         push(@uniq, $item);
     }
}

# for each store in our @uniq store list we look for
# lines in our file that have a matching store

foreach my $uniqStore (@uniq) {

	foreach my $line(@lines) {
	
		($fmt, $store, $tikey, $title, $qty, $dc, $co, $zone) = 
split(/:/,$line);
	
# if they match we create our entry for the destination file....
		if ($store eq $uniqStore) {
		
			$text1= "Street Date $source \n". "Company $co \n". "Distribution 
Center $dc \n". "Zone $zone \n". "Store Number $store \n\n";
		
			$text2 .= "$tikey \t $title \t $qty\n";
			
			}
	
	 }
	
# then add it to the final $output.
	if ($text1 ne "") {
	$output .= 
"$text1$text2\n####################\n####################\n\n\f\n";
	$text2 = "";
	}
}

# and finally print all out

# print $output; #for debugging

open(my $DATA, "> $destination")
     or die (" Error 1.3 Couldn't file:: $destination for writing: \n");

print $DATA ($output);

close $DATA;