loncom/metadata_database/parse_activity_log.pl - diff

Return to parse_activity_log.pl CVS log

Up to [LON-CAPA] / loncom / metadata_database

Diff for /loncom/metadata_database/parse_activity_log.pl between versions 1.2 and 1.7

version 1.2, 2004/08/18 19:33:27	version 1.7, 2004/12/16 19:29:20
Line 26	Line 26
#	#
# http://www.lon-capa.org/	# http://www.lon-capa.org/
#	#
###############################################################################	#--------------------------------------------------------------------
#
# Expects
#
# ../key/$class.key - key file $username:$keynumber
# ../rawdata/$class.log - log file
# ../rawdata/$class.seq - sequence file
# ../data writable
# ------------------------------------------------------------------ Course log

#	#
# Exit codes	# Exit codes
# 0 Everything is okay	# 0 Everything is okay
Line 52 use lib '/home/httpd/lib/perl/Apache';	Line 43 use lib '/home/httpd/lib/perl/Apache';
use lonmysql();	use lonmysql();
use Time::HiRes();	use Time::HiRes();
use Getopt::Long();	use Getopt::Long();
	use IO::File;
	use File::Copy;
	use Fcntl qw(:flock);

#	#
# Determine parameters	# Determine parameters
my ($help,$course,$domain,$drop,$file,$time_run,$nocleanup,$log);	my ($help,$course,$domain,$drop,$file,$time_run,$nocleanup,$log,$backup);
&Getopt::Long::GetOptions( "course=s" => \$course,	&Getopt::Long::GetOptions( "course=s" => \$course,
"domain=s" => \$domain,	"domain=s" => \$domain,
	"backup" => \$backup,
"help" => \$help,	"help" => \$help,
"logfile=s" => \$file,	"logfile=s" => \$file,
"timerun" => \$time_run,	"timerun" => \$time_run,
Line 72 Process a lon-capa activity log into a d	Line 67 Process a lon-capa activity log into a d
Parameters:	Parameters:
course Required	course Required
domain Optional	domain Optional
	backup optional if present, backup the activity log file
	before processing it
drop optional if present, drop all course	drop optional if present, drop all course
specific activity log tables.	specific activity log tables.
file optional Specify the file to parse, including path	file optional Specify the file to parse, including path
Line 87 USAGE	Line 84 USAGE

##	##
## Set up timing code	## Set up timing code
##
my $time_this = \&nothing;	my $time_this = \&nothing;
if ($time_run) {	if ($time_run) {
$time_this = \&time_action;	$time_this = \&time_action;
Line 95 if ($time_run) {	Line 91 if ($time_run) {
my $initial_time = Time::HiRes::time;	my $initial_time = Time::HiRes::time;

##	##
## Set up logging code	## Read in configuration parameters
##	##
	my %perlvar;
	&initialize_configuration();
	if (! defined($domain) \|\| $domain eq '') {
	$domain = $perlvar{'lonDefDomain'};
	}
	&update_process_name($course.'@'.$domain);

	##
	## Set up logging code
my $logthis = \&nothing;	my $logthis = \&nothing;
if ($log) {	if ($log) {
my $logfile = "/tmp/parse_activity_log.log.".time;	my $logfile = $perlvar{'lonDaemons'}.'/tmp/parse_activity_log.log.'.time;
print STDERR "$0: logging to $logfile".$/;	print STDERR "$0: logging to $logfile".$/;
if (! open(LOGFILE,">$logfile")) {	if (! open(LOGFILE,">$logfile")) {
die "Unable to open $logfile for writing. Run aborted.";	die "Unable to open $logfile for writing. Run aborted.";
Line 107 if ($log) {	Line 112 if ($log) {
$logthis = \&log_to_file;	$logthis = \&log_to_file;
}	}
}	}
##
## Read in configuration parameters
##
my %perlvar;
&initialize_configuration();
if (! defined($domain) \|\| $domain eq '') {
$domain = $perlvar{'lonDefDomain'};
}
&update_process_name($course.'@'.$domain);

##	##
## Determine filenames	## Determine filenames
##	##
my $sourcefilename; # activity log data	my $sourcefilename; # activity log data
my $newfilename; # $sourcefilename will be renamed to this	my $newfilename; # $sourcefilename will be renamed to this
my $sql_filename; # the mysql backup data file name.	my $gz_sql_filename; # the gzipped mysql backup data file name.
	my $error_filename; # Errors in parsing the activity log will be written here
if ($file) {	if ($file) {
$sourcefilename = $file;	$sourcefilename = $file;
} else {	} else {
$sourcefilename = &get_filename($course,$domain);	$sourcefilename = &get_filename($course,$domain);
}	}
$sql_filename = $sourcefilename;	my $sql_filename = $sourcefilename;
$sql_filename =~ s\|[^/]*$\|activity.log.sql\|;	$sql_filename =~ s\|[^/]*$\|activity.log.sql\|;
	$gz_sql_filename = $sql_filename.'.gz';
	$error_filename = $sourcefilename;
	$error_filename =~ s\|[^/]*$\|activity.log.errors\|;
	$logthis->('Beginning logging '.time);


	#
	# Wait for a lock on the lockfile to avoid collisions
	my $lockfilename = $sourcefilename.'.lock';
	open(LOCKFILE,'>'.$lockfilename);
	flock(LOCKFILE,LOCK_EX) \|\| die("Unable to lock $lockfilename. Aborting".$/);

##	##
## There will only be a $newfilename file if a copy of this program is already	## There will only be a $newfilename file if a copy of this program is already
Line 137 $sql_filename =~ s\|[^/]*$\|activity.log.s	Line 146 $sql_filename =~ s\|[^/]*$\|activity.log.s
my $newfilename = $sourcefilename.'.processing';	my $newfilename = $sourcefilename.'.processing';
if (-e $newfilename) {	if (-e $newfilename) {
warn "$newfilename exists";	warn "$newfilename exists";
$logthis->($newfilename.' exists');	$logthis->($newfilename.' exists, so I cannot work on it.');
exit 2;	exit 2;
}	}

if (-e $sourcefilename) {	if (-e $sourcefilename) {
	$logthis->('renaming '.$sourcefilename.' to '.$newfilename);
rename($sourcefilename,$newfilename);	rename($sourcefilename,$newfilename);
	Copy($newfilename,$newfilename.'.'.time) if ($backup);
$logthis->("renamed $sourcefilename to $newfilename");	$logthis->("renamed $sourcefilename to $newfilename");
	} else {
	my $command = 'touch '.$newfilename;
	$logthis->($command);
	system($command);
	$logthis->('touch was completed');
}	}

	close(LOCKFILE);

##	##
## Table definitions	## Table definitions
##	##
Line 181 my $res_table_def =	Line 199 my $res_table_def =
'PRIMARY KEY' => ['res_id'],	'PRIMARY KEY' => ['res_id'],
};	};

my $action_table = $prefix.'actions';	#my $action_table = $prefix.'actions';
my $action_table_def =	#my $action_table_def =
{ id => $action_table,	#{ id => $action_table,
permanent => 'no',	# permanent => 'no',
columns => [{ name => 'action_id',	# columns => [{ name => 'action_id',
type => 'MEDIUMINT UNSIGNED',	# type => 'MEDIUMINT UNSIGNED',
restrictions => 'NOT NULL',	# restrictions => 'NOT NULL',
auto_inc => 'yes', },	# auto_inc => 'yes', },
{ name => 'action',	# { name => 'action',
type => 'VARCHAR(100)',	# type => 'VARCHAR(100)',
restrictions => 'NOT NULL'},	# restrictions => 'NOT NULL'},
],	# ],
'PRIMARY KEY' => ['action_id',],	# 'PRIMARY KEY' => ['action_id',],
};	#};

my $machine_table = $prefix.'machine_table';	my $machine_table = $prefix.'machine_table';
my $machine_table_def =	my $machine_table_def =
Line 225 my $activity_table_def =	Line 243 my $activity_table_def =
{ name => 'student_id',	{ name => 'student_id',
type => 'MEDIUMINT UNSIGNED',	type => 'MEDIUMINT UNSIGNED',
restrictions => 'NOT NULL',},	restrictions => 'NOT NULL',},
{ name => 'action_id',	{ name => 'action',
type => 'MEDIUMINT UNSIGNED',	type => 'VARCHAR(10)',
restrictions => 'NOT NULL',},	restrictions => 'NOT NULL',},
{ name => 'idx', # This is here in case a student	{ name => 'idx', # This is here in case a student
type => 'MEDIUMINT UNSIGNED', # has multiple submissions during	type => 'MEDIUMINT UNSIGNED', # has multiple submissions during
Line 238 my $activity_table_def =	Line 256 my $activity_table_def =
{ name => 'action_values',	{ name => 'action_values',
type => 'MEDIUMTEXT', },	type => 'MEDIUMTEXT', },
],	],
'PRIMARY KEY' => ['res_id','time','student_id','action_id','idx'],	'PRIMARY KEY' => ['time','student_id','res_id','idx'],
	'KEY' => [{columns => ['student_id']},
	{columns => ['time']},],
};	};
my @Activity_Tables = ($student_table_def,$res_table_def,	my @Activity_Table = ($activity_table_def);
$action_table_def,$machine_table_def,
$activity_table_def);	#my @ID_Tables = ($student_table_def,$res_table_def,
	# $action_table_def,$machine_table_def);

	my @ID_Tables = ($student_table_def,$res_table_def,$machine_table_def);


##	##
## End of table definitions	## End of table definitions
##	##

#	#
	$logthis->('Connectiong to mysql');
&Apache::lonmysql::set_mysql_user_and_password($perlvar{'lonSqlUser'},	&Apache::lonmysql::set_mysql_user_and_password($perlvar{'lonSqlUser'},
$perlvar{'lonSqlAccess'});	$perlvar{'lonSqlAccess'});
if (!&Apache::lonmysql::verify_sql_connection()) {	if (!&Apache::lonmysql::verify_sql_connection()) {
Line 258 if (!&Apache::lonmysql::verify_sql_conne	Line 282 if (!&Apache::lonmysql::verify_sql_conne
exit 3;	exit 3;
}	}

	$logthis->('SQL connection is up');

if ($drop) { &drop_tables(); $logthis->('dropped tables'); }	if ($drop) { &drop_tables(); $logthis->('dropped tables'); }
if (-e $sql_filename) {
$logthis->('reading in from '.$sql_filename);	if (-s $gz_sql_filename) {
# if ANY one of the tables does not exist, load the tables from the	# if ANY one of the tables does not exist, load the tables from the
# backup.	# backup.
my @Current_Tables = &Apache::lonmysql::tables_in_db();	my @Current_Tables = &Apache::lonmysql::tables_in_db();
my %Found;	my %Found;
foreach my $tablename (@Current_Tables) {	foreach my $tablename (@Current_Tables) {
foreach my $table (@Activity_Tables) {	foreach my $table (@Activity_Table,@ID_Tables) {
if ($tablename eq $table->{'id'}) {	if ($tablename eq $table->{'id'}) {
$Found{$tablename}++;	$Found{$tablename}++;
}	}
}	}
}	}
foreach my $table (@Activity_Tables) {	foreach my $table (@Activity_Table,@ID_Tables) {
if (! $Found{$table->{'id'}}) {	if (! $Found{$table->{'id'}}) {
$time_this->();	$time_this->();
&load_backup_tables($sql_filename);	&load_backup_tables($gz_sql_filename);
$time_this->('load backup tables');	$time_this->('load backup tables');
last;	last;
}	}
}	}
}	}

	##
	## Ensure the tables we need exist
# create_tables does not complain if the tables already exist	# create_tables does not complain if the tables already exist
	$logthis->('creating tables');
if (! &create_tables()) {	if (! &create_tables()) {
warn "Unable to create tables";	warn "Unable to create tables";
$logthis->('Unable to create tables');	$logthis->('Unable to create tables');
exit 4;	exit 4;
}	}

	##
	## Read the ids used for various tables
$logthis->('reading id tables');	$logthis->('reading id tables');
&read_id_tables();	&read_id_tables();
$logthis->('finished reading id tables');	$logthis->('finished reading id tables');

##	##
## Do the main bit of work	## Set up the errors file
if (-e $newfilename) {	my $error_fh = IO::File->new(">>$error_filename");
my $result = &process_courselog($newfilename);
	##
	## Parse the course log
	$logthis->('processing course log');
	if (-s $newfilename) {
	my $result = &process_courselog($newfilename,$error_fh);
if (! defined($result)) {	if (! defined($result)) {
# Something went wrong along the way...	# Something went wrong along the way...
$logthis->('process_courselog returned undef');	$logthis->('process_courselog returned undef');
exit 5;	exit 5;
} elsif ($result > 0) {	} elsif ($result > 0) {
$time_this->();	$time_this->();
$logthis->('process_courselog returned '.$result.' backup up tables');	$logthis->('process_courselog returned '.$result.' backing up tables');
&backup_tables($sql_filename);	&backup_tables($gz_sql_filename);
$time_this->('write backup tables');	$time_this->('write backup tables');
}	}
}	}
	close($error_fh);

##	##
## Clean up the filesystem	## Clean up the filesystem
##
&Apache::lonmysql::disconnect_from_db();	&Apache::lonmysql::disconnect_from_db();
unlink($newfilename) if (! $nocleanup);	unlink($newfilename) if (-e $newfilename && ! $nocleanup);

	##
	## Print timing data
	$logthis->('printing timing data');
if ($time_run) {	if ($time_run) {
my $elapsed_time = Time::HiRes::time - $initial_time;	my $elapsed_time = Time::HiRes::time - $initial_time;
print "Overall time: ".$elapsed_time.$/;	print "Overall time: ".$elapsed_time.$/;
Line 339 exit 0; # Everything is okay, so end h	Line 378 exit 0; # Everything is okay, so end h
#	#
# Returns the number of lines in the activity.log file that were processed.	# Returns the number of lines in the activity.log file that were processed.
sub process_courselog {	sub process_courselog {
my ($inputfile) = @_;	my ($inputfile,$error_fh) = @_;
if (! open(IN,$inputfile)) {	if (! open(IN,$inputfile)) {
warn "Unable to open '$inputfile' for reading";	warn "Unable to open '$inputfile' for reading";
$logthis->("Unable to open '$inputfile' for reading");	$logthis->("Unable to open '$inputfile' for reading");
Line 384 sub process_courselog {	Line 423 sub process_courselog {
if (! defined($res) \|\| $res =~ /^\s*$/) {	if (! defined($res) \|\| $res =~ /^\s*$/) {
$res = '/adm/roles';	$res = '/adm/roles';
$action = 'LOGIN';	$action = 'LOGIN';
# $warningflag .= 'res';
}	}
if ($res =~ m\|^/prtspool/\|) {	if ($res =~ m\|^/prtspool/\|) {
$res = '/prtspool/';	$res = '/prtspool/';
}	}
if (! defined($action) \|\| $action eq '') {	if (! defined($action) \|\| $action eq '') {
$action = 'VIEW';	$action = 'VIEW';
# $warningflag .= 'action';
}	}
if ($action !~ /^(LOGIN\|VIEW\|POST\|CSTORE\|STORE)$/) {	if ($action !~ /^(LOGIN\|VIEW\|POST\|CSTORE\|STORE)$/) {
$warningflag .= 'action';	$warningflag .= 'action';
	print $error_fh 'full log entry:'.$log.$/;
	print $error_fh 'error on chunk:'.$chunk.$/;
	$logthis->('(action) Unable to parse '.$/.$chunk.$/.
	'got '.
	'time = '.$time.$/.
	'res = '.$res.$/.
	'uname= '.$uname.$/.
	'udom = '.$udom.$/.
	'action='.$action.$/.
	'@values = '.join(':',@values));
	next; #skip it if we cannot understand what is happening.
}	}
if (! defined($student) \|\| $student eq ':') {	if (! defined($student) \|\| $student eq ':') {
$student = 'unknown';	$student = 'unknown';
Line 415 sub process_courselog {	Line 463 sub process_courselog {
#	#
$time_this->('split_and_error_check');	$time_this->('split_and_error_check');
my $student_id = &get_id($student_table,'student',$student);	my $student_id = &get_id($student_table,'student',$student);
my $res_id = &get_id($res_table,'resource',$res);	my $res_id = &get_id($res_table,'resource',$res);
my $action_id = &get_id($action_table,'action',$action);	# my $action_id = &get_id($action_table,'action',$action);
my $sql_time = &Apache::lonmysql::sqltime($time);	my $sql_time = &Apache::lonmysql::sqltime($time);
#	#
if (! defined($student_id) \|\| $student_id eq '') {	if (! defined($student_id) \|\| $student_id eq '') {
$warningflag.='student_id';	$warningflag.='student_id';
Line 425 sub process_courselog {	Line 473 sub process_courselog {
if (! defined($res_id) \|\| $res_id eq '') {	if (! defined($res_id) \|\| $res_id eq '') {
$warningflag.='res_id';	$warningflag.='res_id';
}	}
if (! defined($action_id) \|\| $action_id eq '') {	# if (! defined($action_id) \|\| $action_id eq '') {
$warningflag.='action_id';	# $warningflag.='action_id';
}	# }
if ($warningflag ne '') {	if ($warningflag ne '') {
	print $error_fh 'full log entry:'.$log.$/;
	print $error_fh 'error on chunk:'.$chunk.$/;
$logthis->('warningflag ('.$warningflag.') on chunk '.	$logthis->('warningflag ('.$warningflag.') on chunk '.
$/.$chunk.$/.'prevchunk = '.$/.$prevchunk);	$/.$chunk.$/.'prevchunk = '.$/.$prevchunk);
$prevchunk .= $chunk;	$prevchunk .= $chunk;
Line 441 sub process_courselog {	Line 491 sub process_courselog {
my $row = [$res_id,	my $row = [$res_id,
qq{'$sql_time'},	qq{'$sql_time'},
$student_id,	$student_id,
$action_id,	"'".$action."'",
	# $action_id,
qq{''}, # idx	qq{''}, # idx
$machine_id,	$machine_id,
$values];	$values];
Line 540 sub outputtimes {	Line 591 sub outputtimes {
## Use mysqldump to store backups of the tables	## Use mysqldump to store backups of the tables
##	##
sub backup_tables {	sub backup_tables {
my ($sql_filename) = @_;	my ($gz_sql_filename) = @_;
my $command = qq{mysqldump --opt loncapa };	my $command = qq{mysqldump --opt loncapa };

foreach my $table (@Activity_Tables) {	foreach my $table (@ID_Tables,@Activity_Table) {
my $tablename = $table->{'id'};	my $tablename = $table->{'id'};
$command .= $tablename.' ';	$command .= $tablename.' ';
}	}
$command .= '>'.$sql_filename;	$command .= '\| gzip >'.$gz_sql_filename;
$logthis->($command);	$logthis->($command);
system($command);	system($command);
}	}
Line 556 sub backup_tables {	Line 607 sub backup_tables {
## Load in mysqldumped files	## Load in mysqldumped files
##	##
sub load_backup_tables {	sub load_backup_tables {
my ($sql_filename) = @_;	my ($gz_sql_filename) = @_;
return undef if (! -e $sql_filename);	if (-s $gz_sql_filename) {
# Check for .my.cnf	&logthis('loading data from gzipped sql file');
my $command = 'mysql -e "SOURCE '.$sql_filename.'" loncapa';	my $command='gzip -dc activity.log.sql.gz \| mysql --database=loncapa';
$logthis->('loading previously saved sql table'.$/.$command);	system($command);
system($command);	$logthis->('finished loading gzipped data');;
	} else {
	return undef;
	}
}	}

##	##
Line 587 sub get_filename {	Line 641 sub get_filename {
}	}

sub create_tables {	sub create_tables {
foreach my $table (@Activity_Tables) {	foreach my $table (@ID_Tables,@Activity_Table) {
my $table_id = &Apache::lonmysql::create_table($table);	my $table_id = &Apache::lonmysql::create_table($table);
	# print STDERR "Unable to create table ".$table->{'id'}.$/;
	# print STDERR join($/,&Apache::lonmysql::build_table_creation_request($table)).$/;
if (! defined($table_id)) {	if (! defined($table_id)) {
warn "Unable to create table ".$table->{'id'}.$/;	warn "Unable to create table ".$table->{'id'}.$/;
warn &Apache::lonmysql::build_table_creation_request($table).$/;	warn join($/,&Apache::lonmysql::build_table_creation_request($table)).$/;
return 0;	return 0;
}	}
}	}
Line 599 sub create_tables {	Line 655 sub create_tables {
}	}

sub drop_tables {	sub drop_tables {
foreach my $table (@Activity_Tables) {	foreach my $table (@ID_Tables,@Activity_Table) {
my $table_id = $table->{'id'};	my $table_id = $table->{'id'};
&Apache::lonmysql::drop_table($table_id);	&Apache::lonmysql::drop_table($table_id);
}	}
Line 616 sub drop_tables {	Line 672 sub drop_tables {
my %IDs;	my %IDs;

sub read_id_tables {	sub read_id_tables {
foreach my $table (@Activity_Tables) {	foreach my $table (@ID_Tables) {
my @Data = &Apache::lonmysql::get_rows($table->{'id'});	my @Data = &Apache::lonmysql::get_rows($table->{'id'});
	my $count = 0;
foreach my $row (@Data) {	foreach my $row (@Data) {
$IDs{$table->{'id'}}->{$row->[1]} = $row->[0];	$IDs{$table->{'id'}}->{$row->[1]} = $row->[0];
}	}
}	}
	return;
}	}

sub get_id {	sub get_id {

FreeBSD-CVSweb <freebsd-cvsweb@FreeBSD.org>

Removed from v.1.2
changed lines
	Added in v.1.7