#!/usr/bin/perl -w
# This script (sophomorix-project) is maintained by Rüdiger Beck
# It is Free Software (License GPLv3)
# If you find errors, contact the author
# jeffbeck@web.de  jeffbeck@linuxmuster.net

# modules
use strict;
use Getopt::Long;
Getopt::Long::Configure ("bundling");
use Sophomorix::SophomorixConfig;
use List::MoreUtils qw(uniq);
use Time::Local;
use Net::LDAP;
use Data::Dumper;
$Data::Dumper::Indent = 1;
$Data::Dumper::Sortkeys = 1;
$Data::Dumper::Useqq = 1;
$Data::Dumper::Terse = 1; 
use JSON;

use File::Basename qw( basename
                       dirname
                     );   
use Sophomorix::SophomorixBase qw(
                                 print_line
                                 print_title
                                 log_script_start
                                 log_script_end
                                 log_script_exit
                                 check_options
                                 config_sophomorix_read
                                 result_sophomorix_init
                                 result_sophomorix_add
                                 result_sophomorix_check_exit
                                 result_sophomorix_print
                                 );
use Sophomorix::SophomorixSambaAD qw(
                                 AD_bind_admin
                                 AD_unbind_admin
                                 AD_group_list
                                 AD_object_search
                                 AD_get_AD_for_check
                                    );

my @arguments = @ARGV;


# ===========================================================================
# Optionen verarbeiten
# ==========================================================================

# Variablen für Optionen
$Conf::log_level=1;
my $help=0;
my $info=0;
my $json=0;

# default
my $datadir="/root/linuxmuster6";

# --restore-config-files
my $restore_config_files=0;

# --analyze
my $analyze=0;
my $classes_map="/root/sophomorix-vampire/sophomorix-vampire-classes.map";
my $users_to_migrate="/root/sophomorix-vampire/sophomorix-vampire-users-to-migrate.txt";
my $groups_to_migrate="/root/sophomorix-vampire/sophomorix-vampire-groups-to-migrate.txt";

# --create-class-script
my $create_class_script=0;
my $class_script_input="/data/root/sophomorix-dump-viewdumps/classdata_view.sql";
my $class_script_output="/root/sophomorix-vampire/sophomorix-vampire-classes.sh";
my $extrakurse="/data/etc/sophomorix/user/extrakurse.txt";

# --create-add-file
my $create_add_file=0;
my $sophomorix_add_input_sql="/data/root/sophomorix-dump-viewdumps/userdata_view.sql";
my $sophomorix_add_output="/root/sophomorix-vampire/sophomorix.add";

# --import-user-password-hashes
my $import_user_password_hashes=0;
my $import_user_pwd_ldif_template="/usr/share/sophomorix-vampire/ldif-pwdload.ldif.template";
my $import_user_pwd_ldif="/root/sophomorix-vampire/sophomorix-vampire-import-pwd.ldif";

# --create-project-script
my $create_project_script=0;
my $project_script_input_sql="/data/root/sophomorix-dump-viewdumps/projectdata_view.sql";
my $project_script_input="/data/root/sophomorix-dump.projects";
my $project_script_output="/root/sophomorix-vampire/sophomorix-vampire-projects.sh";

# --create-class-adminadd-script
my $create_class_adminadd_script=0;
my $class_adminadd_script_input="/data/root/sophomorix-dump-viewdumps/memberdata_view.sql";
my $class_adminadd_script_output="/root/sophomorix-vampire/sophomorix-vampire-classes-adminadd.sh";

############################################################
# rsync data
# --path-oldserver
my $path_oldserver="";

# --rsync-all-student-homes
my $rsync_all_student_homes=0;
# --rsync-student-home <student>
my $rsync_student_home="";

# --rsync-all-teacher-homes
my $rsync_all_teacher_homes=0;
# --rsync-teacher-home <teacher>
my $rsync_teacher_home="";

# --rsync-all-class-shares
my $rsync_all_class_shares=0;
# --rsync-class-share <class> 
my $rsync_class_share="";

# --rsync-all-project-shares
my $rsync_all_project_shares=0;
# --rsync-project-share <project>
my $rsync_project_share="";

# --rsync-linbo
my $rsync_linbo=0;

my $missig_files_error_count=0;
my $missig_files_warn_count=0;
my $overwriting_files_count=0;

system("mkdir -p /root/sophomorix-vampire");

# ===========================================================================
# config
# ==========================================================================
my $config="/usr/share/sophomorix-vampire/migration-6-to-7.conf";

# Parsen der Optionen
my $testopt=GetOptions(
           "help|h" => \$help,
           "info|i" => \$info,
           "analyze" => \$analyze,
           "restore-config-files" => \$restore_config_files,
           "datadir=s" => \$datadir,
           "create-add-file" => \$create_add_file,
           "import-user-password-hashes" => \$import_user_password_hashes,
           "create-project-script" => \$create_project_script,
           "create-class-script" => \$create_class_script,
           "create-class-adminadd-script" => \$create_class_adminadd_script,
           "path-oldserver=s" => \$path_oldserver,
           "rsync-all-student-homes" => \$rsync_all_student_homes,
           "rsync-student-home=s" => \$rsync_student_home,
           "rsync-all-teacher-homes" => \$rsync_all_teacher_homes,
           "rsync-teacher-home=s" => \$rsync_teacher_home,
           "rsync-all-class-shares" => \$rsync_all_class_shares,
           "rsync-class-share=s" => \$rsync_class_share,
           "rsync-all-project-shares" => \$rsync_all_project_shares,
           "rsync-project-share=s" => \$rsync_project_share,
           "rsync-linbo" => \$rsync_linbo,
           "verbose|v+" => \$Conf::log_level,
         );

# Prüfen, ob Optionen erkannt wurden
&check_options($testopt);

# modify paths
# --datadir
$sophomorix_add_input_sql=$datadir.$sophomorix_add_input_sql;
$project_script_input=$datadir.$project_script_input;
$project_script_input_sql=$datadir.$project_script_input_sql;
$class_script_input=$datadir.$class_script_input;
$extrakurse=$datadir.$extrakurse;
$class_adminadd_script_input=$datadir.$class_adminadd_script_input;

# --help
if ($help==1) {
   # Scriptname ermitteln
   my @list = split(/\//,$0);
   my $scriptname = pop @list;
   # Befehlbeschreibung
   print('
sophomorix-vampire imports users, classes, projects and devices from a linuxmuster.net 6.x server 

Options
  -h  / --help
  -v  / --verbose
  -vv / --verbose --verbose

Getting information:
  -i / --info
Configuring:
  --datadir /path/to/dir/sophomorix-dump

Dump data on old server V6:
1. Exporting user data on linuxmusternet6 server:
   sophomorix-dump
2. Save/Copy the following dir to new Server:
   /root/sophomorix-dump

Vampire data into the new server V7:
1.  Test if dump is complete, read INFO, ERRORS and WARNINGS:
    # sophomorix-vampire --datadir /path/to/dir/sophomorix-dump --analyze
2.  Create class script and run it:
    # sophomorix-vampire --datadir /path/to/dir/sophomorix-dump --create-class-script
    # /root/sophomorix-vampire/sophomorix-vampire-classes.sh
3.  Create file sophomorix.add and add these users:
    Allow simple passwords:
    # samba-tool domain passwordsettings set --complexity=off
    # samba-tool domain passwordsettings set --min-pwd-length=2
    Create the users:
    # sophomorix-vampire --datadir ../sophomorix4-migration-data --create-add-file
    # cp /root/sophomorix-vampire/sophomorix.add /var/lib/sophomorix/check-result/sophomorix.add
    # sophomorix-add -i (Read ERRORS and WARNINGS, fix them manually)
    # sophomorix-add (Add the users)
4.  Import the hashed passwords into AD (if you want users to keep their password) 
    # sophomorix-vampire --datadir /path/to/dir/sophomorix-dump --import-user-password-hashes
    Force complex passwords (which is the default):
    # samba-tool domain passwordsettings set --complexity=default
    # samba-tool domain passwordsettings set --min-pwd-length=default
5.  Create script to add administrators to classes and run it:
    # sophomorix-vampire --datadir /path/to/dir/sophomorix-dump --create-class-adminadd-script
    # /root/sophomorix-vampire/sophomorix-vampire-classes-adminadd.sh
6.  Create project script and run it:
    # sophomorix-vampire --datadir /path/to/dir/sophomorix-dump --create-project-script
    # /root/sophomorix-vampire/sophomorix-vampire-projects.sh
7.  Copy configuration files into new server:
    # sophomorix-vampire --datadir /path/to/dir/sophomorix-dump --restore-config-files
    # sophomorix-vampire --datadir /path/to/dir/sophomorix-dump --restore-config-files
    YES, run it twice!
8.  Do a sophomorix run to update utf8, webui-permissions and maybe more
    sophomorix-check
    sophomorix-add -i (should show no new users)
    sophomorix-update
    sophomorix-kill (this might delete some users that are overdue to be deleted)
9.  Import the workstations 
    # linuxmster-import-devices

10. To rsync data from the old server, do the following steps

    A) Mount the old server home somewhere (for example to /mnt), so you can see:
        /mnt/home/share
        /mnt/home/students
        /mnt/home/teachers
    and specify your mount directory as:  --path-oldserver /mnt

    B) Do some tests for a single student, teacher, class, project:
    # sophomorix-vampire --rsync-student-home <student>  --path-oldserver /mnt 
    # sophomorix-vampire --rsync-teacher-home <teacher>  --path-oldserver /mnt
    # sophomorix-vampire --rsync-class-share <class>     --path-oldserver /mnt
    # sophomorix-vampire --rsync-project-share <project> --path-oldserver /mnt

    C) Sync all data of students,teachers,classe,projects:
    # sophomorix-vampire --rsync-all-student-homes  --path-oldserver /mnt 
    # sophomorix-vampire --rsync-all-teacher-homes  --path-oldserver /mnt
    # sophomorix-vampire --rsync-all-class-shares   --path-oldserver /mnt
    # sophomorix-vampire --rsync-all-project-shares --path-oldserver /mnt

    D) Sync linbo data
    # sophomorix-vampire --rsync-linbo  --path-oldserver /mnt

1x. Manually do the rest (will be automated):
    Copy files in /etc/linuxmuster/sophomorix/user/mail/* to new server.

Getting information:
  -i / --info
Configuring:
  --datadir /path/to/migration/files
');
   print "\n";
   exit;
}

#&log_script_start(\@arguments,\%sophomorix_result,\%sophomorix_config);

##################################################
# Testing files in the dump
print "Testing files in ${datadir}:\n";
open(CONFIG,"<$config") || die "ERROR: $!";
while(<CONFIG>){
    if(/^\#/){ # # am Anfang bedeutet Kommentarzeile
	next;
    }
    chomp();
    my ($old_path,$new_path,$permissions,$may_must,$type)=split(/::/);
    my $old_path_dump=$datadir."/data".$old_path;
    if ($type eq "file" and not -f $old_path_dump){
        if ($may_must eq "must"){
            $missig_files_error_count++;
            print "ERROR: Missing file $old_path_dump\n";
        } elsif ($may_must eq "may"){
            $missig_files_warn_count++;
            print "WARNING: Missing file $old_path_dump\n";
        }
    }
    if ($type eq "dir" and not -d $old_path_dump){
        if ($may_must eq "must"){
            $missig_files_error_count++;
            print "ERROR: Missing directory $old_path_dump\n"; 
        } elsif ($may_must eq "may") {
            $missig_files_warn_count++;
            print "WARNING: Missing directory $old_path_dump\n"; 
        }
    }
    if ($new_path eq "---"){
        # files that must exist
        print "   * Exists: $old_path_dump\n";
    } else {
        # files to install
        print "   * Exists: $old_path_dump\n";
        print "     Target: $new_path\n";
        if (-e $new_path){
            print "     Target Exists WARNING: $new_path\n";
            $overwriting_files_count++;
        }
        # --restore-config-files
        if ($restore_config_files==1){ 
            my $parent_dir = dirname($new_path);
            my $mkdir_command="mkdir -p $parent_dir";
            print "     $mkdir_command\n";
            system($mkdir_command);
            my $command="cp $old_path_dump $new_path";
            print "     $command\n";
            if (-f $old_path_dump){
                system($command);
            }
            my $permission_command="chmod $permissions $new_path";
            print "     $permission_command\n";
            if (-f $new_path){
                system($permission_command);
            }
        }
    }

}
close(CONFIG);




##################################################
# print ERRORS and WARNINGS
print "\n";
if ($missig_files_warn_count > 0){
    print "WARNING:  $missig_files_warn_count file(s) are missing in the dump. Might be an unused feature. Please check!\n";
}
if ($missig_files_error_count > 0){
    print "ERROR:    $missig_files_error_count file(s) are missing in the dump! You don't have all data in the dump!\n";
}
if ($restore_config_files==1){
    my %classes_map=&read_classes_map();
    print "$overwriting_files_count file(s) were overwritten!\n";
    &teachers_rewrite("/etc/linuxmuster/sophomorix/default-school/teachers.csv.tmp",
                      "/etc/linuxmuster/sophomorix/default-school/teachers.csv");
    &devices_rewrite(\%classes_map,
                     "/etc/linuxmuster/sophomorix/default-school/devices.csv.orig",
                     "/etc/linuxmuster/sophomorix/default-school/devices.csv",
                     "/etc/linuxmuster/sophomorix/default-school/devices.csv.tmp");
    &subnets_rewrite("/etc/linuxmuster/subnets.csv.orig",
                     "/etc/linuxmuster/subnets.csv",
                     "/etc/linuxmuster/subnets.csv.tmp");
    &extrastudents_rewrite("/etc/linuxmuster/sophomorix/default-school/extrastudents.csv.orig",
  		           "/etc/linuxmuster/sophomorix/default-school/extrastudents.csv");
    &killog_rewrite("/var/log/sophomorix/user-modify-archive.log.migrated.orig",
		    "/var/log/sophomorix/user-modify-archive.log.migrated");
}
print "\n";



# --info
if ($info==1){
    
    exit;
}



# --analyze
if ($analyze==1){
    my %login=();
    my %groupnames=();
    print "Analyzing ...\n";
    {
    # USERNAMES
    print "   * Fetching loginnames of users ...\n";
    my $line_number=0;
    open(USERMIGRATE,">$users_to_migrate") || die "ERROR: $!";
    open(GROUPMIGRATE,">$groups_to_migrate") || die "ERROR: $!";
    open(USER,"<$sophomorix_add_input_sql") || die "ERROR: $!";
    while(<USER>){
        my $line=$_;
        $line_number++;
        my @attrs=split(/\|/,$line);
        my $count=0;
        #print "Line $line_number:\n";
        if ($line_number==1){
            # skip;
        } else {
            my $login=$attrs[2];
            $login=~s/^\s+//g;# remove leading whitespace
            $login=~s/\s+$//g;# remove trailing whitespace

            my $uidnumber=$attrs[1];
            $uidnumber=~s/^\s+//g;# remove leading whitespace
            $uidnumber=~s/\s+$//g;# remove trailing whitespace

            my $gidnumber=$attrs[3];
            $gidnumber=~s/^\s+//g;# remove leading whitespace
            $gidnumber=~s/\s+$//g;# remove trailing whitespace

            my $gecos=$attrs[7];
            $gecos=~s/^\s+//g;# remove leading whitespace
            $gecos=~s/\s+$//g;# remove trailing whitespace

            my $status=$attrs[52];
            $status=~s/^\s+//g;# remove leading whitespace
            $status=~s/\s+$//g;# remove trailing whitespace

            my $samba_account_flag=$attrs[21];
            $samba_account_flag=~s/^\s+//g;# remove leading whitespace
            $samba_account_flag=~s/\s+$//g;# remove trailing whitespace
            if ($status eq "P" or
                $status eq "U" or
                $status eq "E" or
                $status eq "A" or
                $status eq "F" or
                $status eq "D" or
                $status eq "T"
                ){
                # this are stati that lead to an account on a lmn7 server
                if ( $samba_account_flag=~m/U/ and $gecos ne "ExamAccount"){
                    # U: User account
                    # W: Workstation Account
                    print USERMIGRATE $login."::".$uidnumber."::".$status."::".$samba_account_flag."::".$gidnumber."::\n";
                }
            } elsif ($status eq "R" or
                     $status eq "K"
                ){
                # this are stati of users that do not need to be added
            } else {
                # should be a bug
                print "Unknown status: <$login> --> <$status>\n";
            }
            $login{$login}="user";
        }
    }
    }
    {
    print "   * Fetching classnames ...\n";
    my $line_number=0;

    # GROUPNAMES
    open(MAP,">$classes_map") || die "ERROR: $!";
    open(USER,"<$class_script_input") || die "ERROR: $!";
    while(<USER>){
        my $line=$_;
        $line_number++;
        my @attrs=split(/\|/,$line);
        my $count=0;
        #print "Line $line_number:\n";
        if ($line_number==1){
            # skip;
        } else {
            my $group=$attrs[8];
            my $type=$attrs[5];
            $group=~s/^\s+//g;# remove leading whitespace
            $group=~s/\s+$//g;# remove trailing whitespace
            $type=~s/^\s+//g;# remove leading whitespace
            $type=~s/\s+$//g;# remove trailing whitespace
            #print "<$group>\n";
            $groupnames{$group}=$type;
            print GROUPMIGRATE $group."::".$type."::\n";
        }
    }
    }


    # OUTPUT
    # create file/output lines (and order them) 
    my @file_lines=();
    foreach my $old_group (keys %groupnames) {
        my $value = $groupnames{$old_group};
        if (exists $login{$old_group}){
            my $new_group=$old_group."-grp";
            my $entry=$old_group."::".$new_group."::".$value."::\n";
            push @file_lines, $entry;
        }
    }
    @file_lines = sort @file_lines;

    # printout
    print "\n";
    print "INFO: The following groupnames will be converted (old (old type of group) ---> new):\n";
    foreach my $line (@file_lines){
        print MAP $line;
        my ($old_group,$new_group,$type)=split(/::/,$line);
        printf "  %-30s---> %-26s\n",$old_group." (".$type.")",$new_group;
    }

    print "\n";
    print "The group renaming is saved in the file:\n";
    print "    $classes_map\n";
    print "You can edit the new group name in this file, but make sure to use an unused object name\n";
    print "\n";
    close(MAP);
    close(USER);
    close(USERMIGRATE);
    close(GROUPMIGRATE);
    #print Dumper (\%groupnames);
    exit;
}



##################################################
# --create-class-script
# creating /root/sophomorix-vampire-classes.sh
if ($create_class_script==1){
    my %classes_map=&read_classes_map();
    my @command_list_create=();
    my @command_list_modify=();
    my @skipped_lines=();
    my $line_number=0;
    my %line=();
    my %tags=();
    my %extraclass=();

    print "\n";
    print "Analyzing $extrakurse\n";
    open(EXTRAKURSE,"<$extrakurse") || die "ERROR: $!";
    while(<EXTRAKURSE>){
	my $line=$_;
        chomp($line);
        $line=~s/^\s+//g;# remove leading whitespace
        if(/^\#/){
             # ignore comment lines
        } elsif ($line eq ""){
             # ignore empty lines
        } else {
            print "$line\n";
            my ($course,
            $course_basename,
            $course_count,
            $course_date,
            $course_comment,
            $course_password,
            $course_enddate)=split(/;/,$line);
            my ($end_day, $end_month, $end_year)=split(/\./, $course_enddate);
            $extraclass{$course}="extraclass";
            print "    EXTRCLASS: <$course>\n";
        }
    }
    close(EXTRAKURSE);

    print "\n";
    print "Creating $class_script_output\n";
    open(CLASS,"<$class_script_input") || die "ERROR: $!";
    open(CLASSSCRIPT,">$class_script_output") || die "ERROR: $!";
    while(<CLASS>){
        my $line=$_;
        chomp($line);
        $line_number++;
        my @attrs=split(/\|/,$line);
        my $count=0;
        if ($line_number==1){
            foreach my $attr (@attrs){
                $attr=~s/^\s+//g;# remove leading whitespace
                $attr=~s/\s+$//g;# remove trailing whitespace
                print "   $count: * >$attr<\n";
                $tags{$count}=$attr;
                $count++;
            }
        } else {
            my %line=();
            my $skip_line=0;
            my $join;
            my $quota;
            my $mailquota;
            my $maillist;
            my $mailalias;
            foreach my $attr (@attrs){
                $attr=~s/^\s+//g;# remove leading whitespace
                $attr=~s/\s+$//g;# remove trailing whitespace
                #print "   $count: $tags{$count}: >$attr<\n";
                $line{$tags{$count}}=$attr;
                $count++;
            }

            # select teacher,adminclasses and hiddenclasses            
            if ($line{"type"} eq "teacher"){
                $join=" --nojoin";
            } elsif ($line{"type"} eq "adminclass"){
                $join=" --join";
            } elsif ($line{"type"} eq "hiddenclass"){
                $join=" --join";
            } elsif ($line{"type"} eq "room"){
                $skip_line=1;
                next;
            } elsif ($line{"type"} eq "project"){
                $skip_line=1;
                next;
            } else {
                # others are skipped
	        print "SKIPPED: $line\n";
                push @skipped_lines, $line;
                $skip_line=1;
                next;
            }


            # creating commands
            ##################################################
            my $class; # the mapped name
            if (exists $classes_map{$line{'gid'}}){
                $class=$classes_map{$line{'gid'}};
            } else {
                $class=$line{'gid'};
            }
            # adminclass OR extraclass
            my $class_type="--adminclass";
            my $description=$line{'gid'}." migrated"; # use unmapped name
            if (exists $extraclass{$class}){
                $class_type="--extraclass";
                $description="extrakurs ".$line{'gid'}." migrated"; # use unmapped name
            }
            my $command_create="sophomorix-class --skip-school-creation --create -c $class --gidnumber-migrate $line{'gidnumber'} $class_type";
            my $command_modify="sophomorix-class --class $class";

            # --description
            $command_modify = $command_modify." --description \"$description\"";

            # --join/--nojoin
            $command_modify = $command_modify.$join;

#            # --quota
#            if ($line{'quota'} eq "quota"){
#                $quota=" --quota \"---\"";
#            } else {
#                $quota=" --quota $line{'quota'}";
#            }
#            $command_modify = $command_modify.$quota;

#            # --mailquota
#            if ($line{'mailquota'} eq "-1"){
#                $mailquota=" --mailquota \"-1\"";
#            } else {
#                $mailquota=" --mailquota $line{'mailquota'}";
#            }
#            $command_modify = $command_modify.$mailquota;

            # --mailalias
            if ($line{'mailalias'} eq "t"){
                $mailalias=" --mailalias";
            } else {
                $mailalias=" --nomailalias";
            }
            $command_modify = $command_modify.$mailalias;

            # --maillist
            if ($line{'maillist'} eq "t"){
                $maillist=" --maillist";
            } else {
                $maillist=" --nomaillist";
            }
            $command_modify = $command_modify.$maillist;


            if ($skip_line==0){
                #print "$command_create\n";
                #print "$command_modify\n";
                push @command_list_create, $command_create;
                push @command_list_modify, $command_modify;
	        #print "LINE: $line";
            }
        }
    }

    # sorting lines and assembling them
    @command_list_create = sort @command_list_create;
    @command_list_modify = sort @command_list_modify;
    my @command_list=("# Create Classes",
                      @command_list_create,
                     "# Modify Classes", 
                      @command_list_modify);
    my $count_command=0;
    my $max_count_command=$#command_list+1;
    foreach my $command (@command_list){
        $count_command++;
        print CLASSSCRIPT "#!/bin/sh\n";
        print CLASSSCRIPT "echo \"\"\n";
        print CLASSSCRIPT "echo \"progress  $count_command/$max_count_command\"\n";
        print CLASSSCRIPT "echo \"\"\n";
        print CLASSSCRIPT "$command\n";
    }
    close(CLASS);
    close(CLASSSCRIPT);
    system("chmod 755 $class_script_output");
}



##################################################
# --create-add-file
# creating sophomorix.add
if ($create_add_file==1){
    my %classes_map=&read_classes_map();
    my @skipped_lines=();
    my $line_number=0;
    my %tags=();
    print "Creating $class_script_output\n";
    open(ACCOUNTS,"<$sophomorix_add_input_sql") || die "ERROR: $!";
    open(SOPHOMORIXADD,">$sophomorix_add_output") || die "ERROR: $!";
    while(<ACCOUNTS>){
        my $line=$_;
        $line_number++;
        my @attrs=split(/\|/,$line);
        my $count=0;
        print "Line $line_number:\n";
        if ($line_number==1){
            foreach my $attr (@attrs){
                $attr=~s/^\s+//g;# remove leading whitespace
                $attr=~s/\s+$//g;# remove trailing whitespace
                print "   $count: * >$attr<\n";
                $tags{$count}=$attr;
                $count++;
            }
        } else {
            my %line=();
            my $role="";
            my $file="";
            my $creationdate="---";
            my $tolerationdate="---";
            my $deactivationdate="---";
            my $skip_line=0;
            foreach my $attr (@attrs){
                $attr=~s/^\s+//g;# remove leading whitespace
                $attr=~s/\s+$//g;# remove trailing whitespace
                print "   $count: $tags{$count}: >$attr<\n";
                $line{$tags{$count}}=$attr;
                $count++;
            }
            if ($line{"homedirectory"}=~m/\/home\/teachers\//){
                $role="teacher";
                $file="teachers.csv";
            } elsif ($line{"homedirectory"}=~m/\/home\/students\//){
                $role="student";
                $file="students.csv";
            } elsif ($line{"homedirectory"}=~m/\/home\/administrators\//){
                $role="administrator";
                $skip_line=1;
            } elsif ($line{"homedirectory"}=~m/\/home\/attic\//){
                $role="student";
                $file="students.csv";
            } elsif ($line{"homedirectory"}=~m/\/home\/workstations\//){
                $skip_line=1;
            } else {
                # exam accounts are skipped
                if($line{"firstname"} ne "Computer"){
	            print "SKIPPED: $line \n";
                    push @skipped_lines, $line;
                }
                $skip_line=1;
            }

            # skip Computer accounts
            if($line{"firstname"} eq "Computer"){
                $skip_line=1;
            }

            if ($line{"unid"} eq""){
                $line{"unid"}="---";
            }

            my ($year,$month,$day) = split(/-/,$line{"birthday"});
            my $birthdate=$day.".".$month.".".$year;

            $creationdate=&convert_date($line{"creationdate"});
            if ($line{"tolerationdate"} ne ""){
                $tolerationdate=&convert_date($line{"tolerationdate"});
            }
            if ($line{"deactivationdate"} ne ""){
                $deactivationdate=&convert_date($line{"deactivationdate"});
            }

            my $class; # the mapped name
            if (exists $classes_map{$line{'gid'}}){
                $class=$classes_map{$line{'gid'}};
            } else {
                $class=$line{'gid'};
            }
   
            # password hashes are converted to utf8
            my $unicodePwd=`/usr/sbin/sophomorix-vampire-pwdconvert $line{"sambantpassword"}`;
            chomp($unicodePwd);
            # create_line
            my $line= $file."::".
                      $class."::".
                      $line{"surname"}.";".
                      $line{"firstname"}.";".
                      $birthdate."::".
                      $line{"uid"}."::".
                      $line{"firstpassword"}."::".
                      $line{"uidnumber"}."::".
                      $line{"gidnumber"}."::".
                      $line{"unid"}."::".
                      "---"."::".
                      $role."::".
                      $line{"surname"}."::".
                      $line{"firstname"}."::".
                      $line{"sophomorixstatus"}."::".
                      $creationdate."::".
                      $tolerationdate."::".
                      $deactivationdate."::".
                      $unicodePwd."::".
                      $line{"userpassword"}."::".
                      "---"."::".
                      "\n";
            if ($skip_line==0){
	        print "LINE: $line";
	        print SOPHOMORIXADD $line;
            }
        }
        print "\n";
    }
    close(ACCOUNTS);
    close(SOPHOMORIXADD);
    my $skipped=$#skipped_lines+1;
    print "$skipped skipped lines:\n";
    foreach my $line (@skipped_lines){
        $line=~s/\s//g;
        my $line_part=substr($line,0,65);
        print "  LINE: $line_part ...\n";
    }
}



##################################################
# --import-user-password-hashes
if ($import_user_password_hashes==1){
    my @loglines=();
    my ($ldap,$root_dse) = &AD_bind_admin();
    print "Importing user password hashes\n";
    open(ADD,"<$sophomorix_add_output") || die "ERROR: $!";
    my $lines=0;
    my $errors=0;
    while (<ADD>){
        $lines++;
        chomp();
        my (@add)=split(/::/);
        my $sam_account=$add[3];
        my $pass=$add[16];
        my ($count,$dn,$cn)=&AD_object_search($ldap,$root_dse,"user",$sam_account);
        print "\n";
        print "$lines) $sam_account: importing password hash $pass\n";
        print "   DN: $dn\n";

        # creating ldif file
        open(LDIF,">$import_user_pwd_ldif")|| die "ERROR: $!";
	print LDIF "dn: $dn\n";
        print LDIF "changetype: modify\n";
        print LDIF "replace: unicodePwd\n";
        print LDIF "unicodePwd:: $pass\n";
        close(LDIF);

        # load ldif file
        my $com="ldbmodify -H /var/lib/samba/private/sam.ldb --controls=local_oid:1.3.6.1.4.1.7165.4.3.12:0 $import_user_pwd_ldif";
        my $res=system($com);
        if (not $res==0){
            $errors++;
            print "ERROR: $res returned\n";
            push @loglines, "Not updated Hash (Return: $res): $sam_account\n";
        }
    }
    print "\n$errors ERRORS:\n";
    foreach my $line (@loglines){
        print "  * $line";
    }
    &AD_unbind_admin($ldap);
    # clean up
    system("rm -f $import_user_pwd_ldif");
}



##################################################
# --create-class-adminadd-script
if ($create_class_adminadd_script==1){
    # read memberships
    my $line_number=0;
    my %classes_admins=();
    my %classes_map=&read_classes_map();
    print "Creating $class_adminadd_script_output\n";
    open(MEMBER,"<$class_adminadd_script_input") || die "ERROR: $!";
    open(MEMBERSCRIPT,">$class_adminadd_script_output") || die "ERROR: $!";
    while(<MEMBER>){
        my $line=$_;
        $line_number++;
        my @attrs=split(/\|/,$line);
        my $count=0;
        #print "Line $line_number:\n";
        if ($line_number==1){
            # skip;
        } else {
            my $admin=$attrs[0];
            my $class_admin=$attrs[3];
            my $group=$attrs[5];
            $group=~s/^\s+//g;# remove leading whitespace
            $group=~s/\s+$//g;# remove trailing whitespace
            $admin=~s/^\s+//g;# remove leading whitespace
            $admin=~s/\s+$//g;# remove trailing whitespace
            $class_admin=~s/^\s+//g;# remove leading whitespace
            $class_admin=~s/\s+$//g;# remove trailing whitespace

            if ($class_admin ne "teachers"){
                next;
            }
            if ($group eq ""){
                next;
            }

            print "<$group> : <$admin> ($class_admin) is admin\n";
            if (exists $classes_admins{$group}){
		my $new=$classes_admins{$group}.",".$admin;
                $classes_admins{$group}=$new;
            } else {
                $classes_admins{$group}=$admin;
            } 
        }
    }

    # creating command list
    print MEMBERSCRIPT "#!/bin/sh\n";
    my ($ldap,$root_dse) = &AD_bind_admin();
    my @classes_dn=&AD_group_list($ldap,$root_dse,"adminclass",0);
    foreach my $class_dn (@classes_dn){
        my ($class,@rest)=split(/,/,$class_dn);
        $class=~s/^CN=//;
        if (exists $classes_admins{$class}){
            my $class_mapped;
            if (exists $classes_map{$class}){
                $class_mapped=$classes_map{$class};
            } else {
                $class_mapped=$class;
            }

            my $command="sophomorix-class --class $class_mapped --admins $classes_admins{$class}";
            print MEMBERSCRIPT "$command\n";
        }
    }

    &AD_unbind_admin($ldap);
    close(MEMBER);
    close(MEMBERSCRIPT);
    system("chmod 755 $class_adminadd_script_output");

}



##################################################
# --create-project-script
# creating /root/sophomorix-vampire-projects.sh
if ($create_project_script==1){
    my %data=();
    my @pro=();
    my %name_gid_map=();
    print "Creating $project_script_output\n";
    # ceating hash project_name -> gidnumber
    print "$project_script_input_sql\n";
    open(PROSQL,"<$project_script_input_sql") || die "ERROR: $!";
    while(<PROSQL>){
        my $line=$_;
        my @attrs=split(/\|/,$line);
        my $name=$attrs[16];
        my $gid=$attrs[17];
        $name=~s/^\s+//g;# remove leading whitespace
        $name=~s/\s+$//g;# remove trailing whitespace
        $gid=~s/^\s+//g;# remove leading whitespace
        $gid=~s/\s+$//g;# remove trailing whitespace
        $name_gid_map{$name}=$gid;
        #print "<$name> ha <$gid>\n";
    }
    close(PROSQL);
    open(PRO,"<$project_script_input") || die "ERROR: $!";
    open(PROSCRIPT,">$project_script_output") || die "ERROR: $!";
    print PROSCRIPT "#!/bin/sh\n";
    while(<PRO>){
        if(m/\./){
            chomp();
            my ($pro,$setting) = split(/\./);
            my ($para,$set) = split(/=/,$setting);
            $pro=~s/^\s+//g;# remove leading whitespace
            # saving data
            push @pro, $pro;
            $data{$pro}{$para}="$set";
        }
    }
    if($Conf::log_level>=2){
        print Dumper(%data);
    }
    @pro = uniq(@pro);

    # creating script
    # Part1: create projects
    # (projects need to exist for memberships in other projects)
    foreach my $pro (@pro){
        my $command="sophomorix-project --skip-school-creation --create -p $pro --gidnumber-migrate $name_gid_map{$pro}";
        print "  $command\n";
        print PROSCRIPT "$command\n";
   
    }


    # Part2: change projects
    foreach my $pro (@pro){
        my $command="sophomorix-project -p $pro";
        # --join/--nojoin
        if ($data{$pro}{'joinable'}==0){
            $command = $command." --nojoin";
        } else {
            $command = $command." --join";
        }
        # --mailalias/--nomailalias
        if ($data{$pro}{'mailalias'}==0){
            $command = $command." --nomailalias";
        } else {
            $command = $command." --mailalias";
        }
        # --maillist/--nomaillist
        if ($data{$pro}{'maillist'}==0){
            $command = $command." --nomaillist";
        } else {
            $command = $command." --maillist";
        }

        # --maxmembers
        $command = $command." --maxmembers ".$data{$pro}{'maxmembers'};

        # --description
        $command = $command." --description ".$data{$pro}{'longname'};

        # # --addquota
        # if ($data{$pro}{'addquota'} eq "quota"){
        #     $command = $command." --addquota ---";
        # } else {
        #     $command = $command." --addquota ".$data{$pro}{'addquota'};
        # }

        # --status
        $command = $command." --status ".$data{$pro}{'sophomorixstatus'};
        # --creationdate
        my $creationdate=&convert_date($data{$pro}{'creationdate'});
        $command = $command." --creationdate ".$creationdate;

        # --admins
        $command = $command." --admins ".$data{$pro}{'admins'};
        # --members
        $command = $command." --members ".$data{$pro}{'members'};

        my @membergroups=split(/,/,$data{$pro}{'membergroups'});
        my @memberprojects=split(/,/,$data{$pro}{'memberprojects'});
        my @all=(@membergroups,@memberprojects);

        my $all="\"\"";
        if ($#all >= 0){
            $all = join(",",@all);
        }
  
        # --membergroups
        $command = $command." --membergroups ".$all;

        # print the command
        my $substr_command=substr($command,0,65);
        if($Conf::log_level>=2){
            # full multiline command
            print "  $command\n";
        } else {
            # truncated command
            print "  $substr_command ...\n";
        }
        print PROSCRIPT "$command\n";
    }
    close(PRO);
    close(PROSCRIPT);
    system("chmod 755 $project_script_output");
}



############################################################
# rsync data
############################################################
# --rsync-all-student-homes
if ($rsync_all_student_homes==1 or $rsync_student_home ne ""){
    my ($ldap,$root_dse) = &AD_bind_admin();
    print "\n";
    &print_line();
    &print_title("rsync student homes");
    &print_line();
    &before_rsync();
    my $filter="(& (objectClass=user) (sophomorixRole=student) )";
    my $mesg = $ldap->search( # perform a search
                      base   => $root_dse,
                      scope => 'sub',
                      filter => $filter,
                      attrs => ['sAMAccountName',
                                'sophomorixAdminClass',
                                'objectClass',
                               ]);
    my $max = $mesg->count;
    for ( my $index = 0 ; $index < $max ; $index++) {
        my $entry = $mesg->entry($index);
        my $sam=$entry->get_value('sAMAccountName');
        my $class=$entry->get_value('sophomorixAdminClass');
        my $source_dir=$path_oldserver."/home/students/".$class."/".$sam."/";
        my $target_dir="/srv/samba/mounts/schools/default-school/students/".$class."/".$sam;

        if ($rsync_student_home ne ""){
            if ($sam ne $rsync_student_home){
                next;
            }
        }

	if (-d "$source_dir" and -d "$target_dir"){
            print "   * rsync home of student $sam in class $class\n";
            print "      * Source: $source_dir\n";
            print "      * Target: $target_dir\n";
            my $rsync_command="rsync -rv $source_dir $target_dir";
            print "$rsync_command\n";
            system($rsync_command);
        } elsif (not -d "$source_dir") {
            print "   * No oldhome found for student $sam in class $class\n";
        } elsif (not -d "$source_dir") {
            print "   * No target dir found for student $sam in class $class\n";
        }
    }
    &AD_unbind_admin($ldap);
    &after_rsync();
}



# --rsync-all-teacher-homes
if ($rsync_all_teacher_homes==1 or $rsync_teacher_home ne ""){
    my ($ldap,$root_dse) = &AD_bind_admin();
    print "\n";
    &print_line();
    &print_title("rsync teacher homes");
    &print_line();
    &before_rsync();
    my $filter="(& (objectClass=user) (sophomorixRole=teacher) )";
    my $mesg = $ldap->search( # perform a search
                      base   => $root_dse,
                      scope => 'sub',
                      filter => $filter,
                      attrs => ['sAMAccountName',
                                'sophomorixAdminClass',
                                'objectClass',
                               ]);
    my $max = $mesg->count;
    for ( my $index = 0 ; $index < $max ; $index++) {
        my $entry = $mesg->entry($index);
        my $sam=$entry->get_value('sAMAccountName');
        my $class=$entry->get_value('sophomorixAdminClass');
        my $source_dir=$path_oldserver."/home/teachers/".$sam."/";
        my $target_dir="/srv/samba/mounts/schools/default-school/teachers/".$sam;

        if ($rsync_teacher_home ne ""){
            if ($sam ne $rsync_teacher_home){
                next;
            }
        }

	if (-d "$source_dir" and -d "$target_dir"){
            print "   * rsync home of teacher $sam\n";
            print "      * Source: $source_dir\n";
            print "      * Target: $target_dir\n";
            my $rsync_command="rsync -rv $source_dir $target_dir";
            print "$rsync_command\n";
            system($rsync_command);
        } elsif (not -d "$source_dir") {
            print "   * No oldhome found for teacher $sam\n";
        } elsif (not -d "$target_dir") {
            print "   * No target dir found for teacher $sam\n";
        }
    }
    &AD_unbind_admin($ldap);
    &after_rsync();
}



# --rsync-all-class-shares
if ($rsync_all_class_shares==1 or $rsync_class_share ne ""){
    my ($ldap,$root_dse) = &AD_bind_admin();
    print "\n";
    &print_line();
    &print_title("rsync class shares");
    &print_line();
    &before_rsync();
    my $filter="(& (objectClass=group) (sophomorixType=adminclass) )";
    my $mesg = $ldap->search( # perform a search
                      base   => $root_dse,
                      scope => 'sub',
                      filter => $filter,
                      attrs => ['sAMAccountName',
                                'objectClass',
                               ]);
    my $max = $mesg->count;
    for ( my $index = 0 ; $index < $max ; $index++) {
        my $entry = $mesg->entry($index);
        my $sam=$entry->get_value('sAMAccountName');
        my $source_dir=$path_oldserver."/home/share/classes/".$sam."/";
        my $target_dir="/srv/samba/mounts/schools/default-school/share/classes/".$sam;

        if ($rsync_class_share ne ""){
            if ($sam ne $rsync_class_share){
                next;
            }
        }

	if (-d "$source_dir" and -d "$target_dir"){
            print "   * rsync share of class $sam\n";
            print "      * Source: $source_dir\n";
            print "      * Target: $target_dir\n";
            my $rsync_command="rsync -rv $source_dir $target_dir";
            print "$rsync_command\n";
            system($rsync_command);
        } elsif (not -d "$source_dir") {
            print "   * No share data found for class $sam\n";
        } elsif (not -d "$target_dir") {
            print "   * No target dir found for class $sam\n";
        }
    }
    &AD_unbind_admin($ldap);
    &after_rsync();
}



# --rsync-all-project-shares
if ($rsync_all_project_shares==1 or $rsync_project_share ne ""){
    my ($ldap,$root_dse) = &AD_bind_admin();
    print "\n";
    &print_line();
    &print_title("rsync project shares");
    &print_line();
    &before_rsync();
    my $filter="(& (objectClass=group) (sophomorixType=project) )";
    my $mesg = $ldap->search( # perform a search
                      base   => $root_dse,
                      scope => 'sub',
                      filter => $filter,
                      attrs => ['sAMAccountName',
                                'objectClass',
                               ]);
    my $max = $mesg->count;
    for ( my $index = 0 ; $index < $max ; $index++) {
        my $entry = $mesg->entry($index);
        my $sam=$entry->get_value('sAMAccountName');
        my $source_dir=$path_oldserver."/home/share/projects/".$sam."/";
        my $target_dir="/srv/samba/mounts/schools/default-school/share/projects/".$sam;

        if ($rsync_project_share ne ""){
            if ($sam ne $rsync_project_share){
                next;
            }
        }

	if (-d "$source_dir" and -d "$target_dir"){
            print "   * rsync share of project $sam\n";
            print "      * Source: $source_dir\n";
            print "      * Target: $target_dir\n";
            my $rsync_command="rsync -rv $source_dir $target_dir";
            print "$rsync_command\n";
            system($rsync_command);
        } elsif (not -d "$source_dir") {
            print "   * No share data found for project $sam\n";
        } elsif (not -d "$target_dir") {
            print "   * No target dir found for project $sam\n";
        }
    }
    &AD_unbind_admin($ldap);
    &after_rsync();
}



# --rsync-linbo
if ($rsync_linbo==1){
    my $source_dir=$path_oldserver."/var/linbo/";
    my $target_dir="/srv/linbo";
    my $rsync_command="rsync -rv $source_dir $target_dir";
    print "$rsync_command\n";
    system($rsync_command);
}



#my ($ldap,$root_dse) = &AD_bind_admin();
#&AD_unbind_admin($ldap);

#&log_script_end(\@arguments);


############################################################
# sub
############################################################

sub before_rsync {
    # looks if the source loks good and the target, exits if not

    print "\n#####  Mounting share default-school  #####\n";
    system("sophomorix-school --mount default-school 1> /dev/null");

    print "RSYNC TEST\n";

    # source
    if (-d "$path_oldserver/home/students/" and
        -d "$path_oldserver/home/teachers" and
        -d "$path_oldserver/home/share/classes" and
        -d "$path_oldserver/home/share/projects"
	){
        print "  * OK: There is source data in: $path_oldserver\n";
    } else {
        print "  * ERROR: There is no source data in: $path_oldserver\n";
        exit;
    }

    # target
    if (-d "/srv/samba/mounts/schools/default-school/students" and
        -d "/srv/samba/mounts/schools/default-school/teachers" and
        -d "/srv/samba/mounts/schools/default-school/share/classes" and
        -d "/srv/samba/mounts/schools/default-school/share/projects"
	){
        print "  * Target default-school seems OK\n";
    } else {
        print "  * Target default-school seems to be broken\n";
        exit;
    }

    print "RSYNC TEST OK\n";
    print "\n";
}



sub after_rsync {
    print "\n#####  Umounting share default-school  #####\n";
    system("sophomorix-school --umount default-school 1> /dev/null");
}



sub killog_rewrite {
    my ($in,$out)=@_;

    print "\n";
    print "### Writing $out\n";
    open(IN,"<$in") || die "Cannot open input $in \n";
    open(OUT,">$out") || die "Cannot create output $out \n";
    while(<IN>){
        my $line=$_;
        my $new_line;
        chomp($line);

        $line=~s/^\s+//g;# remove leading whitespace
        if(/^\#/){
             # keep comment lines
             $new_line=$line."\n";
        } elsif ($line eq ""){
             # keep empty lines
             $new_line=$line."\n";
        } else {
            my (@items)=split(/::/,$line);
            my ($day,$month,$year)=split(/\./,$items[1]);
            my $epoch=timelocal(0, 0, 0, $day , ($month-1), $year);
            if ($items[0] eq "user archived" 
                and $year > 2016 
                and $items[4] ne "Computer"
                and $items[4] ne "Account"){
                my $date_ad=$year.$month.$day."000000.0Z";
                $new_line="KILL::".$epoch."::".
                      $date_ad."::".
                      "---::".
                      $items[2]."::".
                      $items[4]."::".
                      $items[5]."::".
                      "---::".
                      "---::".
                      "---::".
                      "MIGRATED::".
                      "\n";
                print OUT "$new_line";
            }
        }
    }
    close(IN);
    close(OUT);

    # prepend old log to existing log
    my $logdir="/var/log/sophomorix/userlog";
    my $killog=$logdir."/user-kill.log";
    my $killog_tmp=$killog.".tmp";
    my @oldlines=();
    my %oldlines=();
    system("mkdir -p $logdir");
    open(TMPLOG,">$killog_tmp");

    # read old killog lines and remember them
    if (-f $killog){
        open(LOG,"<$killog") || die "Cannot open $killog\n";
        while(<LOG>){
            push @oldlines,$_;
            $oldlines{$_}="old";    
        }
        close(LOG);
    }

    # first write very old migrated lines if not done already
    open(OUT,"<$out") || die "Cannot create $out \n";
    while(<OUT>){
        if (not exists $oldlines{$_}){
            print TMPLOG "$_";
        }
    }
    close(OUT);

    # write old lines (newer tha migrated lines) 
    foreach my $line (@oldlines){
        print TMPLOG "$line";
    }
    close(TMPLOG);

    # copy modified log obver log
    system("cp $killog_tmp $killog");
}



sub extrastudents_rewrite {
    my ($in,$out)=@_;
    my %classes_map=&read_classes_map();

    print "\n";
    print "### Writing $out\n";
    open(IN,"<$in") || die "Cannot create input $in \n";
    open(OUT,">$out") || die "Cannot create output $out \n";
    while(<IN>){
        my $line=$_;
        my $new_line;
        chomp($line);

        $line=~s/^\s+//g;# remove leading whitespace
        if(/^\#/){
             # keep comment lines
             $new_line=$line."\n";
        } elsif ($line eq ""){
             # keep empty lines
             $new_line=$line."\n";
        } else {
        my (@items)=split(/;/,$line);

        # rename groups according to classes_map
        if (exists $classes_map{$items[0]}){
            $items[0]=$classes_map{$items[0]}; 
        }
        $new_line=$items[0].";".
                  $items[1].";".
                  $items[2].";".
                  $items[3].";".
                  $items[4].";".
                  "\n";
        }
        print OUT "$new_line";
    }
    close(IN);
    close(OUT);
}



sub devices_rewrite {
    my ($ref_classes_map,$in,$out,$tmp)=@_;
    open(TMP,">$tmp") || die "Cannot create $tmp \n";
    my $special_line="######################### The following devices were migrated #########################";
    print "\n";

    print Dumper ($ref_classes_map);
    # reading existing devices.csv until special_line
    if (-f $out){
        print "### Reading already existing $out\n";
        open(OUT,"<$out") || die "Cannot create $out \n";
        while(<OUT>){
            my $line=$_;
            chomp($line);
            if ($line eq $special_line){
                last;
            }
            print TMP $line."\n";
        }
        # finalize with special line
        print TMP $special_line."\n";
        print TMP "\n";
        close(OUT);
    } else {
        print "### Not reading nonexisting: $out\n";
        print TMP $special_line."\n";
        print TMP "\n";
    }

    print "### Writing temp file $tmp\n";
    open(IN,"<$in") || die "Cannot open $in \n";
    while(<IN>){
        my $line=$_;
        my $new_line;
        chomp($line);

        $line=~s/^\s+//g;# remove leading whitespace
        if(/^\#/){
             # keep comment lines
             $new_line=$line."\n";
        } elsif ($line eq ""){
             # keep empty lines
             $new_line=$line."\n";
        } else {
        my (@items)=split(/;/,$line);




        # field 6 / $item[5]
        # MS office key --> ---
        # Todo ??????: if netmask/ip --> ---
        # Test for ip
        my $is_ip=0;
        my $dots_in_string=$items[5]=~tr/\.//;
        if($dots_in_string==3){
            #print " ($dots_in_string dots) in $items[5]\n";
            my @octets = split(/\./,$items[5]);
            foreach my $octet (@octets){
                #print "   Testing $octet\n";
                if ( int($octet) < 0 or int($octet) > 255 ) {
                    # not an ip
	        } else {
                    #print "LINE: $line\n";
                    #print "   Field 6: $items[5] is an IP --> replacing it with ---\n";
                    $is_ip=1;
                }
            }
        }

        if ($items[5] eq "" or $is_ip==1){
            $items[5]="---";
        }

        # field 1 (group)
        my $new_group;
        if (exists $ref_classes_map->{$items[0]}){
            print "Rename group $items[0] ---> $ref_classes_map->{$items[0]}\n";
            $new_group=$ref_classes_map->{$items[0]};
        } else {
            $new_group=$items[0];
        }

        # field 7 (MS windows key)
        if ($items[6] eq "" or $items[6] eq "1"){## ???? why 1?
        #if ($items[6] eq ""){## 
            $items[6]="---";
        }

        if (not defined $items[9]){
            $items[9]="";
        }
        if (not defined $items[10]){
            $items[10]="";
        }
        if (not defined $items[11]){
            $items[11]="";
        }
        if (not defined $items[12]){
            $items[12]="";
        }
        if (not defined $items[13]){
            $items[13]="";
        }
        if (not defined $items[14]){
            $items[14]="MIGRATION";
        }

        # modify pxe to 0, if empty
        if ($items[10] eq "22"){
            $items[10]="1";
        } elsif ($items[10] eq ""){
            $items[10]="0";
        }
        my $sophomorix_type;
        if ($items[9] eq "0"){
            # default device without account
	    $sophomorix_type="iponly";
        } else {
            # default device with account
	    $sophomorix_type="classroom-studentcomputer";
        }
        # account_flag field 10 not used anymore
        $items[9]="---";

        $new_line=$new_group.";".
                  $items[1].";".
                  $items[2].";".
                  $items[3].";".
                  $items[4].";".
                  $items[5].";".
                  $items[6].";".
                  $items[7].";".
                  $sophomorix_type.";".
                  $items[9].";".
                  $items[10].";".
                  $items[11].";".
                  $items[12].";".
                  $items[13].";".
                  $items[14].";".
                  $items[8].";". # save in field 16 the value of field 9
                  "\n";
        }

        print TMP "$new_line";
    }
    close(IN);
    close(TMP);
    print "### Copying temp file to $out\n";
    system("cp $tmp $out");
}



sub subnets_rewrite {
    my ($in,$out,$tmp)=@_;
    open(TMP,">$tmp") || die "Cannot create $tmp \n";
    my $special_line="######################### The following subnets were migrated #########################";
    print "\n";

    # reading existing subnets.csv until special_line
    if (-f $out){
        print "### Reading already existing $out\n";
        open(OUT,"<$out") || die "Cannot create $out \n";
        while(<OUT>){
            my $line=$_;
            chomp($line);
            if ($line eq $special_line){
                last;
            }
            print TMP $line."\n";
        }
        # finalize with special line
        print TMP $special_line."\n";
        print TMP "\n";
        close(OUT);
    } else {
        print "### Not reading nonexisting: $out\n";
        print TMP $special_line."\n";
        print TMP "\n";
    }

    print "### Writing temp file $tmp\n";
    open(IN,"<$in") || die "Cannot open $in \n";
    while(<IN>){
        my $line=$_;
        my $new_line;
        chomp($line);

        $line=~s/^\s+//g;# remove leading whitespace
        if(/^\#/){
             # keep comment lines
             $new_line=$line."\n";
        } elsif ($line eq ""){
             # keep empty lines
             $new_line=$line."\n";
        } else {
            # do not use fiels 5 and 6 (i.e. [4][5])
            my (@items)=split(/;/,$line);
            $new_line=$items[0].";".
                      $items[1].";".
                      $items[2].";".
                      $items[3].";".
                      "MIGRATION;\n";
        }

        print TMP "$new_line";
    }
    close(IN);
    close(TMP);
    print "### Copying temp file to $out\n";
    system("cp $tmp $out");
}



sub teachers_rewrite {
    my ($in,$out)=@_;
    print "\n";
    print "### Writing $out\n";
    open(IN,"<$in") || die "Cannot create $in \n";
    open(OUT,">$out") || die "Cannot create $out \n";
    while(<IN>){
        my $line=$_;
        chomp($line);
        my (@items)=split(/;/,$line);
        $items[1]=~s/^\s+//g;# remove leading whitespace
        $items[1]=~s/\s+$//g;# remove trailing whitespace
        $items[2]=~s/^\s+//g;# remove leading whitespace
        $items[2]=~s/\s+$//g;# remove trailing whitespace
        $items[3]=~s/^\s+//g;# remove leading whitespace
        $items[3]=~s/\s+$//g;# remove trailing whitespace
        $items[4]=~s/^\s+//g;# remove leading whitespace
        $items[4]=~s/\s+$//g;# remove trailing whitespace

        my $new_line="teachers;".$items[1].";".$items[2].";".$items[3].";".$items[4].";\n";
        print OUT "$new_line";
    }
    close(IN);
    close(OUT);
}



sub convert_date {
    my ($olddate) = @_;
    my ($date,$time)=split(/ /,$olddate);

    if (not defined $time){
        $time="00:00:00";
    }
    if ($time eq ""){
        $time="00:00:00";
    }
    my ($year,$month,$day)=split(/-/,$date);
    my ($hour,$minute,$second)=split(/:/,$time);
    my $newdate=$year.$month.$day.$hour.$minute.$second.".0Z";
    return $newdate;
}



sub read_classes_map {
    my %classes_map=();
    open(MAP,"<$classes_map") || die "ERROR: $!";
    while(<MAP>){
        my $line=$_;
        my ($old,$new)=split(/::/,$line);
        $classes_map{$old}=$new;
    }
    return %classes_map;
}
