#!/usr/bin/perl -w
# This script (sophomorix-check) is maintained by Rüdiger Beck
# It is Free Software (License GPLv3)
# If you find errors, contact the author
# jeffbeck@web.de  or  jeffbeck@linuxmuster.net

# modules
use strict;
#use Quota;
use Getopt::Long;
use Time::Local;
Getopt::Long::Configure ("bundling");
use Sophomorix::SophomorixConfig;
use List::MoreUtils qw(uniq);
use String::Approx 'amatch';
use String::Approx 'adist';
use Net::LDAP;use JSON;
use Data::Dumper;
$Data::Dumper::Indent = 1;
$Data::Dumper::Sortkeys = 1;
$Data::Dumper::Useqq = 1;
$Data::Dumper::Terse = 1; 
use File::Basename qw( basename
                       dirname
                     ); 
use Text::Iconv;
use Sophomorix::SophomorixBase qw(
                                 print_line
                                 print_title
                                 create_schoollist
                                 ymdhms_to_epoch
                                 unlock_sophomorix
                                 lock_sophomorix
                                 log_script_start
                                 log_script_end
                                 log_script_exit
                                 get_login_avoid
                                 create_test_login
                                 backup_auk_file
                                 get_passwd_charlist
                                 get_plain_password
                                 check_options
                                 test_webui_permission
                                 config_sophomorix_read
                                 result_sophomorix_init
                                 result_sophomorix_add
                                 result_sophomorix_add_summary
                                 result_sophomorix_add_log
                                 result_sophomorix_check_exit
                                 result_sophomorix_print
                                 filelist_fetch
                                 remove_whitespace
                                 remove_embracing_whitespace
                                 json_dump
                                 read_encoding_data
                                 analyze_encoding
                                 print_analyzed_encoding
                                 recode_utf8_to_ascii
                                 );
use Sophomorix::SophomorixSambaAD qw(
                                 AD_school_create
                                 AD_bind_admin
                                 AD_unbind_admin
                                 AD_user_create
                                 AD_group_create
                                 AD_group_addmember
                                 AD_group_update
                                 AD_get_schoolname
                                 AD_get_name_tokened
                                 AD_dn_fetch_multivalue
                                 AD_get_AD_for_check
                                 AD_dns_get
                                 AD_create_new_webui_string
                                 AD_create_new_mail
                                 AD_object_search
                                    );

my @arguments = @ARGV;

my $user_count=0;

my %users_file=();
my %unid_seen=();

my %counters=();
$counters{'ADD'}=0;
$counters{'UPDATE'}=0;
$counters{'KILL'}=0;
$counters{'NOCHANGE'}=0;
$counters{'ERROR'}=0;

# allowed numbers for days
my %convert_day = qw(
     1     01          01    01
     2     02          02    02
     3     03          03    03
     4     04          04    04
     5     05          05    05
     6     06          06    06
     7     07          07    07
     8     08          08    08
     9     09          09    09
     10    10          11    11
     12    12          13    13
     14    14          15    15
     16    16          17    17
     18    18          19    19
     20    20          21    21
     22    22          23    23
     24    24          25    25
     26    26          27    27
     28    28          29    29
     30    30          31    31
);

# allowed numbers for months
my %convert_month = qw(
     1     01          01    01
     2     02          02    02
     3     03          03    03
     4     04          04    04
     5     05          05    05
     6     06          06    06
     7     07          07    07
     8     08          08    08
     9     09          09    09
     10    10          11    11
     12    12
);

# allowed numbers for years
my %convert_year = qw(
     1     2001          01    2001
     2     2002          02    2002
     3     2003          03    2003
     4     2004          04    2004
     5     2005          05    2005
     6     2006          06    2006
     7     2007          07    2007
     8     2008          08    2008
     9     2009          09    2009
     10    2010          11    2011
     12    2012          13    2013
     14    2014          15    2015
     16    2016          17    2017
     18    2018          19    2019
     20    2020          21    2021
     22    2022          23    2023
     24    2024          25    2025
     26    2026          27    2027
     28    2028          29    2029
     30    1930          31    1931
     32    1932          33    1933
     34    1934          35    1935
     36    1936          37    1937
     38    1938          39    1939
     40    1940          41    1941
     42    1942          43    1943
     44    1944          45    1945
     46    1946          47    1947
     48    1948          49    1949
     50    1950          51    1951
     52    1952          53    1953
     54    1954          55    1955
     56    1956          57    1957
     58    1958          59    1959
     60    1960          61    1961
     62    1962          63    1963
     64    1964          65    1965
     66    1966          67    1967
     68    1968          69    1969
     70    1970          71    1971
     72    1972          73    1973
     74    1974          75    1975
     76    1976          77    1977
     78    1978          79    1979
     80    1980          81    1981
     82    1982          83    1983
     84    1984          85    1985
     86    1986          87    1987
     88    1988          89    1989
     90    1990          91    1991
     92    1992          93    1993
     94    1994          95    1995
     96    1996          97    1997
     98    1998          99    1999
);
for( my $year = 1930 ; $year < 2029 ; $year++) {
    $convert_year{$year}=$year;
}


# encoding tests
my %firstnames_data=();
my %firstnames_errors=();
my %lastnames_data=();
my %lastnames_errors=();
my %encoding_check_results=();
my @encodings_to_check=("UTF8","ISO_8859-1","WINDOWS-1252");

my @loglines=();

# ===========================================================================
# Optionen verarbeiten
# ==========================================================================

# Variablen für Optionen
$Conf::log_level=1;
my $help=0;
my $info=0;
my $level=0;
my $json=0;
my %Match=();

my $dump_matches=0;
my $dump_files=0;
my $dump_ad=0;
my $dump_login_avoid=0;

my $school="";

my $list_files=0;
my $cat_files=0;
my $lock=0;
my $unlock=0;
my $analyze_encoding="";
my $show_special_char_lines=0;
my $non_umlaut=0;
my $follow="";
my $opt_edit_distance=4;
my @injectlines=(); # list of injected lines

# Parsen der Optionen
my $testopt=GetOptions(
           "help|h" => \$help,
           "info|i" => \$info,
           "level+" => \$level,
           "json|j+" => \$json,
           "verbose|v+" => \$Conf::log_level,
           "lock" => \$lock,
           "unlock" => \$unlock,
           "dump-files" => \$dump_files,
           "dump-ad" => \$dump_ad,
           "dump-login-avoid" => \$dump_login_avoid,
           "dump-matches" => \$dump_matches,
           "list-files|listfiles" => \$list_files,
           "cat-files|catfiles" => \$cat_files,
           "analyze-encoding=s" => \$analyze_encoding,
           "follow=s" => \$follow,
           "school=s" => \$school,
           "injectline=s" => \@injectlines,
           "show-special-char-lines" => \$show_special_char_lines,
           "non-umlaut|nonumlaut" => \$non_umlaut,
           "edit-distance=i" => \$opt_edit_distance,
          );

my %sophomorix_result=&result_sophomorix_init("sophomorix-check");
# Prüfen, ob Optionen erkannt wurden, sonst Abbruch
&check_options($testopt,\%sophomorix_result,$json);


# --level
if ($level>2){
    print "\nERROR: Max 2 levels allowed\n\n";
    exit;
}


# --follow
my $follow_file="";
my $follow_line="";
if ($follow ne ""){
    ($follow_file,$follow_line)=split(/:/,$follow);
}


# Reading Configuration
my ($ldap,$root_dse) = &AD_bind_admin(\@arguments,\%sophomorix_result,$json);
my $root_dns=&AD_dns_get($root_dse);
my %sophomorix_config=&config_sophomorix_read($ldap,$root_dse,\%sophomorix_result);
my @filelist=&filelist_fetch({filetype=>"users",
                              sophomorix_config=>\%sophomorix_config,
                            });

# --help
if ($help==1) {
   # Scriptname ermitteln
   my @list = split(/\//,$0);
   my $scriptname = pop @list;
   # Befehlsbeschreibung
   print('
sophomorix-check checks the configured user files and finds out which users should be added, updated and killed


Options:
  -h  / --help
  -v  / --verbose
  -vv / --verbose --verbose
  -i  / --info
  -i --level 
  -i --level --level

  --list-files                     (list processed files and exit)
  --cat-files                      (cat/show processed files and exit)
  --lock / --unlock

  --follow filename:linenumber     (show what happens to this line)
  --edit-distance <num>            (default: 4, how similar must user be to be matched)

Analyze the encoding of a file:
  --analyze-encoding /path/to/students.csv
Analyze the encoding of a file and show Umlaut and other special chars
  --show-special-char-lines --analyze-encoding /path/to/students.csv
Analyze the encoding of a file and show special chars (not umlauts)
  --non-umlaut --show-special-char-lines --analyze-encoding /path/to/students.csv

Show usable encodings in school.conf:
   iconv -l

Dumping data:                     
  -ij / --info --json              (dump configuration)
  --dump-files -j                  (dump contents of user files)
  --dump-ad -j                     (dump AD data)
  --dump-login-avoid -j            (dump historic data from log files)
  --dump-matches -j                (dump matched users)

  -j / --json                      (dump as a nice json object)
  -jj / --json --json              (dump as a compact json object)
  -jjj / --json --json --json      (dump as a perl hash)

Injecting lines (repeat the option to inject multiple lines):                     
  --injectline "teachers.csv:7a;Meier;Richard;23.09.1996;mue999"
  --injectline "bsz.students.csv:7b;Schmid;Richard;26.04.1995;"
 
Please see the sophomorix-check(8) man pages for full documentation
');
   print "\n";
   exit;
}


# --school <school1,school2, ...>
my @opt_schoollist=&create_schoollist($school,\%sophomorix_config);


# --unlock
if ($unlock==1) {
    &unlock_sophomorix();
    exit;
}


# --lock
if ($lock==1) {
    &lock_sophomorix("lock",0,\@arguments);
    exit;
}



# --info
# --info --level
# --info --level --level 
if ($info==1 and $level==0) {
    &json_dump({json => $json,
                jsoninfo => "CONFIG",
                jsoncomment => "The sophomorix configuration",
                log_level => $Conf::log_level,
                hash_ref=>\%sophomorix_config,
                sophomorix_config=>\%sophomorix_config,
               });
    print "\n";
    &print_title("Reading the following user files:");
    foreach my $file (@filelist){
        print "   * $file\n";
    }
    exit;
} elsif ($info==1 and $level==1) {
    # LEVEL 1 ########################################
    &print_title("Showing 1 level of the config hash:");
    my @list=();
    foreach my $key (keys %sophomorix_config) {
        push @list,$key;
    }
    @list = sort @list;
    foreach my $item (@list){
        print "   * $item\n";
    }
    exit;
} elsif ($info==1 and $level==2) {
    # LEVEL 2 ########################################
    &print_title("Showing 2 levels of the config hash:");
    my @list=();
    foreach my $key (keys %sophomorix_config) {
        push @list,$key;
    }
    @list = sort @list;
    foreach my $item (@list){
        my @sublist=();
        print "   * $item\n";
        if ($item eq "ENCODINGS" or 
            $item eq "REPDIR_FILES" or 
            $item eq "INI" or 
            $item eq "GLOBAL" or 
            $item eq "SCHOOLS"){
            print "      -> ... omitted ...\n";
            next;
        }
        foreach my $subkey (keys %{ $sophomorix_config{$item} }) {
            push @sublist,$subkey;
        }
        @sublist = sort @sublist;
        foreach my $item (@sublist){
            print "      -> $item\n";
        }
    }
    exit;
} 




# --list-files / --cat-files
if ($list_files==1 or $cat_files==1) {
    &print_title("Reading the following user files:");
    foreach my $file (@filelist){
        if ($list_files==1){
            print "   * $file\n";
        } elsif ($cat_files==1){
            print "======================================================================\n";
            print "  $file\n";
            print "----------------------------------------------------------------------\n";
            system("cat $file");
            print "======================================================================\n";
            print "\n";
        }
    }
    exit;
}



# --analyze-encoding
# analyze one file
if ($analyze_encoding ne ""){
    if($Conf::log_level>=3){
        print "\n";
        print "Analyzing encoding\n";
    }

    my $ref_encoding_data=&read_encoding_data();

    # print Dumper($ref_encoding_data);

    my $enc;
    my $ref_encoding_check_results;
    ($enc,$ref_encoding_check_results)=&analyze_encoding($analyze_encoding,
                                                         $analyze_encoding,
                                                         $show_special_char_lines,
                                                         $non_umlaut,
                                                         $ref_encoding_data,
                                                         $ref_encoding_check_results,
                                                         \%sophomorix_config,
                                                         \%sophomorix_result
                                                        );
    &print_analyzed_encoding($analyze_encoding,$ref_encoding_check_results,$ref_encoding_data);
    my $result="$analyze_encoding is $enc encoded";
    &result_sophomorix_add_summary({
                         NAME=>"ANALYZE-ENCODING", 
                         RESULT=>$result, 
                         FORMAT_TYPE => 0,
                         FILE => $analyze_encoding,
                         ENCODING => $enc,
                         sophomorix_result=>\%sophomorix_result,
			       });
    &log_script_end(\@arguments,\%sophomorix_result,\%sophomorix_config,$json);
}



&result_sophomorix_check_exit(\%sophomorix_result,\%sophomorix_config,$json);
&log_script_start(\@arguments,\%sophomorix_result,\%sophomorix_config);

# exit with error message if samba needs longer password than configured
foreach my $user_file (@filelist){
    my $filename = basename($user_file);
    if ($sophomorix_config{'FILES'}{'USER_FILE'}{$filename}{'RANDOM_PWD'} eq "TRUE" and
	$sophomorix_config{'samba'}{'domain_passwordsettings'}{'Minimum_password_length'}>$sophomorix_config{'FILES'}{'USER_FILE'}{$filename}{'PWD_LENGTH'}
       ){
        my $error_message="Samba requires a password lenth of $sophomorix_config{'samba'}{'domain_passwordsettings'}{'Minimum_password_length'}. ".
	                  "You configured PWD_LENGTH=$sophomorix_config{'FILES'}{'USER_FILE'}{$filename}{'PWD_LENGTH'} for $filename" ;
        &log_script_exit($error_message,1,1,0,
                     \@arguments,\%sophomorix_result,\%sophomorix_config,$json);
    }
}


################################################################################
# Start
################################################################################
# cleaning up last check
system("mkdir -p $sophomorix_config{'INI'}{'PATHS'}{'LOG'}");
&print_title("Cleaning up last check","slim");
system("rm -f $DevelConf::path_conf_tmp/*.csv*");
system("rm -f $sophomorix_config{'INI'}{'PATHS'}{'CHECK_RESULT'}/sophomorix.*");
system("rm -f $sophomorix_config{'INI'}{'PATHS'}{'REPORT_OFFICE'}/report.office.*");

# create result files
my $add_file=$sophomorix_config{'INI'}{'PATHS'}{'CHECK_RESULT'}."/sophomorix.add";
open (ADD, ">$add_file");
my $update_file=$sophomorix_config{'INI'}{'PATHS'}{'CHECK_RESULT'}."/sophomorix.update";
open (UPDATE, ">$update_file");
my $kill_file=$sophomorix_config{'INI'}{'PATHS'}{'CHECK_RESULT'}."/sophomorix.kill";
open (KILL, ">$kill_file");
my $nochange_file=$sophomorix_config{'INI'}{'PATHS'}{'CHECK_RESULT'}."/sophomorix.nochange";
open (NOCHANGE, ">$nochange_file");
my $error_file=$sophomorix_config{'INI'}{'PATHS'}{'CHECK_RESULT'}."/sophomorix.error";
open (ERROR, ">$error_file");

# ============================================================
# analyzing encoding of all filtered user files in tmp
# ============================================================
my $ref_encoding_data=&read_encoding_data();
my $ref_encoding_check_results;
foreach my  $user_file (@filelist){
    my $enc;
    ($enc,$ref_encoding_check_results)=&analyze_encoding($user_file,
                                                         $user_file,
                                                         $show_special_char_lines,
                                                         $non_umlaut,
                                                         $ref_encoding_data,
                                                         $ref_encoding_check_results,
                                                         \%sophomorix_config,
                                                         \%sophomorix_result
                                                        );
}


#print Dumper ($ref_encoding_check_results);

# ============================================================
# reading and checking all filtered user files
# ============================================================
system("mkdir -p $DevelConf::path_conf_tmp");

if($Conf::log_level>=2){
    &print_title("Analyzing user files ...");
}
foreach my $user_file (@filelist){
    my %lines_seen=();
    my $count=0; # reset line counter
    my $filename = basename($user_file);
    # fetching some config data
    $school=$sophomorix_config{'FILES'}{'USER_FILE'}{$filename}{'SCHOOL'};

    my $enc_used=&get_encoding($filename);
    open(USERS,"$user_file") || 
         die "ERROR: $user_file not found!";  
    open(USERSUTF8,">$sophomorix_config{'FILES'}{'USER_FILE'}{$filename}{'PATH_ABS_UTF8'}") || 
         die "ERROR: $sophomorix_config{'FILES'}{'USER_FILE'}{$filename}{'PATH_ABS_UTF8'} not found!";  
    open(REPORT,">$sophomorix_config{'FILES'}{'USER_FILE'}{$filename}{'PATH_ABS_REPORT_OFFICE'}") || 
         die "ERROR: $sophomorix_config{'FILES'}{'USER_FILE'}{$filename}{'PATH_ABS_REPORT_OFFICE'} not found!"; 
    print REPORT "Office Report for: $user_file\n"; 
    print REPORT "Date_Time:         $sophomorix_config{'DATE'}{'LOCAL'}{'TIMESTAMP_FILE'}\n";
 
    while(<USERS>){
        $count++;
        my $line=$_;
        
        &analyze_user_line($line,$user_file,$filename,$school,$count,$enc_used,"no","USER_FILE",\%lines_seen);
    }

    close(USERS);
    close(USERSUTF8);
    close(REPORT);
}


# ============================================================
# injecing lines given by option
# ============================================================
&print_title("Reading injected lines ...");
my $count=0;
foreach my $inject (@injectlines){
    print "    * $inject\n";
    my ($filename,$line)=split(/:/,$inject,2); # split in 2 parts
    if (exists $sophomorix_config{'FILES'}{'USER_FILE'}{$filename}{'SCHOOL'}){
        $school=$sophomorix_config{'FILES'}{'USER_FILE'}{$filename}{'SCHOOL'};
        print "        file:   $filename\n";
        print "        school: $school\n";
        print "        line:   $line\n";
        $count++;
        my $file_abs=$sophomorix_config{'FILES'}{'USER_FILE'}{$filename}{'PATH_ABS'};
        # opening the output file to APPEND entries
        open(USERSUTF8,">>$sophomorix_config{'FILES'}{'USER_FILE'}{$filename}{'PATH_ABS_UTF8'}") || 
             die "ERROR: $sophomorix_config{'FILES'}{'USER_FILE'}{$filename}{'PATH_ABS_UTF8'} not found!";  
        open(REPORT,">>$sophomorix_config{'FILES'}{'USER_FILE'}{$filename}{'PATH_ABS_REPORT_OFFICE'}") || 
         die "ERROR: $sophomorix_config{'FILES'}{'USER_FILE'}{$filename}{'PATH_ABS_REPORT_OFFICE'} not found!";  
        &analyze_user_line($line,$file_abs,$filename,$school,$count,"utf8","yes","USER_FILE");
        close(USERSUTF8);
        close(REPORT);
    } else {
        print "        WARNING: $filename is not valid on this ",
              "server (skipping this line)\n";
    }
}


# --dump-files
if ($dump_files==1){
    &json_dump({json => $json,
                jsoninfo => "USERS",
                jsoncomment => "All users read from all user files",
                log_level => $Conf::log_level,
                hash_ref=>\%users_file,
                sophomorix_config=>\%sophomorix_config,
               });
}



# --follow
# show the line
if ($follow ne ""){

    my $identifier=$users_file{$follow_file}{'file'}{$follow_file}{$follow_line}{'identifier_ascii'};
    if (not defined $identifier){
        print "   * Line $follow_line in file $follow_file contains no usable data\n";
    } else {
        print "   * LINE ORIG: $users_file{$follow_file}{'identifier_ascii'}{$identifier}{LINE_OLD}\n";
        print "   * LINE UTF8: $users_file{$follow_file}{'identifier_ascii'}{$identifier}{LINE_NEW}\n";
    }
}

# set WARNING to exit if multile identifiers exist
foreach my $school (@opt_schoollist){
    foreach my $identifier_ascii (keys %{ $users_file{'LOOKUP'}{$school}{'identifier_ascii'} } ) {
	my $count=$users_file{'LOOKUP'}{$school}{'identifier_ascii'}{$identifier_ascii}{'COUNT'};
	my @list=();
	if ($count>1){
	    for (my $i=1;$i<=$count;$i++){
		my $file=$users_file{'LOOKUP'}{$school}{'identifier_ascii'}{$identifier_ascii}{$i}{'FILE'};
		my $class=$users_file{'LOOKUP'}{$school}{'identifier_ascii'}{$identifier_ascii}{$i}{'CLASS'};
		my $item="FILE:".$file."|CLASS:".$class;
		push @list,$item;
	    }
	    my $appendix=join(",",@list);
            my $message="Identifier ".$identifier_ascii." seen ".$count." times in school ".$school.
                        ": $appendix (please fix this in your files)";
            &result_sophomorix_add(\%sophomorix_result,"WARNING",-1,\@arguments,$message);
	}
    }
}


# After reading all files, exit if there are errors
&result_sophomorix_check_exit(\%sophomorix_result,\%sophomorix_config,$json);



# ============================================================
# asking AD for users
# ============================================================
&print_title("Searching AD for users ...");
# fetch AD data
my ($ref_AD_check) = &AD_get_AD_for_check({ldap=>$ldap,
                                           root_dse=>$root_dse,
                                           root_dns=>$root_dns,
                                           admins=>"FALSE",
                                           sophomorix_config=>\%sophomorix_config,
                                         });


# --dump-ad
if ($dump_ad==1){
    &json_dump({json => $json,
                jsoninfo => "SEARCH",
                jsoncomment => "AD Content",
                log_level => $Conf::log_level,
                hash_ref=>$ref_AD_check,
                sophomorix_config=>\%sophomorix_config,
              });
}



my ($ref_login_avoid) = &get_login_avoid(\%sophomorix_config);
# --dump-login-avoid
if ($dump_login_avoid==1){
    &json_dump({json => $json,
                jsoninfo => "LOGIN_AVOID",
                jsoncomment => "Logins to avoid from log files",
                log_level => $Conf::log_level,
                hash_ref=>$ref_login_avoid,
                sophomorix_config=>\%sophomorix_config,
              });
}


# ============================================================
# Find Matches: Walk through all schools and identifiers in %users_file
# ============================================================
# AD data is in: $ref_AD_check
# file data is in %users_file

# dynamic matching data ist contained in %Match
# reading user files populates these subhashes (school-> identifiers):
#    $Match{$school}{'file_dynamic_identifiers_ascii'}{$filename} 
#    $users_file{$school}{'file_dynamic_identifiers_utf8'}
# and populated from AD by subs (AD identifiers): 
# $Match{$school}{'ad_dynamic_identifiers_ascii'} $Match{$school}{'ad_dynamic_identifiers_utf8'}
foreach my $school (@opt_schoollist) {
    # no migration
    #&populate_ad_dynamic_identifiers_utf8();

    # migration
    &populate_ad_dynamic_identifiers_ascii($school);
    # find exact matches
    print "\n";
    &print_title("### School $school: Matching lines in files to users in AD ...");
    &find_unid_matches($school);
    &find_ascii_matches($school);
    # find approx matches
    for (my $i=1;$i<$opt_edit_distance+1;$i++){
        my ($return,
            $line_count_school,
            $school,
            $matched_count_school,
            $not_matched_count_school)=&find_ascii_approx_matches($school,$i);
	if ($return>0){
	    $i=$opt_edit_distance+1;
            &print_title("approxMATCH: $line_count_school CHECKED in $school, MATCH: $matched_count_school (Edit distance: $return)","slim");
	}
    }

    ############################################################
    # Users not in AD: sophomorix.add
    ############################################################
    foreach my $line_new ( @{ $Match{$school}{'lines_with_filename'} } ){
        my ($file,$line)=split(/::/,$line_new);
        if (not defined $Match{$school}{'ACTION'}{$line}{'sAMAccountName'}){
            &write_add_file($school,$line_new);
        }
    }

    ############################################################
    # Users in AD: sophomorix.update/sophomorix.kill/sophomorix.nochange
    ############################################################
    foreach my $sam ( keys %{ $ref_AD_check->{'LOOKUP_by_school'}{$school}{'user_BY_sAMAccountName'} } ) {
        # test all users in AD
        my $update=0; # if > 0 write to sophomorix.update
        my $kill=0; # if > 0 write to sophomorix.kill
        my $old_unid=$ref_AD_check->{'sAMAccountName'}{$sam}{'sophomorixUnid'};
        my $new_unid="---";
        my $old_surname_ascii=$ref_AD_check->{'sAMAccountName'}{$sam}{'sophomorixSurnameASCII'};
        my $new_surname_ascii="---";
        my $old_firstname_ascii=$ref_AD_check->{'sAMAccountName'}{$sam}{'sophomorixFirstnameASCII'};
        my $new_firstname_ascii="---";
        my $old_birthdate=$ref_AD_check->{'sAMAccountName'}{$sam}{'sophomorixBirthdate'};
        my $new_birthdate="---";
        my $old_surname_utf8=$ref_AD_check->{'sAMAccountName'}{$sam}{'sn'};
        my $new_surname_utf8="---";
        my $old_firstname_utf8=$ref_AD_check->{'sAMAccountName'}{$sam}{'givenName'};
        my $new_firstname_utf8="---";

        my $old_surname_initial_utf8=$ref_AD_check->{'sAMAccountName'}{$sam}{'sophomorixSurnameInitial'};
        my $new_surname_initial_utf8="---";
        my $old_firstname_initial_utf8=$ref_AD_check->{'sAMAccountName'}{$sam}{'sophomorixFirstnameInitial'};
        my $new_firstname_initial_utf8="---";

        my $old_filename=$ref_AD_check->{'sAMAccountName'}{$sam}{'sophomorixAdminFile'};
        my $new_filename="---";
        my $old_status=$ref_AD_check->{'sAMAccountName'}{$sam}{'sophomorixStatus'};
        my $new_status="---";
        my $old_role=$ref_AD_check->{'sAMAccountName'}{$sam}{'sophomorixRole'};
        my $new_role="---";

        my $old_school=$ref_AD_check->{'sAMAccountName'}{$sam}{'sophomorixSchoolname'};
        my $new_school="---";

        # dirty fix to have class without school
        my $old_class=$ref_AD_check->{'sAMAccountName'}{$sam}{'sophomorixAdminClass'};
        $old_class=~s/^$old_school-//;
        my $old_class_with_school=$ref_AD_check->{'sAMAccountName'}{$sam}{'sophomorixAdminClass'};
        my $new_class="---";

        my $old_homedirectory=$ref_AD_check->{'sAMAccountName'}{$sam}{'homeDirectory'};;
        my $new_homedirectory="---";
    
        ############################################################
        # This might be temporary
        my $old_homedirectory_rel=$ref_AD_check->{'sAMAccountName'}{$sam}{'sophomorixIntrinsic2'};;
        my $new_homedirectory_rel="---";
    
        ############################################################

        my $old_mail=$ref_AD_check->{'sAMAccountName'}{$sam}{'mail'};
        my $new_mail="---";
    
        # old webui string is sorted already
        my $old_webui_string=join(",",@{ $ref_AD_check->{'sAMAccountName'}{$sam}{'sophomorixWebuiPermissionsCalculated'} });
        # new webui string
        my $new_webui_string="---";

        &test_attic_move($school,$sam,\$old_class,\$new_class,\$old_status,\%sophomorix_config);
        ##################################################

        ##################################################
        # check status update for users in AD
        &test_UEAS_TM_update($school,$sam,\$update,\$new_status);
        &test_TM_DA_update($school,$sam,\$update,\$new_status);
        &test_DL_RA_update($school,$sam,\$update,\$new_status);
        &test_R_A_update($school,$sam,\$update,\$new_status);

        if (exists $Match{$school}{'action_BY_sAMAccountName'}{$sam}){
            # check identifier update for matched users in AD
            &test_user_update($school,
                              $sam,
                              \%sophomorix_config,
                              \$update,
                              \$new_unid,
                              \$old_surname_ascii,
                              \$new_surname_ascii,
                              \$old_firstname_ascii,
                              \$new_firstname_ascii,
                              \$new_birthdate,
                              \$new_surname_utf8,
                              \$new_firstname_utf8,
                              \$new_surname_initial_utf8,
                              \$new_firstname_initial_utf8,
                              \$new_filename,
                              \$new_role,
                              \$new_class,
	                      \$new_school,
                              \$old_mail,
                              \$new_mail,
                              \$old_webui_string,
                              \$new_webui_string,
#                          \$old_homedirectory,
                              \$new_homedirectory,
#                          \$old_homedirectory_rel,
                              \$new_homedirectory_rel,
                             );
        }
        if ($update>0){
	    if (not defined $old_homedirectory_rel){
		# avoid undefined value for users created with old version of sophomorix
                $old_homedirectory_rel="";
            }
            # write a line to sophomorix.update
            my $update_line=$sam."::".
                            $old_unid."::".
                            $new_unid."::".
                            $old_surname_ascii."::".
                            $new_surname_ascii."::".
                            $old_firstname_ascii."::".
                            $new_firstname_ascii."::".
                            $old_birthdate."::".
                            $new_birthdate."::".
                            $old_surname_utf8."::".
                            $new_surname_utf8."::".
                            $old_firstname_utf8."::".
                            $new_firstname_utf8."::".
                            $old_filename."::".
                            $new_filename."::".
                            $old_status."::".
                            $new_status."::".
                            $old_role."::".
                            $new_role."::".
                            $old_class."::".
                            $new_class."::".
                            $old_school."::".
                            $new_school."::".
                            $old_surname_initial_utf8."::".
                            $new_surname_initial_utf8."::".
                            $old_firstname_initial_utf8."::".
                            $new_firstname_initial_utf8."::".
                            $old_mail."::".
                            $new_mail."::".
                            $old_webui_string."::".
                            $new_webui_string."::".
                            $old_homedirectory."::".
                            $new_homedirectory."::".
                            $old_homedirectory_rel."::".
                            $new_homedirectory_rel."::".
                            "\n";
            print UPDATE $update_line;
            # create result
            push @{ $sophomorix_result{'CHECK_RESULT'}{'UPDATELIST'} }, $sam;
            if ($new_unid ne "---"){
                $sophomorix_result{'CHECK_RESULT'}{'UPDATE'}{$sam}{'sophomorixUnid'}{'OLD'}=$old_unid;
                $sophomorix_result{'CHECK_RESULT'}{'UPDATE'}{$sam}{'sophomorixUnid'}{'NEW'}=$new_unid;
            }
            if ($new_surname_ascii ne "---"){
                $sophomorix_result{'CHECK_RESULT'}{'UPDATE'}{$sam}{'sophomorixSurnameASCII'}{'OLD'}=$old_surname_ascii;
                $sophomorix_result{'CHECK_RESULT'}{'UPDATE'}{$sam}{'sophomorixSurnameASCII'}{'NEW'}=$new_surname_ascii;
            }
            if ($new_firstname_ascii ne "---"){
                $sophomorix_result{'CHECK_RESULT'}{'UPDATE'}{$sam}{'sophomorixFirstnameASCII'}{'OLD'}=$old_firstname_ascii;
                $sophomorix_result{'CHECK_RESULT'}{'UPDATE'}{$sam}{'sophomorixFirstnameASCII'}{'NEW'}=$new_firstname_ascii;
            }
            if ($new_birthdate ne "---"){
                $sophomorix_result{'CHECK_RESULT'}{'UPDATE'}{$sam}{'sophomorixBirthdate'}{'OLD'}=$old_birthdate;
                $sophomorix_result{'CHECK_RESULT'}{'UPDATE'}{$sam}{'sophomorixBirthdate'}{'NEW'}=$new_birthdate;
            }
            if ($new_surname_utf8 ne "---"){
                $sophomorix_result{'CHECK_RESULT'}{'UPDATE'}{$sam}{'sn'}{'OLD'}=$old_surname_utf8;
                $sophomorix_result{'CHECK_RESULT'}{'UPDATE'}{$sam}{'sn'}{'NEW'}=$new_surname_utf8;
            }
            if ($new_firstname_utf8 ne "---"){
                $sophomorix_result{'CHECK_RESULT'}{'UPDATE'}{$sam}{'givenName'}{'OLD'}=$old_firstname_utf8;
                $sophomorix_result{'CHECK_RESULT'}{'UPDATE'}{$sam}{'givenName'}{'NEW'}=$new_firstname_utf8;
            }
            if ($new_filename ne "---"){
                $sophomorix_result{'CHECK_RESULT'}{'UPDATE'}{$sam}{'sophomorixAdminFile'}{'OLD'}=$old_filename;
                $sophomorix_result{'CHECK_RESULT'}{'UPDATE'}{$sam}{'sophomorixAdminFile'}{'NEW'}=$new_filename;
            }
            if ($new_status ne "---"){
                $sophomorix_result{'CHECK_RESULT'}{'UPDATE'}{$sam}{'sophomorixStatus'}{'OLD'}=$old_status;
                $sophomorix_result{'CHECK_RESULT'}{'UPDATE'}{$sam}{'sophomorixStatus'}{'NEW'}=$new_status;
            }
            if ($new_role ne "---"){
                $sophomorix_result{'CHECK_RESULT'}{'UPDATE'}{$sam}{'sophomorixRole'}{'OLD'}=$old_role;
                $sophomorix_result{'CHECK_RESULT'}{'UPDATE'}{$sam}{'sophomorixRole'}{'NEW'}=$new_role;
            }
            if ($new_class ne "---"){
                $sophomorix_result{'CHECK_RESULT'}{'UPDATE'}{$sam}{'sophomorixAdminClass'}{'OLD'}=$old_class;
                $sophomorix_result{'CHECK_RESULT'}{'UPDATE'}{$sam}{'sophomorixAdminClass'}{'NEW'}=$new_class;
            }
            if ($new_school ne "---"){
                $sophomorix_result{'CHECK_RESULT'}{'UPDATE'}{$sam}{'sophomorixSchoolname'}{'OLD'}=$old_school;
                $sophomorix_result{'CHECK_RESULT'}{'UPDATE'}{$sam}{'sophomorixSchoolname'}{'NEW'}=$new_school;
            }
            $counters{'UPDATE'}++;
        }

        &test_RK_kill($school,$sam,\$kill);
        if ($kill>0){
            # write to sophomorix.kill
            my $kill_line=$ref_AD_check->{'sAMAccountName'}{$sam}{'IDENTIFIER_ASCII'}."::".
                          $sam."::".
                          $ref_AD_check->{'sAMAccountName'}{$sam}{'sophomorixAdminClass'}."::".
                          $ref_AD_check->{'sAMAccountName'}{$sam}{'sophomorixSchoolname'}."::\n";

            # creating the line
            my $kill_line=
                $ref_AD_check->{'sAMAccountName'}{$sam}{'IDENTIFIER_ASCII'}."::".
                $sam."::".
                $ref_AD_check->{'sAMAccountName'}{$sam}{'sophomorixAdminClass'}."::".
                $ref_AD_check->{'sAMAccountName'}{$sam}{'sophomorixSchoolname'}."::\n";
            print KILL $kill_line;
            # create result
            push @{ $sophomorix_result{'CHECK_RESULT'}{'KILLLIST'} }, $sam;
            $sophomorix_result{'CHECK_RESULT'}{'KILL'}{$sam}{'IDENTIFIER_ASCII'}=
               $ref_AD_check->{'sAMAccountName'}{$sam}{'IDENTIFIER_ASCII'};
            $sophomorix_result{'CHECK_RESULT'}{'KILL'}{$sam}{'sAMAccountName'}=$sam;
            $sophomorix_result{'CHECK_RESULT'}{'KILL'}{$sam}{'sophomorixAdminClass'}=
               $ref_AD_check->{'sAMAccountName'}{$sam}{'sophomorixAdminClass'};
            $sophomorix_result{'CHECK_RESULT'}{'KILL'}{$sam}{'sophomorixSchoolname'}=
               $ref_AD_check->{'sAMAccountName'}{$sam}{'sophomorixSchoolname'};
            $counters{'KILL'}++;
        }

        # neither updated nor killed: nochange
        if ($update==0 and $kill==0){
            # write to sophomorix.nochange
            my $nochange_line=$sam."::".
                              $ref_AD_check->{'sAMAccountName'}{$sam}{'sn'}."::".
                              $ref_AD_check->{'sAMAccountName'}{$sam}{'givenName'}."::".
                              $ref_AD_check->{'sAMAccountName'}{$sam}{'sophomorixBirthdate'}."::".
                              $ref_AD_check->{'sAMAccountName'}{$sam}{'sophomorixStatus'}."::".
                              $ref_AD_check->{'sAMAccountName'}{$sam}{'sophomorixAdminClass'}."::".
                              $ref_AD_check->{'sAMAccountName'}{$sam}{'sophomorixAdminFile'}."::".
                              "\n";
            print NOCHANGE $nochange_line;
            #push @{ $sophomorix_result{'CHECK_RESULT'}{'NOCHANGELIST'} }, $sam;
            $counters{'NOCHANGE'}++;
        }
    }
    &print_title("## School $school ... Done!");
}



close(ADD);
close(UPDATE);
close(KILL);
close(NOCHANGE);
close(ERROR);


# --dump-matches
if ($dump_matches==1){
    &json_dump({json => $json,
                jsoninfo => "MATCH",
                jsoncomment => "File user -> AD matches",
                log_level => $Conf::log_level,
                hash_ref=>\%Match,
                sophomorix_config=>\%sophomorix_config,
              });
}


############################################################
# create overview in %sophomorix_result
############################################################
&result_sophomorix_add_summary({
                     NAME=>"HEADER", 
                     TITLE=>"Overview of what can be done:", 
                     sophomorix_result=>\%sophomorix_result,
			       });

&result_sophomorix_add_summary({
                     NAME=>"ADD", 
                     RESULT=>$counters{'ADD'}, 
                     RESULT_TYPE => "integer",
                     DESCRIPTION_POST => "users can be added in ".basename($add_file), 
                     DESCRIPTION_PRE => "users in ".basename($add_file), 
                     FORMAT_TYPE => 1,
                     sophomorix_result=>\%sophomorix_result,
			       });

&result_sophomorix_add_summary({
                     NAME=>"UPDATE", 
                     RESULT=>$counters{'UPDATE'}, 
                     RESULT_TYPE => "integer",
                     DESCRIPTION_POST => "users can be updated in ".basename($update_file), 
                     DESCRIPTION_PRE => "users in ".basename($update_file), 
                     FORMAT_TYPE => 1,
                     sophomorix_result=>\%sophomorix_result,
			       });

&result_sophomorix_add_summary({
                     NAME=>"KILL", 
                     RESULT=>$counters{'KILL'}, 
                     RESULT_TYPE => "integer",
                     DESCRIPTION_POST => "users can be killed in ".basename($kill_file), 
                     DESCRIPTION_PRE => "users in ".basename($kill_file), 
                     FORMAT_TYPE => 1,
                     sophomorix_result=>\%sophomorix_result,
			       });

&result_sophomorix_add_summary({
                     NAME=>"NOCHANGE", 
                     RESULT=>$counters{'NOCHANGE'}, 
                     RESULT_TYPE => "integer",
                     DESCRIPTION_POST => "users are not to be changed in ".basename($nochange_file), 
                     DESCRIPTION_PRE => "users in ".basename($nochange_file), 
                     FORMAT_TYPE => 1,
                     sophomorix_result=>\%sophomorix_result,
			       });

&result_sophomorix_add_summary({
                     NAME=>"ERROR", 
                     RESULT=>$counters{'ERROR'}, 
                     RESULT_TYPE => "integer",
                     DESCRIPTION_POST => "users with errors in ".basename($error_file), 
                     DESCRIPTION_PRE => "users in ".basename($error_file), 
                     FORMAT_TYPE => 1,
                     sophomorix_result=>\%sophomorix_result,
			       });

# sort the result lists
if ($#{ $sophomorix_result{'CHECK_RESULT'}{'ADDLIST'} }>0){
    @{ $sophomorix_result{'CHECK_RESULT'}{'ADDLIST'} } = 
        sort @{ $sophomorix_result{'CHECK_RESULT'}{'ADDLIST'} };
}
if ($#{ $sophomorix_result{'CHECK_RESULT'}{'UPDATELIST'} }>0){
    @{ $sophomorix_result{'CHECK_RESULT'}{'UPDATELIST'} } = 
        sort @{ $sophomorix_result{'CHECK_RESULT'}{'UPDATELIST'} };
}
if ($#{ $sophomorix_result{'CHECK_RESULT'}{'KILLLIST'} }>0){
    @{ $sophomorix_result{'CHECK_RESULT'}{'KILLLIST'} } = 
        sort @{ $sophomorix_result{'CHECK_RESULT'}{'KILLLIST'} };
}
if ($#{ $sophomorix_result{'CHECK_RESULT'}{'ERRORLIST'} }>0){
    @{ $sophomorix_result{'CHECK_RESULT'}{'ERRORLIST'} } = 
        sort @{ $sophomorix_result{'CHECK_RESULT'}{'ERRORLIST'} };
}

if (exists $ref_AD_check->{'WARNINGS'}){
    # show warnings
    print "\nWARNINGS:\n";
    print Dumper ($ref_AD_check->{'WARNINGS'});
}

&AD_unbind_admin($ldap);
&log_script_end(\@arguments,\%sophomorix_result,\%sophomorix_config,$json);


################################################################################
# Sub
################################################################################

# test status updates
################################################################################

sub test_UEAS_TM_update{
    my ($school,$sam,$update_ref,$new_status_ref)=@_;
    my $new_line;
    if (exists $Match{$school}{'action_BY_sAMAccountName'}{$sam}){
        $new_line=$Match{$school}{'action_BY_sAMAccountName'}{$sam};
    } else {
        $new_line="---";
    }
    if ( $ref_AD_check->{'sAMAccountName'}{$sam}{'sophomorixStatus'} eq "U" or
             $ref_AD_check->{'sAMAccountName'}{$sam}{'sophomorixStatus'} eq "E" or
             $ref_AD_check->{'sAMAccountName'}{$sam}{'sophomorixStatus'} eq "A" or
             $ref_AD_check->{'sAMAccountName'}{$sam}{'sophomorixStatus'} eq "S"
           ){
        # update to T if not matched to a user in files
        if (not exists $Match{$school}{'action_BY_sAMAccountName'}{$sam}){
            if($Conf::log_level>=2){
                print "   * $sam: $ref_AD_check->{'sAMAccountName'}{$sam}{'sophomorixStatus'} -> T\n";
	    }
            ${$new_status_ref}="T";
            ${$update_ref}++;
        } else {
            # existing from test_attic_move
            if (exists $Match{$school}{'ACTION'}{$new_line}{'NOT_IN_FILES'} and 
                $Match{$school}{'ACTION'}{$new_line}{'NOT_IN_FILES'} eq "TRUE"
               ){
                if($Conf::log_level>=2){
                    print "   * $sam: $ref_AD_check->{'sAMAccountName'}{$sam}{'sophomorixStatus'} -> T\n";
	        }
                ${$new_status_ref}="T";
                ${$update_ref}++;
            }
        }
    }
}



sub test_TM_DA_update{
    my ($school,$sam,$update_ref,$new_status_ref)=@_;
    my $new_line;
    if (exists $Match{$school}{'action_BY_sAMAccountName'}{$sam}){
        $new_line=$Match{$school}{'action_BY_sAMAccountName'}{$sam};
    } else {
        $new_line="---";
    }

    if ( $ref_AD_check->{'sAMAccountName'}{$sam}{'sophomorixStatus'} eq "T" or 
         $ref_AD_check->{'sAMAccountName'}{$sam}{'sophomorixStatus'} eq "M"
       ){
        if($Conf::log_level>=2){
            print "* Testing $sam: $ref_AD_check->{'sAMAccountName'}{$sam}{'sophomorixStatus'},",
                  " TolerationDate: $ref_AD_check->{'sAMAccountName'}{$sam}{'sophomorixTolerationDate'}\n";
        }
        my $file=$ref_AD_check->{'sAMAccountName'}{$sam}{'sophomorixAdminFile'};
        my $toleration_date_epoch=&ymdhms_to_epoch($ref_AD_check->{'sAMAccountName'}{$sam}{'sophomorixTolerationDate'});
        my $deactivation_date_epoch=&ymdhms_to_epoch($ref_AD_check->{'sAMAccountName'}{$sam}{'sophomorixDeactivationDate'});
        my $toleration_time_days;
        if (defined $sophomorix_config{'FILES'}{'USER_FILE'}{$file}{'TOLERATION_TIME'} ){
            $toleration_time_days=$sophomorix_config{'FILES'}{'USER_FILE'}{$file}{'TOLERATION_TIME'};
        } else {
            # for migration, high enough to be sure it is not updated
            $toleration_time_days=1000;
        }
        my $toleration_end_epoch=$toleration_date_epoch+86400*$toleration_time_days;
        if($Conf::log_level>=2){
            print "    Toleration end: $toleration_end_epoch ($ref_AD_check->{'sAMAccountName'}{$sam}{'sophomorixTolerationDate'}",
                  " +  $toleration_time_days days)\n";
            print "    Today:          $sophomorix_config{'DATE'}{'LOCAL'}{'EPOCH'}\n";
        }
        if (
               (not exists $Match{$school}{'action_BY_sAMAccountName'}{$sam} and 
                $sophomorix_config{'DATE'}{'LOCAL'}{'EPOCH'} > $toleration_end_epoch 
               ) or (
                exists $Match{$school}{'action_BY_sAMAccountName'}{$sam} and 
                defined $Match{$school}{'ACTION'}{$new_line}{'NOT_IN_FILES'} and
                $Match{$school}{'ACTION'}{$new_line}{'NOT_IN_FILES'} eq "TRUE"
               ) or (
                exists $Match{$school}{'action_BY_sAMAccountName'}{$sam} and 
                defined $Match{$school}{'ACTION'}{$new_line}{'NOT_IN_FILES'} and
                $Match{$school}{'ACTION'}{$new_line}{'NOT_IN_FILES'} eq "TRUE"
               )
            ){
            # update to D if 
            #     - not matched to a user in files
            #     - and toleration time expired
            if ($sophomorix_config{'DATE'}{'LOCAL'}{'EPOCH'} > $deactivation_date_epoch and
                $ref_AD_check->{'sAMAccountName'}{$sam}{'sophomorixStatus'} eq "M"
                ){
                print "    ----> $sam: $ref_AD_check->{'sAMAccountName'}{$sam}{'sophomorixStatus'} -> D\n";
                ${$new_status_ref}="D";
                ${$update_ref}++;
            } elsif ($sophomorix_config{'DATE'}{'LOCAL'}{'EPOCH'} > $toleration_end_epoch and
                $ref_AD_check->{'sAMAccountName'}{$sam}{'sophomorixStatus'} eq "T"
		){
                print "    ----> $sam: $ref_AD_check->{'sAMAccountName'}{$sam}{'sophomorixStatus'} -> D\n";
                ${$new_status_ref}="D";
                ${$update_ref}++;
            } else {
                # do nothing
	    }
	} elsif (exists $Match{$school}{'action_BY_sAMAccountName'}{$sam} and
                 not defined $Match{$school}{'ACTION'}{$new_line}{'NOT_IN_FILES'}
                ){
            # update to A if 
            #     - matched to a user in files
            if($Conf::log_level>=2){
                print "    ----> $sam: $ref_AD_check->{'sAMAccountName'}{$sam}{'sophomorixStatus'} -> A\n";
            }
            ${$new_status_ref}="A";
            ${$update_ref}++;
	} else {
            # do nothing
            if($Conf::log_level>=2){
                print "    ----> Toleration end not reached\n";
            }
        }
    }
}



sub test_DL_RA_update{
    my ($school,$sam,$update_ref,$new_status_ref)=@_;
    my $new_line;
    if (exists $Match{$school}{'action_BY_sAMAccountName'}{$sam}){
        $new_line=$Match{$school}{'action_BY_sAMAccountName'}{$sam};
    } else {
        $new_line="---";
    }

    if ( $ref_AD_check->{'sAMAccountName'}{$sam}{'sophomorixStatus'} eq "D" or
        $ref_AD_check->{'sAMAccountName'}{$sam}{'sophomorixStatus'} eq "L"
       ){
        if($Conf::log_level>=2){
            print "* Testing $sam: $ref_AD_check->{'sAMAccountName'}{$sam}{'sophomorixStatus'},",
                  " DeactivationDate: $ref_AD_check->{'sAMAccountName'}{$sam}{'sophomorixDeactivationDate'}\n";
        }
        my $file=$ref_AD_check->{'sAMAccountName'}{$sam}{'sophomorixAdminFile'};
        my $deactivation_date_epoch=&ymdhms_to_epoch($ref_AD_check->{'sAMAccountName'}{$sam}{'sophomorixDeactivationDate'});
        my $deactivation_time_days;
        if (defined $sophomorix_config{'FILES'}{'USER_FILE'}{$file}{'DEACTIVATION_TIME'} ){
            $deactivation_time_days=$sophomorix_config{'FILES'}{'USER_FILE'}{$file}{'DEACTIVATION_TIME'};
        } else {
            # for migration, high enough to be sure it is not updated
            $deactivation_time_days=1000;
        }
        my $deactivation_end_epoch=$deactivation_date_epoch+86400*$deactivation_time_days;
        if($Conf::log_level>=2){
            print "    Deactivation end: $deactivation_end_epoch ($ref_AD_check->{'sAMAccountName'}{$sam}{'sophomorixDeactivationDate'}",
                  " + $deactivation_time_days days)\n";
            print "    Today:            $sophomorix_config{'DATE'}{'LOCAL'}{'EPOCH'}\n";
        }
        ########################################
        if (
               ( not exists $Match{$school}{'action_BY_sAMAccountName'}{$sam} and 
                 $sophomorix_config{'DATE'}{'LOCAL'}{'EPOCH'} > $deactivation_end_epoch
               ) or (
                 exists $Match{$school}{'action_BY_sAMAccountName'}{$sam} and
                 defined $Match{$school}{'ACTION'}{$new_line}{'NOT_IN_FILES'} and
                 $Match{$school}{'ACTION'}{$new_line}{'NOT_IN_FILES'} eq "TRUE" and 
                 $sophomorix_config{'DATE'}{'LOCAL'}{'EPOCH'} > $deactivation_end_epoch
               )
           ){
            # update to R/put in kill-file  if 
            #     - not matched to a user in files
            #     - and deactivation time expired
            # 1) update status to R
            if($Conf::log_level>=2){
                print "    ----> $sam: $ref_AD_check->{'sAMAccountName'}{$sam}{'sophomorixStatus'} -> R\n";
	    }
            ${$new_status_ref}="R";
            ${$update_ref}++;
            # 2) put user in kill file
            if($Conf::log_level>=2){
                print "    ----> $sam is put into kill file\n";
	    }
            # creating the line
            my $kill_line=
                $ref_AD_check->{'sAMAccountName'}{$sam}{'IDENTIFIER_ASCII'}."::".
                $sam."::".
                $ref_AD_check->{'sAMAccountName'}{$sam}{'sophomorixAdminClass'}."::".
                $ref_AD_check->{'sAMAccountName'}{$sam}{'sophomorixSchoolname'}."::\n";               
            print KILL $kill_line;
            # create result
            push @{ $sophomorix_result{'CHECK_RESULT'}{'KILLLIST'} }, $sam;
            $sophomorix_result{'CHECK_RESULT'}{'KILL'}{$sam}{'IDENTIFIER_ASCII'}=
               $ref_AD_check->{'sAMAccountName'}{$sam}{'IDENTIFIER_ASCII'};
            $sophomorix_result{'CHECK_RESULT'}{'KILL'}{$sam}{'sAMAccountName'}=$sam;
            $sophomorix_result{'CHECK_RESULT'}{'KILL'}{$sam}{'sophomorixAdminClass'}=
               $ref_AD_check->{'sAMAccountName'}{$sam}{'sophomorixAdminClass'};
            $sophomorix_result{'CHECK_RESULT'}{'KILL'}{$sam}{'sophomorixSchoolname'}=
               $ref_AD_check->{'sAMAccountName'}{$sam}{'sophomorixSchoolname'};
            $counters{'KILL'}++;
        } elsif (exists $Match{$school}{'action_BY_sAMAccountName'}{$sam} and 
                 $ref_AD_check->{'sAMAccountName'}{$sam}{'sophomorixStatus'} eq "D" and
                 not exists $Match{$school}{'ACTION'}{$new_line}{'NOT_IN_FILES'}
                ){
            # update to A if 
            #     - matched to a user in files
            #     - old status is D (not for status L)
            if($Conf::log_level>=2){
                print "    ----> $sam: $ref_AD_check->{'sAMAccountName'}{$sam}{'sophomorixStatus'} -> A\n";
	    }
            ${$new_status_ref}="A";
            ${$update_ref}++;
        ########################################
	} else {
            if($Conf::log_level>=2){
                print "    ----> Deactivation end not reached\n";
  	    }
        }
     }
}



sub test_R_A_update{
    my ($school,$sam,$update_ref,$new_status_ref)=@_;
    my $new_line;
    if (exists $Match{$school}{'action_BY_sAMAccountName'}{$sam}){
        $new_line=$Match{$school}{'action_BY_sAMAccountName'}{$sam};
    } else {
        $new_line="---";
    }
    if ( $ref_AD_check->{'sAMAccountName'}{$sam}{'sophomorixStatus'} eq "R" and 
        exists $Match{$school}{'action_BY_sAMAccountName'}{$sam} and 
        not exists $Match{$school}{'ACTION'}{$new_line}{'NOT_IN_FILES'}
       ){
        # update R to A if 
        #     - matched to a user in files
        if($Conf::log_level>=2){
            print "    ----> $sam: $ref_AD_check->{'sAMAccountName'}{$sam}{'sophomorixStatus'} -> A\n";
	}
        ${$new_status_ref}="A";
        ${$update_ref}++;
    }
}



sub test_RK_kill{
    my ($school,$sam,$kill_ref)=@_;
    my $new_line;
    if (exists $Match{$school}{'action_BY_sAMAccountName'}{$sam}){
        $new_line=$Match{$school}{'action_BY_sAMAccountName'}{$sam};
    } else {
        $new_line="---";
    }
    if ( ($ref_AD_check->{'sAMAccountName'}{$sam}{'sophomorixStatus'} eq "K" and 
          not exists $Match{$school}{'ACTION'}{$new_line}{'NOT_IN_FILES'}
       ) or (
          $ref_AD_check->{'sAMAccountName'}{$sam}{'sophomorixStatus'} eq "K" and
          defined $Match{$school}{'ACTION'}{$new_line}{'NOT_IN_FILES'} and
          $Match{$school}{'ACTION'}{$new_line}{'NOT_IN_FILES'} eq "TRUE"
       ) or (
          $ref_AD_check->{'sAMAccountName'}{$sam}{'sophomorixStatus'} eq "R" and
          defined $Match{$school}{'ACTION'}{$new_line}{'NOT_IN_FILES'} and
          $Match{$school}{'ACTION'}{$new_line}{'NOT_IN_FILES'} eq "TRUE"
       ) or (
          $ref_AD_check->{'sAMAccountName'}{$sam}{'sophomorixStatus'} eq "R" 
          and not exists $Match{$school}{'action_BY_sAMAccountName'}{$sam}
         )
       ){
        # kill if status is  R or K
        if($Conf::log_level>=2){
            print "    ----> $sam: $ref_AD_check->{'sAMAccountName'}{$sam}{'sophomorixStatus'} -> sophomorix.kill\n";
        }
        ${$kill_ref}++;
    }
}



# test class movement (moves previously unmoved classes due to a bug prior to 3.71.1)
################################################################################
sub test_attic_move {
    my ($school,$sam,$old_class_ref,$new_class_ref,$old_status_ref,$ref_sophomorix_config)=@_;
    # not matched (=not in files anymore), but also not in attic -> move to attic
    # 3.74.5. do not move status M to attic
    if (not exists $Match{$school}{'action_BY_sAMAccountName'}{$sam} and
        ${$old_class_ref} ne $ref_sophomorix_config->{'INI'}{'VARS'}{'ATTIC_GROUP_BASENAME'} and
        not ${$old_status_ref} eq "M"
       ){
        ${$new_class_ref}=$ref_sophomorix_config->{'INI'}{'VARS'}{'ATTIC_GROUP_BASENAME'};
        my $new_line=${$new_class_ref}.";".$ref_AD_check->{'sAMAccountName'}{$sam}{'IDENTIFIER_ASCII'}.";---";

        # add ACTION data for moving to 'attic'-group
        $Match{$school}{'action_BY_sAMAccountName'}{$sam}=$new_line;
        $Match{$school}{'ACTION'}{$new_line}{'sAMAccountName'}=$sam;
        $Match{$school}{'ACTION'}{$new_line}{'identifier_ascii'}=$ref_AD_check->{'sAMAccountName'}{$sam}{'IDENTIFIER_ASCII'};
        $Match{$school}{'ACTION'}{$new_line}{'class'}=${$new_class_ref};
        $Match{$school}{'ACTION'}{$new_line}{'surname_ascii'}=$ref_AD_check->{'sAMAccountName'}{$sam}{'sophomorixSurnameASCII'};
        $Match{$school}{'ACTION'}{$new_line}{'surname_utf8'}=$ref_AD_check->{'sAMAccountName'}{$sam}{'sn'};
        $Match{$school}{'ACTION'}{$new_line}{'firstname_ascii'}=$ref_AD_check->{'sAMAccountName'}{$sam}{'sophomorixFirstnameASCII'};
        $Match{$school}{'ACTION'}{$new_line}{'firstname_utf8'}=$ref_AD_check->{'sAMAccountName'}{$sam}{'givenName'};
        $Match{$school}{'ACTION'}{$new_line}{'birthdate'}=$ref_AD_check->{'sAMAccountName'}{$sam}{'sophomorixBirthdate'};
        $Match{$school}{'ACTION'}{$new_line}{'unid'}=$ref_AD_check->{'sAMAccountName'}{$sam}{'sophomorixUnid'};

        $Match{$school}{'ACTION'}{$new_line}{'school'}=$ref_AD_check->{'sAMAccountName'}{$sam}{'sophomorixSchoolname'};
        $Match{$school}{'ACTION'}{$new_line}{'filename'}=$ref_AD_check->{'sAMAccountName'}{$sam}{'sophomorixAdminFile'};
        $Match{$school}{'ACTION'}{$new_line}{'status'}=$ref_AD_check->{'sAMAccountName'}{$sam}{'sophomorixStatus'};
#        $Match{$school}{'ACTION'}{$new_line}{'status'}="KEEP_STATUS";
        $Match{$school}{'ACTION'}{$new_line}{'line_count'}=0;

        my $sub=(caller(0))[3];
        $Match{$school}{'ACTION'}{$new_line}{'MATCH_SUB'}="$sub";

        $Match{$school}{'ACTION'}{$new_line}{'class_type'}="???";
        $Match{$school}{'ACTION'}{$new_line}{'inject'}="???";
        $Match{$school}{'ACTION'}{$new_line}{'login'}="???";
        $Match{$school}{'ACTION'}{$new_line}{'NOT_IN_FILES'}="TRUE";

        # role in attic is student
        $Match{$school}{'ACTION'}{$new_line}{'role'}=$sophomorix_config{'INI'}{'ROLE_USER'}{'STUDENT'};
    }
}



# test user update (matched users only)
################################################################################
sub test_user_update {
#    my ($filename,
    my ($school,
        $sam,
	$ref_sophomorix_config,
        $update_ref,
        $new_unid_ref,
        $old_surname_ascii_ref,
        $new_surname_ascii_ref,
        $old_firstname_ascii_ref,
        $new_firstname_ascii_ref,
        $new_birthdate_ref,
        $new_surname_utf8_ref,
        $new_firstname_utf8_ref,
        $new_surname_initial_utf8_ref,
        $new_firstname_initial_utf8_ref,
        $new_filename_ref,
        $new_role_ref,
        $new_class_ref,
        $new_school_ref,
        $old_mail_ref,
        $new_mail_ref,
        $old_webui_string_ref,
        $new_webui_string_ref,
#        $old_homedirectory_ref,
        $new_homedirectory_ref,
        $new_homedirectory_rel_ref,
	)=@_;
    my $old_filename=$ref_AD_check->{'sAMAccountName'}{$sam}{'sophomorixAdminFile'};
    my $line_new=$Match{$school}{'action_BY_sAMAccountName'}{$sam};
    # count changes to attributes, 
    # update unid only when ALL other values are the same
    my $identifier_class_changes=0;

    # new_surname_ascii old
    if ( $Match{$school}{'ACTION'}{$line_new}{'surname_ascii'} eq 
          $ref_AD_check->{'sAMAccountName'}{$sam}{'sophomorixSurnameASCII'}
       ){
        # no update
        ${$new_surname_ascii_ref}="---";
    } else {
        # update
        ${$update_ref}++;
        ${$new_surname_ascii_ref}=$Match{$school}{'ACTION'}{$line_new}{'surname_ascii'};
        $identifier_class_changes++;
    }

    # new_surname_ascii new
#    if ( ($Match{'ACTION'}{$old_filename}{$line_new}{'surname_ascii'} eq 
#          $ref_AD_check->{'sAMAccountName'}{$sam}{'sophomorixSurnameASCII'})){
#        # no update
#        ${$new_surname_ascii_ref}="---";
#    } else {
#	print "HERE update $sam in $old_filename to ${$new_surname_ascii_ref}\n";
#       # update
#       ${$update_ref}++;
#       ${$new_surname_ascii_ref}=$Match{'ACTION'}{$old_filename}{$line_new}{'surname_ascii'};
#       $identifier_class_changes++;
#   }
  
    # new_firstname_ascii
    if ($Match{$school}{'ACTION'}{$line_new}{'firstname_ascii'} eq 
        $ref_AD_check->{'sAMAccountName'}{$sam}{'sophomorixFirstnameASCII'}
       ){
        # no update
        ${$new_firstname_ascii_ref}="---";
    } else {
        # update
        ${$update_ref}++;
        ${$new_firstname_ascii_ref}=$Match{$school}{'ACTION'}{$line_new}{'firstname_ascii'};
        $identifier_class_changes++;
    }

    # new_surname_utf8
    if ( ($Match{$school}{'ACTION'}{$line_new}{'surname_utf8'} eq $ref_AD_check->{'sAMAccountName'}{$sam}{'sn'})){
        # no update
        ${$new_surname_utf8_ref}="---";
        ${$new_surname_initial_utf8_ref}="---";
    } else {
        # update
        ${$update_ref}++;
        ${$new_surname_utf8_ref}=$Match{$school}{'ACTION'}{$line_new}{'surname_utf8'};
        ${$new_surname_initial_utf8_ref}=&Sophomorix::SophomorixBase::extract_initial($Match{$school}{'ACTION'}{$line_new}{'surname_utf8'});
        $identifier_class_changes++;
    }

    # new_firstname_utf8
    if ( ($Match{$school}{'ACTION'}{$line_new}{'firstname_utf8'} eq $ref_AD_check->{'sAMAccountName'}{$sam}{'givenName'} )){
        # no update
        ${$new_firstname_utf8_ref}="---";
        ${$new_firstname_initial_utf8_ref}="---";
    } else {
        # update
        ${$update_ref}++;
        ${$new_firstname_utf8_ref}=$Match{$school}{'ACTION'}{$line_new}{'firstname_utf8'};
        ${$new_firstname_initial_utf8_ref}=
            &Sophomorix::SophomorixBase::extract_initial($Match{$school}{'ACTION'}{$line_new}{'firstname_utf8'});
        $identifier_class_changes++;
    }

    # birthdate
    if ( ($Match{$school}{'ACTION'}{$line_new}{'birthdate'} eq $ref_AD_check->{'sAMAccountName'}{$sam}{'sophomorixBirthdate'} )){
        # no update
        ${$new_birthdate_ref}="---";
    } else {
        # update
        ${$update_ref}++;
        ${$new_birthdate_ref}=$Match{$school}{'ACTION'}{$line_new}{'birthdate'};
        $identifier_class_changes++;
    }

    # filename
    if ( ($Match{$school}{'ACTION'}{$line_new}{'filename'} eq $ref_AD_check->{'sAMAccountName'}{$sam}{'sophomorixAdminFile'} )){
        # no update
        ${$new_filename_ref}="---";
    } else {
        # update
        ${$update_ref}++;
        ${$new_filename_ref}=$Match{$school}{'ACTION'}{$line_new}{'filename'};
        $identifier_class_changes++;
    }

    # role
    if ( ($Match{$school}{'ACTION'}{$line_new}{'role'} eq $ref_AD_check->{'sAMAccountName'}{$sam}{'sophomorixRole'} )){
        # no update
        ${$new_role_ref}="---";
    } else {
        # update
        ${$update_ref}++;
        ${$new_role_ref}=$Match{$school}{'ACTION'}{$line_new}{'role'};
        $identifier_class_changes++;
    }

    # adminclass(with prefix) must be created from school and class
    my $adminclass;
    if ($adminclass=$Match{$school}{'ACTION'}{$line_new}{'school'} eq $DevelConf::name_default_school){
        # default-school
        $adminclass=$Match{$school}{'ACTION'}{$line_new}{'class'};
    } else {
        $adminclass=$Match{$school}{'ACTION'}{$line_new}{'school'}."-".$Match{$school}{'ACTION'}{$line_new}{'class'};
    }

    if ( ($adminclass eq $ref_AD_check->{'sAMAccountName'}{$sam}{'sophomorixAdminClass'} )){
        # no update
        ${$new_class_ref}="---";
    } else {
        # update
        # exit if new group is a user
        if (exists $ref_AD_check->{'sAMAccountName'}{$adminclass}){
	    my $error_message="User '".$sam."' should be moved from class '".
                  $ref_AD_check->{'sAMAccountName'}{$sam}{'sophomorixAdminClass'}.
                  "' to '".
	          $adminclass."' But '".$adminclass."' is a already a user ".
		"(Solution: Map groupname '".$adminclass."' to another name)";
	   &log_script_exit($error_message,1,1,0,
                     \@arguments,\%sophomorix_result,\%sophomorix_config,$json);
        }
        ${$update_ref}++;
        # ${$new_class_ref}=$adminclass; # with prefix
        ${$new_class_ref}=$Match{$school}{'ACTION'}{$line_new}{'class'}; # without prefix
        $identifier_class_changes++;
    }

    # school
    if ( ($Match{$school}{'ACTION'}{$line_new}{'school'} eq $ref_AD_check->{'sAMAccountName'}{$sam}{'sophomorixSchoolname'} )){
        # no update
        ${$new_school_ref}="---";
    } else {
        # update
        ${$update_ref}++;
        ${$new_school_ref}=$Match{$school}{'ACTION'}{$line_new}{'school'};
        $identifier_class_changes++;
    }

    my $role_for_homedirectory;
    if (${$new_role_ref} ne "---"){
        # role changes, use new role
        $role_for_homedirectory=${$new_role_ref};
    } else {
        # no role change: role according to filename
        $role_for_homedirectory=$ref_sophomorix_config->{'FILES'}{'USER_FILE'}{$Match{$school}{'ACTION'}{$line_new}{'filename'}}{'sophomorixRole'};
    }

    # homedirectory could not be calculated for %Match, calculating here
    my ($homedirectory_new,$unix_home,$unc,$smb_rel_path)=
    &Sophomorix::SophomorixBase::get_homedirectory($root_dns,
                                                   $ref_sophomorix_config->{'FILES'}{'USER_FILE'}{$Match{$school}{'ACTION'}{$line_new}{'filename'}}{'SCHOOL'},
                                                   $Match{$school}{'ACTION'}{$line_new}{'class'}, # groupname is the subdir
                                                   $sam,
                                                   $role_for_homedirectory,
                                                   $ref_sophomorix_config);
    if ( $homedirectory_new eq $ref_AD_check->{'sAMAccountName'}{$sam}{'homeDirectory'} ){
        # no update
        ${$new_homedirectory_ref}="---";
    } else {
        # update
        # homeDirectory
        ${$update_ref}++;
        ${$new_homedirectory_ref}=$homedirectory_new;
    }

    if ( defined $ref_AD_check->{'sAMAccountName'}{$sam}{'sophomorixIntrinsic2'} 
         and
         $smb_rel_path eq $ref_AD_check->{'sAMAccountName'}{$sam}{'sophomorixIntrinsic2'} ){
        # no update
        ${$new_homedirectory_rel_ref}="---";
    } else {
        # update
        # homeDirectory
        ${$update_ref}++;
        ${$new_homedirectory_rel_ref}=$smb_rel_path;
    }
    
    # unid
    if ( ($Match{$school}{'ACTION'}{$line_new}{'unid'} eq $ref_AD_check->{'sAMAccountName'}{$sam}{'sophomorixUnid'}) or
         ($Match{$school}{'ACTION'}{$line_new}{'unid'} eq "" and 
          $ref_AD_check->{'sAMAccountName'}{$sam}{'sophomorixUnid'} eq "---") 
       ){
        # no update
        ${$new_unid_ref}="---";
    } else {
        # update only if user is identically matched
        if ($identifier_class_changes==0){
            if ($Match{$school}{'ACTION'}{$line_new}{'unid'} eq "---"){
                # write empty value to sophomorix.update, so sophomorix-update 
                # will not skip it ("---" would mean skip it)
                ${$new_unid_ref}="";
                ${$update_ref}++;
            } else {
                ${$new_unid_ref}=$Match{$school}{'ACTION'}{$line_new}{'unid'};
                ${$update_ref}++;
            }
        } else {
            print "$sam not uploading new unid: identifier or class changed\n";
        }
    }


    ############################################################
    # role/school dependant
    # school and role for mail webui 
    my $role=$Match{$school}{'ACTION'}{$line_new}{'role'};

    # mail
    (${$new_mail_ref})=&AD_create_new_mail($sam,
					   \@arguments,
                                           \%sophomorix_config,
					   \%sophomorix_result,
					   $json,
                                           $role,
                                           $school,
                                           ${$old_firstname_ascii_ref},
                                           ${$new_firstname_ascii_ref},
                                           ${$old_surname_ascii_ref},
                                           ${$new_surname_ascii_ref},
	                                   $Match{$school}{'ACTION'}{$line_new}{'filename'},
	                                   $adminclass);
    # test for update
    if ( ${$new_mail_ref} eq  ${$old_mail_ref} ){
        # no update
        ${$new_mail_ref}="---";
    } else {
        # update
        $sophomorix_result{'CHECK_RESULT'}{'UPDATE'}{$sam}{'mail'}{'OLD'}=${$old_mail_ref};
        $sophomorix_result{'CHECK_RESULT'}{'UPDATE'}{$sam}{'mail'}{'NEW'}=${$new_mail_ref};
        ${$update_ref}++;
    }

    # webui_string
    # creating webui string with NEW role and NEW school
    (${$new_webui_string_ref})=&AD_create_new_webui_string($sam,\%sophomorix_config,$ref_AD_check,$role,$school);

    #print "\n";
    #print "$sam:\n";
    #print "  OLD: ${$old_webui_string_ref}\n";
    #print "  NEW: ${$new_webui_string_ref}\n";

    # test for update
    if ( ${$new_webui_string_ref} eq  ${$old_webui_string_ref} ){
        # no update
        ${$new_webui_string_ref}="---";
    } else {
        # update
        #print "update of $sam:\n";
        #print "   ${$old_webui_string_ref}\n";
        #print "   ${$new_webui_string_ref}\n";
        ${$update_ref}++;
    }
}



# writing lines into files
################################################################################
sub write_add_file {
    my ($school,$line_new)=@_;

    my ($filename,$line_add)=split(/::/,$line_new);
    if($Conf::log_level>=2){
        print "$school $filename -> >$line_add<\n";
        print "   * not matched to any sAMAccountName\n";
    }

    my $status;
    if ($Match{$school}{'ACTION'}{$line_add}{'inject'} eq "yes"){
        $status="S"; # injected users have status S=selfactivated
    } else {
        $status="U";
    }

    my ($login,$error_message)=&create_test_login($school,
						  $Match{$school}{'ACTION'}{$line_add}{'identifier_ascii'}, # user data
                                                  $Match{$school}{'ACTION'}{$line_add}{'filename'}, # filename for password length
                                                  $Match{$school}{'ACTION'}{$line_add}{'login'}, # proposed login (will be tested)
                                                  $ref_AD_check, # forbidden logins
                                                  $ref_login_avoid, # logins to avoid
                                                  $Match{$school}{'ACTION'}{$line_add}{'line_count'},
                                                  \%users_file,
						  \%sophomorix_config);
    my $mail=&AD_create_new_mail($login,
                                 \@arguments,
                                 \%sophomorix_config,
         		         \%sophomorix_result,
				 $json,
                                 $Match{$school}{'ACTION'}{$line_add}{'role'},
                                 $Match{$school}{'ACTION'}{$line_add}{'school'},
                                 "---",
                                 $Match{$school}{'ACTION'}{$line_add}{'firstname_ascii'},
                                 "---",
				 $Match{$school}{'ACTION'}{$line_add}{'surname_ascii'},
				 $Match{$school}{'ACTION'}{$line_add}{'filename'},
				 $Match{$school}{'ACTION'}{$line_add}{'class'}
                                );
    my $webui_permissions_string=join(",",
        @{ $sophomorix_config{'ROLES'}{$Match{$school}{'ACTION'}{$line_add}{'school'}}{$Match{$school}{'ACTION'}{$line_add}{'role'}}{'UI'}{'WEBUI_PERMISSIONS'} });
    my $add_line=$Match{$school}{'ACTION'}{$line_add}{'filename'}."::".
                 $Match{$school}{'ACTION'}{$line_add}{'class'}."::".
                 $Match{$school}{'ACTION'}{$line_add}{'identifier_ascii'}."::".
                 $login."::".
                 "---::". # firstpassword
                 "---::". # uid
                 "---::". # gid
                 $Match{$school}{'ACTION'}{$line_add}{'unid'}."::".
                 $Match{$school}{'ACTION'}{$line_add}{'school'}."::".
                 $Match{$school}{'ACTION'}{$line_add}{'role'}."::".
                 $Match{$school}{'ACTION'}{$line_add}{'surname_utf8'}."::".
                 $Match{$school}{'ACTION'}{$line_add}{'firstname_utf8'}."::".
                 $status."::".   # status
                 "---::". # creationdate
                 "---::". # tolerationdate
                 "---::". # deactivationdate
                 "---::". # sambantpassword
                 "---::". # userpassword
                 $mail."::". # mail
                 $webui_permissions_string."::".
                 $webui_permissions_string."::".
                 "\n";
    if ($login ne "---" and $error_message eq "---"){
        print ADD $add_line;
        # create result
        push @{ $sophomorix_result{'CHECK_RESULT'}{'ADDLIST'} }, $login;
        $sophomorix_result{'CHECK_RESULT'}{'ADD'}{$login}{'sophomorixAdminFile'}=$Match{$school}{'ACTION'}{$line_add}{'filename'};
        $sophomorix_result{'CHECK_RESULT'}{'ADD'}{$login}{'sophomorixAdminClass'}=$Match{$school}{'ACTION'}{$line_add}{'class'};
        $sophomorix_result{'CHECK_RESULT'}{'ADD'}{$login}{'sn'}=$Match{$school}{'ACTION'}{$line_add}{'surname_utf8'};
        $sophomorix_result{'CHECK_RESULT'}{'ADD'}{$login}{'givenName'}=$Match{$school}{'ACTION'}{$line_add}{'firstname_utf8'};
        $sophomorix_result{'CHECK_RESULT'}{'ADD'}{$login}{'sophomorixUnid'}=$Match{$school}{'ACTION'}{$line_add}{'unid'};
        $sophomorix_result{'CHECK_RESULT'}{'ADD'}{$login}{'sophomorixSchoolname'}=$Match{$school}{'ACTION'}{$line_add}{'school'};
        $sophomorix_result{'CHECK_RESULT'}{'ADD'}{$login}{'sophomorixRole'}=$Match{$school}{'ACTION'}{$line_add}{'role'};
        $sophomorix_result{'CHECK_RESULT'}{'ADD'}{$login}{'identifier_ascii'}=$Match{$school}{'ACTION'}{$line_add}{'identifier_ascii'};
        $counters{'ADD'}++;
    } else {
        &log_script_exit($error_message,1,1,0,
                     \@arguments,\%sophomorix_result,\%sophomorix_config,$json);
    }
}



# UTF8 match
################################################################################
sub populate_ad_dynamic_identifiers_utf8 {
    my ($school) = @_;
    foreach my $school (@opt_schoollist){
        foreach my $identifier_utf8 (keys %{ $ref_AD_check->{'LOOKUP_by_school'}{$school}{'user_BY_identifier_utf8'} } ) { 
            $Match{$school}{'ad_dynamic_identifiers_utf8'}{$identifier_utf8}="ad, not matched";
        } 
    }
}



# Migration match
################################################################################
# call this in the end of all matches ??????????????????????ßßß
sub show_dynamic_identifiers_ascii {
    my ($school) = @_;
    my $file_count=0;
    my $ad_count=0;
    my @file_ids=();
    my @ad_ids=();
    foreach my $identifier_ascii (keys %{ $Match{$school}{'file_dynamic_identifiers_ascii'} } ) {
        push @file_ids, $identifier_ascii;
        $file_count++;
    } 
    foreach my $identifier_ascii (keys %{ $Match{$school}{'ad_dynamic_identifiers_ascii'} } ) {
        push @ad_ids, $identifier_ascii;
        $ad_count++;
    }
    my $diff=$ad_count-$file_count;
    if($Conf::log_level>=2){
        &print_title("school $school: $file_count user(s) to be matched with $ad_count in AD (Diff: $diff)","slim");
    }
    if ($file_count==0 or $ad_count==0){
	return 0;
    } else {
        if($Conf::log_level>=3){
	    @file_ids= sort @file_ids;
            print "file_dynamic_identifiers_ascii in school $school:\n";
            foreach my $id (@file_ids){
                print "   * $id (in $school)\n";
            }   
	    @ad_ids = sort @ad_ids;
            print "ad_dynamic_identifiers_ascii in AD:\n";
            foreach my $id (@ad_ids){
                my $status=
                print "   * $id ($ref_AD_check->{'LOOKUP_by_school'}{$school}{'user_BY_identifier_ascii'}{$id} in AD,",
                      " status $ref_AD_check->{'LOOKUP_by_school'}{$school}{'sophomorixStatus_BY_identifier_ascii'}{$id})\n";
            }    
        }
        return 1;
    }
}



sub populate_ad_dynamic_identifiers_ascii {
    my ($school)=@_;
    foreach my $identifier_ascii (keys %{ $ref_AD_check->{'LOOKUP_by_school'}{$school}{'user_BY_identifier_ascii'} } ) { 
        $Match{$school}{'ad_dynamic_identifiers_ascii'}{$identifier_ascii}="ad, not matched";
    } 
}



sub find_ascii_approx_matches {
    my ($school,$max_edit_distance)=@_;
    my $run="approx_".$max_edit_distance."_run";
    my $line_count_school=0;
    my $matched_count_school=0;
    my $no_unid_count_school=0;
    my $not_matched_count_school=0;
    my $ret=&show_dynamic_identifiers_ascii($school);
    if ($ret==0){
        if($Conf::log_level>=2){
            &print_title("approxMATCH: Nothing to do anymore in $school at edit distance $max_edit_distance","slim");
        }
        return ($max_edit_distance,$line_count_school,$school,$matched_count_school,$not_matched_count_school);
    }

    foreach my $identifier_ascii (keys %{ $Match{$school}{'file_dynamic_identifiers_ascii'} } ) {
        $line_count_school++;
        if($Conf::log_level>=2){
            print "  LINE $users_file{$school}{'identifier_ascii'}{$identifier_ascii}{LINE_COUNT}: ",
                  "$identifier_ascii (UNID: $users_file{$school}{'identifier_ascii'}{$identifier_ascii}{UNID})\n";
        }
        foreach my $identifier_ascii_ad ( keys %{ $Match{$school}{'ad_dynamic_identifiers_ascii'} } ){
            my $edit_distance=&edit_distance($identifier_ascii,$identifier_ascii_ad);
            if($Conf::log_level>=2){
                print "  * CHECK: $identifier_ascii -> $identifier_ascii_ad\n";
            }
            if ($edit_distance <= $max_edit_distance){
                # APPROX MATCH
                my $sam=$ref_AD_check->{'LOOKUP_by_school'}{$school}{'user_BY_identifier_ascii'}{$identifier_ascii_ad};
                if($Conf::log_level>=2){
                    print "    * Edit distance is $edit_distance\n";
                    print "    * approxMATCH: $sam -> $identifier_ascii\n";
                }

                delete $Match{$school}{'ad_dynamic_identifiers_ascii'}{$identifier_ascii_ad};
                # save match to delete this match from list
                $Match{$school}{'file_approx_match'}{$run}{$identifier_ascii}=$edit_distance;
                # remove the approx matches from the list
                delete $Match{$school}{'file_dynamic_identifiers_ascii'}{$identifier_ascii};

                # save to write later
                my $line_new=$users_file{$school}{'identifier_ascii'}{$identifier_ascii}{LINE_NEW};

                my $sub=(caller(0))[3];
                $Match{$school}{'ACTION'}{$line_new}{'MATCH_SUB'}="$sub $max_edit_distance";

                $Match{$school}{'ACTION'}{$line_new}{'sAMAccountName'}=$sam;
                $Match{$school}{'ACTION'}{$line_new}{'status'}=
                    $ref_AD_check->{'LOOKUP_by_school'}{$school}{'sophomorixStatus_BY_identifier_ascii'}{$identifier_ascii};
                # reverse action_BY_sAMAccountName
                $Match{$school}{'action_BY_sAMAccountName'}{$sam}=$line_new;
                $matched_count_school++;
            } else {
                $not_matched_count_school++;
                if($Conf::log_level>=2){
                    print "    * NOMATCH: Edit distance is $edit_distance\n";
                }
            }
        }
    }
    &print_title("approxMATCH: $line_count_school CHECKED in $school, MATCH: $matched_count_school (Edit distance: $max_edit_distance)",
		 "slim");
    return (-1,$line_count_school,$school,$matched_count_school,$not_matched_count_school);
}



sub find_ascii_matches {
    my ($school)=@_;
    my $line_count_school=0;
    my $matched_count_school=0;
    my $not_matched_count_school=0;
    &show_dynamic_identifiers_ascii($school);
    foreach my $identifier_ascii (keys %{ $Match{$school}{'file_dynamic_identifiers_ascii'} } ) {
        $line_count_school++;
        if($Conf::log_level>=2){
            print "  LINE $users_file{$school}{'identifier_ascii'}{$identifier_ascii}{LINE_COUNT}: ",
                  "$identifier_ascii (UNID: $users_file{$school}{'identifier_ascii'}{$identifier_ascii}{UNID})\n";
            print "  * CHECK: is $identifier_ascii in AD ($school)?\n";
        }
        if ( exists $ref_AD_check->{'LOOKUP_by_school'}{$school}{'user_BY_identifier_ascii'}{$identifier_ascii} ){
            # MATCH
            my $sam=$ref_AD_check->{'LOOKUP_by_school'}{$school}{'user_BY_identifier_ascii'}{$identifier_ascii};
            if($Conf::log_level>=2){
                print "    * exactMATCH: $sam --> $identifier_ascii\n";
            }

            # save to write later
            my $line_new2=$users_file{$school}{'identifier_ascii'}{$identifier_ascii}{LINE_NEW};
            $Match{$school}{'ACTION'}{$line_new2}{'sAMAccountName'}=$sam;

            my $sub=(caller(0))[3];
            $Match{$school}{'ACTION'}{$line_new2}{'MATCH_SUB'}="$sub";

            $Match{$school}{'ACTION'}{$line_new2}{'status'}=
                $ref_AD_check->{'LOOKUP_by_school'}{$school}{'sophomorixStatus_BY_identifier_ascii'}{$identifier_ascii};
            # reverse action_BY_sAMAccountName
            $Match{$school}{'action_BY_sAMAccountName'}{$sam}=$line_new2;

            # remove from ad_to_be_matched new
            delete $Match{$school}{'ad_dynamic_identifiers_ascii'}{$identifier_ascii};
            delete $Match{$school}{'file_dynamic_identifiers_ascii'}{$identifier_ascii};
            $matched_count_school++;
        } else {
            # NO MATCH
            if($Conf::log_level>=2){
                print "    * NOexactMATCH for $identifier_ascii\n";
            }
            $Match{$school}{'find_ascii_matches'}{$identifier_ascii}="no exact identifier match possible";
            $not_matched_count_school++;
        }
    }
    &print_title("exactMATCH: $line_count_school CHECKED in $school, MATCH: $matched_count_school, NOMATCH: $not_matched_count_school","slim");
}



sub find_unid_matches {
    my ($school)=@_;
    my $line_count_school=0;
    my $matched_count_school=0;
    my $no_unid_count_school=0;
    my $not_matched_count_school=0;
    my $ret=&show_dynamic_identifiers_ascii($school);
    foreach my $identifier_ascii (keys %{$Match{$school}{'file_dynamic_identifiers_ascii'}} ) {
        $line_count_school++;
        if($Conf::log_level>=2){
            print "  LINE $users_file{$school}{'identifier_ascii'}{$identifier_ascii}{LINE_COUNT}: ",
                  "$identifier_ascii (UNID: $users_file{$school}{'identifier_ascii'}{$identifier_ascii}{UNID})\n";
            print "  * CHECK: is unid $users_file{$school}{'identifier_ascii'}{$identifier_ascii}{UNID} in AD ($school)\n";
        }
        if ($users_file{$school}{'identifier_ascii'}{$identifier_ascii}{UNID} ne ""){
            # finding a unid match 
            if ( exists $ref_AD_check->{'LOOKUP_by_school'}{$school}{'user_BY_sophomorixUnid'}{$users_file{$school}{'identifier_ascii'}{$identifier_ascii}{UNID}} and
                $ref_AD_check->{'LOOKUP_by_school'}{$school}{'user_BY_sophomorixUnid'}{$users_file{$school}{'identifier_ascii'}{$identifier_ascii}{UNID}} ne "---"
               ){
                # MATCH
                my $sam=$ref_AD_check->{'LOOKUP_by_school'}{$school}{'user_BY_sophomorixUnid'}{$users_file{$school}{'identifier_ascii'}{$identifier_ascii}{UNID}};
                my $role=$ref_AD_check->{'LOOKUP_by_school'}{$school}{'sophomorixRole_BY_sAMAccountName'}{$sam};
                # fetch the identifier from AD with same unid
                my $identifier_ascii_from_AD=
                    $ref_AD_check->{'LOOKUP_by_school'}{$school}{'identifier_ascii_BY_sophomorixUnid'}{$users_file{$school}{'identifier_ascii'}{$identifier_ascii}{UNID}};
                if($Conf::log_level>=2){
                    print "    * Matching unid found at user $sam ($identifier_ascii_from_AD)\n";
	            print "    * unidMATCH:  $sam --> $identifier_ascii\n";
                }
                $matched_count_school++;
                # remove from dynamic hashes
                delete $Match{$school}{'ad_dynamic_identifiers_ascii'}{$identifier_ascii_from_AD};
                delete $Match{$school}{'file_dynamic_identifiers_ascii'}{$identifier_ascii};

                # save to write later
                $Match{'school'}{$school}{$identifier_ascii}=$sam;
                my $line_new=$users_file{$school}{'identifier_ascii'}{$identifier_ascii}{LINE_NEW};
                $Match{$school}{'ACTION'}{$line_new}{'sAMAccountName'}=$sam;

                my $sub=(caller(0))[3];
                $Match{$school}{'ACTION'}{$line_new}{'MATCH_SUB'}="$sub";

                $Match{$school}{'ACTION'}{$line_new}{'status'}=
                    $ref_AD_check->{'LOOKUP_by_school'}{$school}{'sophomorixStatus_BY_identifier_ascii'}{$identifier_ascii};
                # reverse action_BY_sAMAccountName
                $Match{$school}{'action_BY_sAMAccountName'}{$sam}=$line_new;
            } else {
                # NO MATCH
                $Match{$school}{'find_unid_matches'}{$identifier_ascii}="no unid match";
                $not_matched_count_school++;
                if($Conf::log_level>=2){
                    print "    * NOMATCH: Unid $users_file{$school}{'identifier_ascii'}{$identifier_ascii}{UNID} not found in AD\n";
                }
            }
        } else {
            $no_unid_count_school++;
            $Match{$school}{'find_unid_matches'}{$identifier_ascii}="no unid, no match possible";
        }
    }
    &print_title("unidMATCH: $line_count_school CHECKED in $school, MATCH: $matched_count_school, NOMATCH: $not_matched_count_school, NOunid: $no_unid_count_school","slim");
}



# APPROX stuff
################################################################################
sub edit_distance {
    my ($string1,$string2)=@_;
    my $string;
    my $pattern;
    # the shorter string is the pattern to avoid negative edit distances    
    if (length($string1) < length($string2)){
        $string=$string2;
        $pattern=$string1;
    } else {
	    $string=$string1;
        $pattern=$string2;
    }
    my $edit_distance=adist($pattern,$string);
    return $edit_distance;
}



# checking stuff
################################################################################
sub analyze_user_line {
    my ($line,$file_abs,$filename,$school,$count,$enc_used,$injected,$type,$ref_lines_seen) = @_;
    # $type is USER_FILE
    if ($line=~m/^#/ or $line=~m/^\s*$/){
        # begins with #
        # or contains only whitespace
        return 0;
    } else {
        ########## Analyze the line ##########
        my $line_raw=$line;
        chomp($line);

        my $semikolon_count=$line=~tr/;//;
        if ($semikolon_count<3){
            if ($injected eq "yes"){
               &result_sophomorix_add(\%sophomorix_result,
                   "ERROR",
                   -1,
                   \@arguments,
                   "$filename (Line: $count): Not 3 semicolons in --injectline: $line");
	       return;
            } else {
               &result_sophomorix_add(\%sophomorix_result,
                   "ERROR",
                   -1,
                   \@arguments,
                   "$filename (Line: $count): Not 3 semicolons in option: $line");
	       return;
            }
        }
        $line=&remove_embracing_whitespace($line); # leading and trailing
        # add trailing ; if not there
        if (not $line=~m/;$/){
            $line=$line.";";
        }
        my ($class_raw,
            $last_raw,
            $first_raw,
            $birthdate_raw,
            $field5,
            $field6,
           )=split(/;/,$line);

        my ($class_valid,$class,$error_strg_class)=&check_class($class_raw,$filename,$type);
        if ($class_valid==0){
            print REPORT "\nERROR in $line\n";
            print REPORT "   * Class $class_raw not valid ($error_strg_class)\n";
            # ignore the line, as if it is not there
            print ERROR $filename.":".$line_raw;
            my $line=$line_raw;
            chomp($line);
            push @{ $sophomorix_result{'CHECK_RESULT'}{'ERRORLIST'} }, $filename.":".$line;
            $sophomorix_result{'CHECK_RESULT'}{'ERROR'}{$filename.":".$line}{'REASON'}=
                "Class $class_raw not valid ($error_strg_class)";
            $counters{'ERROR'}++;
            return;
        }

        # givenname
        my ($first_valid,$first,$error_strg_first)=&check_first($first_raw);
        if ($first_valid==0){
            print REPORT "\nERROR in $line\n";
            print REPORT "   * Surname $first_raw not valid ($error_strg_first)\n";
            # ignore the line, as if it is not there
            print ERROR $filename.":".$line_raw;
            my $line=$line_raw;
            chomp($line);
            push @{ $sophomorix_result{'CHECK_RESULT'}{'ERRORLIST'} }, $filename.":".$line;
            $sophomorix_result{'CHECK_RESULT'}{'ERROR'}{$filename.":".$line}{'REASON'}=
                "Surname $first_raw not valid ($error_strg_first)";
            $counters{'ERROR'}++;
            return;
        }

        # surname
        my ($last_valid,$last,$error_strg_last)=&check_last($last_raw);
        if ($last_valid==0){
            print REPORT "\nERROR in $line\n";
            print REPORT "   * Surname $last_raw not valid ($error_strg_last)\n";
            # ignore the line, as if it is not there
            print ERROR $filename.":".$line_raw;
            my $line=$line_raw;
            chomp($line);
            push @{ $sophomorix_result{'CHECK_RESULT'}{'ERRORLIST'} }, $filename.":".$line;
            $sophomorix_result{'CHECK_RESULT'}{'ERROR'}{$filename.":".$line}{'REASON'}=
                "Surname $last_raw not valid ($error_strg_last)";
            $counters{'ERROR'}++;
            return;
        }

        # Birthdate
        my ($birthdate_valid,$birthdate,$error_strg_birth)=&check_birthdate($birthdate_raw);
        if ($birthdate_valid==0){
            print REPORT "\nERROR in $line\n";
            print REPORT "   * Birthdate $birthdate_raw not valid ($error_strg_birth)\n";
            # ignore the line, as if it is not there
            print ERROR $filename.":".$line_raw;
            my $line=$line_raw;
            chomp($line);
            push @{ $sophomorix_result{'CHECK_RESULT'}{'ERRORLIST'} }, $filename.":".$line;
            $sophomorix_result{'CHECK_RESULT'}{'ERROR'}{$filename.":".$line}{'REASON'}=
                "Birthdate $birthdate_raw not valid ($error_strg_birth)";
            $counters{'ERROR'}++;
            return;
        }

        my $unid="";
        my $login="";
        # field5
        my $field5_key=$sophomorix_config{'FILES'}{$type}{$filename}{'FIELD_5'};
        if ($field5_key eq "LOGIN"){
            if (defined $field6){
                $login=&check_login($field5);
            }
        } elsif ($field5_key eq "UNID"){
            if (defined $field6){
                $unid=&check_unid($field5);
            }
        }
        # field6
        my $field6_key=$sophomorix_config{'FILES'}{$type}{$filename}{'FIELD_6'};
        if ($field6_key eq "LOGIN"){
            if (defined $field6){
                $login=&check_login($field6);
            }
        } elsif ($field6_key eq "UNID"){
            if (defined $field6){
                $unid=&check_unid($field6);
            }
        }
        if ($unid eq ""){
	    $unid="---";
        }
        if ($unid ne "---" and exists $unid_seen{$filename}{$unid}){
            &result_sophomorix_add(\%sophomorix_result,
                                   "ERROR",
                                   -1,
                                   \@arguments,
                                   "$filename (Line: $count): Unid (Field5) is double: $unid ");
        } else {
            $unid_seen{$filename}{$unid}="seen";
        }

        if ($login eq ""){
	    $login="---";
        }

        # converting names to utf8 and then to ascii
        my $conv = Text::Iconv->new($enc_used,"utf8");
        my $first_utf8 = $conv->convert($first);
        my $last_utf8 = $conv->convert($last);
        my $first_ascii =&recode_utf8_to_ascii($first_utf8);
        my $last_ascii =&recode_utf8_to_ascii($last_utf8);
        # identifier
        my $identifier_utf8=join(";", ($last_utf8,$first_utf8,$birthdate));
        my $identifier_ascii=join(";", ($last_ascii,$first_ascii,$birthdate));

        # corrected line
        my $line_new=join(";", ($class,$last_utf8,$first_utf8,$birthdate,$unid));
        print USERSUTF8 $line_new."\n";

        # ignore multiple lines of the same content (class and identifier, unid) 
        if (not exists $ref_lines_seen->{'LINES'}{$line_new}){
            # remember the line
            $ref_lines_seen->{'LINES'}{$line_new}="seen";
	} else {
            # ignore this line with WARNING
	    &result_sophomorix_add_log(\%sophomorix_result,"WARNING: LINE $line_new is multiple in $filename (ignoring double line)");
            return;
	}

	# do not ignore first identifier of multiple identifiers within a school
	if (exists $users_file{'LOOKUP'}{$school}{'identifier_ascii'}{$identifier_ascii}){
            my $count=$users_file{'LOOKUP'}{$school}{'identifier_ascii'}{$identifier_ascii}{'COUNT'};
	    $count++;
	    $users_file{'LOOKUP'}{$school}{'identifier_ascii'}{$identifier_ascii}{$count}{'CLASS'}=$class;
	    $users_file{'LOOKUP'}{$school}{'identifier_ascii'}{$identifier_ascii}{$count}{'FILE'}=$filename;
	    $users_file{'LOOKUP'}{$school}{'identifier_ascii'}{$identifier_ascii}{$count}{'UNID'}=$unid;
	    $users_file{'LOOKUP'}{$school}{'identifier_ascii'}{$identifier_ascii}{'COUNT'}=$count;
	    return;
	} else {
	    $users_file{'LOOKUP'}{$school}{'identifier_ascii'}{$identifier_ascii}{'1'}{'CLASS'}=$class;
	    $users_file{'LOOKUP'}{$school}{'identifier_ascii'}{$identifier_ascii}{'1'}{'FILE'}=$filename;
	    $users_file{'LOOKUP'}{$school}{'identifier_ascii'}{$identifier_ascii}{'COUNT'}=1;
        }


        # by schoolname
        $users_file{$school}{'identifier_ascii'}{$identifier_ascii}{LINE_OLD}=$line;
        $users_file{$school}{'identifier_ascii'}{$identifier_ascii}{LINE_NEW}=$line_new;
        $users_file{$school}{'identifier_ascii'}{$identifier_ascii}{FIRSTNAME_ASCII}=$first_ascii;
        $users_file{$school}{'identifier_ascii'}{$identifier_ascii}{FIRSTNAME_UTF8}=$first_utf8;
        $users_file{$school}{'identifier_ascii'}{$identifier_ascii}{LASTNAME_ASCII}=$last_ascii;
        $users_file{$school}{'identifier_ascii'}{$identifier_ascii}{LASTNAME_UTF8}=$last_utf8;
        $users_file{$school}{'identifier_ascii'}{$identifier_ascii}{BIRTHDATE}=$birthdate;
        $users_file{$school}{'identifier_ascii'}{$identifier_ascii}{CLASS}=$class;
        $users_file{$school}{'identifier_ascii'}{$identifier_ascii}{UNID}=$unid;
        $users_file{$school}{'identifier_ascii'}{$identifier_ascii}{LOGIN}=$login;
        $users_file{$school}{'identifier_ascii'}{$identifier_ascii}{FILE}=$filename;
        $users_file{$school}{'identifier_ascii'}{$identifier_ascii}{IDENTIFIER_ASCII}=$identifier_ascii;
        $users_file{$school}{'identifier_ascii'}{$identifier_ascii}{IDENTIFIER_UTF8}=$identifier_utf8;
        $users_file{$school}{'identifier_ascii'}{$identifier_ascii}{LINE_COUNT}=$count;
        $users_file{$school}{'identifier_ascii'}{$identifier_ascii}{FILE_ABS}=$file_abs;
        $users_file{$school}{'identifier_ascii'}{$identifier_ascii}{INJECTED}=$injected;
        $users_file{$school}{'identifier_ascii'}{$identifier_ascii}{ROLE}=
            $sophomorix_config{'FILES'}{$type}{$school}{sophomorixRole};
        $users_file{$school}{'identifier_ascii'}{$identifier_ascii}{SCHOOL}=
            $sophomorix_config{'FILES'}{$type}{$school}{SCHOOL};

        # identifier hash
        $Match{$school}{'file_dynamic_identifiers_ascii'}{$identifier_ascii}="files, not matched";
        $Match{$school}{'file_dynamic_identifiers_utf8'}{$identifier_utf8}="files, not matched";

        # save for action school
        $Match{$school}{'ACTION'}{$line_new}{'identifier_ascii'}=$identifier_ascii;
        $Match{$school}{'ACTION'}{$line_new}{'school'}=
            $sophomorix_config{'FILES'}{$type}{$filename}{SCHOOL};
	$Match{$school}{'ACTION'}{$line_new}{'birthdate'}=$birthdate;
	$Match{$school}{'ACTION'}{$line_new}{'class'}=$class;
	$Match{$school}{'ACTION'}{$line_new}{'class_type'}=$type;
	$Match{$school}{'ACTION'}{$line_new}{'filename'}=$filename;
	$Match{$school}{'ACTION'}{$line_new}{'firstname_ascii'}=$first_ascii;
	$Match{$school}{'ACTION'}{$line_new}{'surname_ascii'}=$last_ascii;
	$Match{$school}{'ACTION'}{$line_new}{'firstname_utf8'}=$first_utf8;
	$Match{$school}{'ACTION'}{$line_new}{'surname_utf8'}=$last_utf8;
	$Match{$school}{'ACTION'}{$line_new}{'line_count'}=$count;
        $Match{$school}{'ACTION'}{$line_new}{'role'}=
            $sophomorix_config{'FILES'}{$type}{$filename}{sophomorixRole};
	$Match{$school}{'ACTION'}{$line_new}{'unid'}=$unid;
	$Match{$school}{'ACTION'}{$line_new}{'login'}=$login;
	$Match{$school}{'ACTION'}{$line_new}{'inject'}=$injected;

        push @{ $Match{$school}{'lines_with_filename'} }, $filename."::".$line_new;
    }
}



sub check_class {
    my $class_valid=1;
    my $error_strg="";
    my ($class,$filename,$type)=@_;
    # type is USER_FILE
    $class=&remove_whitespace($class); # remove all whitespace
    # convert to small letters
    $class=~tr/A-Z/a-z/; 

    foreach my $forbidden ( @{ $sophomorix_config{'FILES'}{'USER_FILE'}{$filename}{'FORBIDDEN_GROUP_BASENAME'} }){
        if ($class eq $forbidden){
	   my $error_message="Class '".$class."' is forbidden in file '".$filename."'";
	   &log_script_exit($error_message,1,1,0,
                     \@arguments,\%sophomorix_result,\%sophomorix_config,$json);
        }
    }

    # replacements in classnames
    if (defined $sophomorix_config{'FILES'}{'USER_FILE'}{$filename}{'CLASSNAME_SLASH_TO_HYPHEN'} and
        $sophomorix_config{'FILES'}{'USER_FILE'}{$filename}{'CLASSNAME_SLASH_TO_HYPHEN'} eq "TRUE"){
        $class=~s/\//-/g;
    } else {
        $class=~s/\//$DevelConf::replace_slash_with/g;
    }

    if ($class=~/[^a-z0-9-_]/) { 
        $error_strg="special char in class $class";
        $class_valid=0;
    }
    if ($class=~/^[^a-z0-9]/) { 
        $error_strg="$class begins not with a-z or 0-9";
        $class_valid=0;
    }
    
    # test if groupname must be forced (i.e. teachers)
    if ($sophomorix_config{'FILES'}{$type}{$filename}{'FORCE_GROUP'} eq "TRUE"){
        $class=$sophomorix_config{'FILES'}{$type}{$filename}{'FORCE_GROUPNAME'};
    }
    # apply CLASSNAME_MAP if existing
    if (exists $sophomorix_config{'FILES'}{'USER_FILE'}{$filename}{CLASSNAME_MAP_KEY_VALUE}{$class}){
	$class=$sophomorix_config{'FILES'}{'USER_FILE'}{$filename}{CLASSNAME_MAP_KEY_VALUE}{$class};
    }
    return ($class_valid,$class,$error_strg);
}



sub check_first {
    my $first_valid=1;
    my $error_strg="";
    my ($first)=@_;
    $first=&remove_embracing_whitespace($first);
    # allowed ASCII: ;'`-._  (; leads to errors anyway)
    if ($first=~/[!\\"#\$%&()*+,\/:<>=@?\{\}|\^~]/) { 
        $error_strg="special char in given name $first";
        $first_valid=0;
    }
    return ($first_valid,$first,$error_strg);
}



sub check_last {
    my $last_valid=1;
    my $error_strg="";
    my ($last)=@_;
    $last=&remove_embracing_whitespace($last);
    # allowed ASCII: ;'`-._  (; leads to errors anyway)
    if ($last=~/[!\\"#\$%&()*+,\/:<>=@?\{\}|\^~]/) { 
        $error_strg="special char in surname $last";
        $last_valid=0;
    }
    return ($last_valid,$last,$error_strg);
}



sub check_birthdate {
    my $birthdate_valid=1;# valid if not found out otherwise
    my $error_strg="";
    my $birthdate_ok="";

    my ($birthdate)=@_;
    $birthdate=&remove_embracing_whitespace($birthdate);

    # Test 1
    my $dot_count=$birthdate=~tr/\.//;
    if ($dot_count==2){
        # continue to Test 2
        if($Conf::log_level>=3){
            print " ... $dot_count dots in date found ... OK\n";
	}
    } elsif ($birthdate eq ""){
        $birthdate_valid=0;
        $error_strg="Birthdate is empty";
        return ($birthdate_valid,$birthdate,$error_strg);
    } else {
        $birthdate_valid=0;
        $error_strg="Birthdate does not have 2 dots: $birthdate";
        return ($birthdate_valid,$birthdate,$error_strg);
    }

    # Test 2
    my ($day,$month,$year)=split(/\./,$birthdate);

    # Instead of undefined use empty string
    # this avoids errors
    if (not defined $day){
        $day="";
    }
    if (not defined $month){
        $month="";
    }
    if (not defined $year){
        $year="";
    }

    if (exists $convert_day{$day}){
        $day=$convert_day{$day};
    } else {
        $birthdate_valid=0;
        $error_strg="Day is not valid: $day";
    }

    if (exists $convert_month{$month}){
        $month=$convert_month{$month};
    } else {
        $birthdate_valid=0;
        $error_strg="Month is not valid: $month";
    }

    if (exists $convert_year{$year}){
        $year=$convert_year{$year};
    } else {
        $birthdate_valid=0;
        $error_strg="Year is not valid: $year";
    }

    $birthdate_ok = join(".",($day,$month,$year));
    return ($birthdate_valid,$birthdate_ok,$error_strg);
}



sub check_unid {
    my ($unid)=@_;
    $unid=&remove_embracing_whitespace($unid);
    return $unid;
}



sub check_login {
    my ($login)=@_;
    $login=&remove_embracing_whitespace($login);
    return $login;
}



# Encoding stuff
################################################################################
sub get_encoding {
    my ($filename)=@_;
    my $enc_used;
    my $enc=$sophomorix_config{'FILES'}{'USER_FILE'}{$filename}{ENCODING};
    my $enc_force=$sophomorix_config{'FILES'}{'USER_FILE'}{$filename}{ENCODING_FORCE};
    my $enc_checked=$sophomorix_config{'FILES'}{'USER_FILE'}{$filename}{ENCODING_CHECKED};

    # misconfigured *.school.conf: exit
    if ($enc eq "ERROR_ENCODING"){
        &log_script_exit("Misconfigured ENCODING in configuration file, see 'iconv --list'",1,1,0,
                     \@arguments,\%sophomorix_result,\%sophomorix_config,$json);
    }   
    if ($enc_force eq "ERROR_ENCODING_FORCE"){
        &log_script_exit("Misconfigured ENCODING_FORCE in configuration file",1,1,0,
                     \@arguments,\%sophomorix_result,\%sophomorix_config,$json);
    }   
 
    if ($enc eq "auto"){
        if ($enc_checked eq "unknown"){
            # auto failed
            &log_script_exit("ENCODING=auto failed: Set ENCODING to the desired value and ENCODING_FORCE to 'True'",1,1,0,
                     \@arguments,\%sophomorix_result,\%sophomorix_config,$json);
        } else {
            # auto was succesful
            $enc_used=$enc_checked;
	}
    } else {
        # encoding configured (NOT auto)
        if ($enc_force eq $sophomorix_config{'INI'}{'VARS'}{'BOOLEAN_FALSE'}){
            # encoding NOT forced
            if ($enc eq $enc_checked){
                # configured as checked: OK
                $enc_used=$enc;
            } else {
                # configured different than checked
		my $error_message="ENCODING and ENCODING_CHECKED do not match |".
                                  " A) set ENCODING_FORCE to 'True' if you are smarter than sophomorix |".
		                  " B) set ENCODING to 'auto' if you trust sophomorix";
                &log_script_exit($error_message,1,1,0,
                    \@arguments,\%sophomorix_result,\%sophomorix_config,$json);
            }
        } else {
            # encoding forced -> use it
            $enc_used=$enc;
        }
    }

    print "Encoding $filename: $enc_used (ENCODING=$enc,*_FORCE=$enc_force, *_CHECKED=$enc_checked)\n";
    return $enc_used;
}
