#!/usr/bin/perl -w
# This script (sophomorix-newfile) is maintained by Rüdiger Beck
# It is Free Software (License GPLv3)
# If you find errors, contact the author
# jeffbeck@web.de  or  jeffbeck@linuxmuster.net

# modules
use strict;
use Getopt::Long;
Getopt::Long::Configure ("bundling");
use Sophomorix::SophomorixConfig;
use List::MoreUtils qw(uniq);
use Net::LDAP;
use Data::Dumper;
$Data::Dumper::Indent = 1;
$Data::Dumper::Sortkeys = 1;
$Data::Dumper::Useqq = 1;
$Data::Dumper::Terse = 1; 
use JSON;
use File::Basename qw( basename
                       dirname
                     ); 
use Cwd 'abs_path';
use Sophomorix::SophomorixBase qw(
                                 print_line
                                 print_title
                                 unlock_sophomorix
                                 lock_sophomorix
                                 read_encoding_data
                                 analyze_encoding
                                 log_script_start
                                 log_script_end
                                 log_script_exit
                                 backup_auk_file
                                 get_passwd_charlist
                                 get_plain_password
                                 check_options
                                 config_sophomorix_read
                                 result_sophomorix_init
                                 result_sophomorix_add
                                 result_sophomorix_check_exit
                                 result_sophomorix_print
                                 filelist_fetch
                                 remove_whitespace
                                 json_dump
                                 recode_utf8_to_ascii
                                 );
use Sophomorix::SophomorixSambaAD qw(
                                 AD_school_create
                                 AD_bind_admin
                                 AD_unbind_admin
                                 AD_user_create
                                 AD_group_create
                                 AD_group_addmember
                                 AD_group_update
                                 AD_get_schoolname
                                 AD_get_name_tokened
                                 AD_dn_fetch_multivalue
                                 AD_dns_get
                                 AD_object_search
                                    );

my @arguments = @ARGV;

# option vars
$Conf::log_level=1;
my $help=0;
my $info=0;
my $json=0;
my $test_only=0;
my $keep_file=0;
my $name="";
my $show_special_char_lines=0;
my $non_umlaut=0;

# Parsen der Optionen
my $testopt=GetOptions(
           "help|h" => \$help,
           "info|i" => \$info,
           "json|j+" => \$json,
           "verbose|v+" => \$Conf::log_level,
           "name=s" => \$name,
           "show-special-char-lines" => \$show_special_char_lines,
           "non-umlaut|nonumlaut" => \$non_umlaut,
           "test-only" => \$test_only,
           "keep-file" => \$keep_file,
          );


my %sophomorix_result=&result_sophomorix_init("sophomorix-newfile");
# Prüfen, ob Optionen erkannt wurden, sonst Abbruch
&check_options($testopt,\%sophomorix_result,$json);

# last option is the abs path of the new file
my $newfile=$ARGV[0];


# --help
if ($help==1) {
   # Scriptname ermitteln
   my @list = split(/\//,$0);
   my $scriptname = pop @list;
   # Befehlsbeschreibung
   print('
sophomorix-newfile filters,converts and installs new user export files to the correct location in /etc/linuxmuster/sophomorix.

Logging ist done to:
  /var/log/sophomorix/etcfiles/     (backups from /etc)
  /var/log/sophomorix/newfiles/     (new files and there conversion steps)
     *.orig           unchanged file
     *.filtered       filtered/copied file(when there is no filter) 
     *.filtered.utf8  filtered and converted to UTF(this file is installed to /etc)

- New files are deleted after registration
  (option --test-only keeps the files)
- New files and overwritten files are logged
- a filter is applied if configured
  (check with --test-only)

Options:
  -h  / --help
  -v  / --verbose
  -vv / --verbose --verbose
  -i  / --info

Examples:
    sophomorix-newfile students.csv                            (target according to given file)
    sophomorix-newfile /path/to/students.csv                   (target according to given file)
    sophomorix-newfile /path/somefile.txt --name students.csv  (target according to given name)

    sophomorix-newfile /path/to/students.csv --test-only     (filtering/utf8 converting to stdout)
  
Options:
  --test-only               (keeps new file and does not install file to /etc)
  --keep-file               (keeps new file in its original location, installs file to /etc)
  --show-special-char-lines (show lines with special characters)
 
Please see the sophomorix-newfile(8) man pages for full documentation
');
   print "\n";
   exit;
}

# Reading Configuration
my ($ldap,$root_dse) = &AD_bind_admin(\@arguments,\%sophomorix_result,$json);
my $root_dns=&AD_dns_get($root_dse);
my %sophomorix_config=&config_sophomorix_read($ldap,$root_dse,\%sophomorix_result);
my @filelist=&filelist_fetch({filetype=>"users",
                              sophomorix_config=>\%sophomorix_config,
                            });


# --info
if ($info==1) {
    print "to be done where to log\n";
    exit;
}


&result_sophomorix_check_exit(\%sophomorix_result,\%sophomorix_config,$json);
################################################################################
# Start
################################################################################
&log_script_start(\@arguments,\%sophomorix_result,\%sophomorix_config);


if (not defined $newfile){
    print "\n ERROR: You need to specify a file\n\n";
    exit 88;
}

if (not -e  $newfile){
    print "\n ERROR: $newfile does not exist\n\n";
    exit 88;
}

if (not -f  $newfile){
    print "\n ERROR: $newfile must be a file\n\n";
    exit 88;
}


&print_title("Registering $newfile:");

my $basename = basename($newfile);
my $newfile_abs = abs_path($newfile);
my $target="";
my $target_school="";
my $newfile_log;
my $newfile_log_filter;
my $newfile_log_utf8;
my $etcfile_log;
my $filter_script;

my $tmp_dir=$sophomorix_config{'INI'}{'PATHS'}{'TMP_FILTERED'};
my $tmp=$tmp_dir."/newfile_filtered.tmp";

# calculate paths for logging
if ($name ne ""){
    # option --name ....
    if (exists $sophomorix_config{'FILES'}{'USER_FILE'}{$name}){
        $target=$sophomorix_config{'FILES'}{'USER_FILE'}{$name}{'PATH_ABS'};
        $target_school=$sophomorix_config{'FILES'}{'USER_FILE'}{$name}{'SCHOOL'};
        $newfile_log=$DevelConf::path_log_newfiles."/".$target_school."/".
            $sophomorix_config{'DATE'}{'LOCAL'}{'TIMESTAMP_FILE'}."_".$name.".orig";
        $newfile_log_filter=$DevelConf::path_log_newfiles."/".$target_school."/".
            $sophomorix_config{'DATE'}{'LOCAL'}{'TIMESTAMP_FILE'}."_".$name.".filter";
        $newfile_log_utf8=$DevelConf::path_log_newfiles."/".$target_school."/".
            $sophomorix_config{'DATE'}{'LOCAL'}{'TIMESTAMP_FILE'}."_".$name.".filter.utf8";
        $etcfile_log=$DevelConf::path_log_etcfiles."/".$target_school."/".
            $sophomorix_config{'DATE'}{'LOCAL'}{'TIMESTAMP_FILE'}."_".$name;
        $filter_script=$sophomorix_config{'FILES'}{'USER_FILE'}{$name}{'FILTERSCRIPT'};
    } else {
        # --name with wrong filename
        print "\nERROR: Your --name $name is not a valid filename on this server\n\n";
        print "   * Please use a valid filename so I can find out where to copy your new file.\n\n";
        &print_valid_filenames;
        exit 88;
    }
} elsif (exists $sophomorix_config{'FILES'}{'USER_FILE'}{$basename}){
    # valid filename
    $target=$sophomorix_config{'FILES'}{'USER_FILE'}{$basename}{'PATH_ABS'};
    $target_school=$sophomorix_config{'FILES'}{'USER_FILE'}{$basename}{'SCHOOL'};
    $newfile_log=$DevelConf::path_log_newfiles."/".$target_school."/".
        $sophomorix_config{'DATE'}{'LOCAL'}{'TIMESTAMP_FILE'}."_".$basename.".orig";
    $newfile_log_filter=$DevelConf::path_log_newfiles."/".$target_school."/".
        $sophomorix_config{'DATE'}{'LOCAL'}{'TIMESTAMP_FILE'}."_".$basename.".filter";
    $newfile_log_utf8=$DevelConf::path_log_newfiles."/".$target_school."/".
        $sophomorix_config{'DATE'}{'LOCAL'}{'TIMESTAMP_FILE'}."_".$basename.".filter.utf8";
    $etcfile_log=$DevelConf::path_log_etcfiles."/".$target_school."/".
        $sophomorix_config{'DATE'}{'LOCAL'}{'TIMESTAMP_FILE'}."_".$basename;
    $filter_script=$sophomorix_config{'FILES'}{'USER_FILE'}{$basename}{'FILTERSCRIPT'};
} else {
    # not valid filename
    print "\nERROR: Your filename $basename is not a valid filename on this server\n\n";
    print "   * Please rename the file or use option --name <name.csv> so I can find out where to copy it.\n\n";
    &print_valid_filenames;
    exit 88;
}


my $ref_encoding_data=&read_encoding_data();
my $enc;
my $ref_encoding_check_results;

# cp or filter temporary 
if ($filter_script eq "---"){
    print "cp, not filtering\n";
    system("mkdir -p $tmp_dir");
    system("cp '$newfile_abs' $tmp")
} elsif ($filter_script eq "ERROR_FILTERSCRIPT"){
    # this will never happen. The config check checks already if the configured script is existing and executable
    print "\nERROR: The configured Filter is not an executable script:\n";
    print "   $sophomorix_config{'FILES'}{'USER_FILE'}{$basename}{FILTERSCRIPT_CONFIGURED}\n";
    print "\n";
    my $error_message="Your configured filter $filter_script ist not an executable script!";
    &log_script_exit($error_message,1,1,0,
                     \@arguments,\%sophomorix_result,\%sophomorix_config,$json);
} else {
    # filter to tmp
    print "filtering, not cp\n";
    system("mkdir -p $tmp_dir");
    my $command="$filter_script '$newfile_abs' $tmp";
    print "$command\n";
    system("$filter_script '$newfile_abs' $tmp");
}


if($Conf::log_level>=3){
    print "\n";
    print "Analyzing encoding of $tmp\n";
}
($enc,$ref_encoding_check_results)=&analyze_encoding($tmp,
                                                     $tmp,
                                                     $show_special_char_lines,
                                                     $non_umlaut,
                                                     $ref_encoding_data,
                                                     $ref_encoding_check_results,
                                                     \%sophomorix_config,
                                                     \%sophomorix_result,
                                                    );


print "\n";
print "ENCODING is $enc ($ref_encoding_check_results->{$tmp}{'SURE'}, $tmp)\n";
print "\n";
print "   * New file:          $basename  (Renamed to: $name)\n";
print "     * Newfile_abs:     $newfile_abs\n";
print "   * Target file:       $target\n";
print "   * Target school:     $target_school\n";
print "   * Logging new files:\n";
print "     * original file:   $newfile_log\n";
print "     * filtered file:   $newfile_log_filter\n";
print "     * filter + utf8:   $newfile_log_utf8\n";
print "   * Loggging old file:\n";
print "     * file from etc:   $etcfile_log\n";
print "   * Filterscript:      $filter_script\n";
print "     * filtered tmp to: $tmp\n";


# exit if something is wrong
############################################################
# EXIT if autoconversion to utf8 can't be done
if ($enc eq "unknown" or ($ref_encoding_check_results->{$tmp}{'SURE'} ne "TRUE")){
    print "\n";
    print "ERROR: Encoding could not be determined for this file:\n";
    print "   $newfile\n";
    print "\n";
    print "NOT doing anything!\n";
    print "\n";
    print "For hints to what went wrong use:\n";
    print "  sophomorix-check -vv --analyze $newfile\n";
    print "\n";
    my $error_message="Encoding for $newfile could not be determined!";
    &log_script_exit($error_message,1,1,0,
                     \@arguments,\%sophomorix_result,\%sophomorix_config,$json);
}




############################################################
# do it

my $mkdir_command_1="mkdir -p $DevelConf::path_log_newfiles"."/".$target_school;
system($mkdir_command_1);

my $mkdir_command_2="mkdir -p $DevelConf::path_log_etcfiles"."/".$target_school;
system($mkdir_command_2);

print "\n";


# 1) backup etcfile, if there
if (-e $target and $test_only==0){
    my $command_1="cp $target $etcfile_log";
    print "1) Logging old etcfile $name:\n";
    print "   $command_1\n";
    system($command_1);
} else {
    print "1) NO file to log: $target\n";
}


# 2) cp newfile to log as *.orig
my $command_2="cp $newfile_abs $newfile_log";
print "2) Logging newfile as *.orig:\n";
print "   $command_2\n";
system($command_2);


# 3) *.filter or NOT to *.filter
if ($filter_script eq "---"){
    print "3) NOT filtering ($filter_script), just logging as *.filter:\n";
    my $command_3="cp $newfile_abs $newfile_log_filter";
    print "   $command_3\n";
    system($command_3);
} else {
    print "3) Logging(moving) temporary filtered file as *.filter\n";
    my $command_3="mv $tmp $newfile_log_filter";
    print "   $command_3\n";
    system($command_3);
}


# 4) cp or iconv to *.filter.utf8
if ($enc eq "UTF8" and $ref_encoding_check_results->{$tmp}{'SURE'} eq "TRUE"){
    my $command_4="cp $newfile_log_filter $newfile_log_utf8";
    print "4) File is $enc: just logging filtered file as *.filter.utf8:\n";
    print "   $command_4\n";
    system($command_4);
} else {
    my $command_4="iconv -f $enc -t UTF8 -o $newfile_log_utf8 $newfile_log_filter";
    print "4) File is $enc: converting to UTF8 as *.filter.utf8:\n";
    print "   $command_4\n";
    system($command_4);
}

if ($test_only==1){
    print "\n";
    print "The following steps were omitted(--test-only):\n";
    print "     1) Logging old etcfile $name\n";
    print "     5) Install filtered utf8 file in /etc\n";
    print "     6) Deleting newfile\n";
    exit;
}


# 5) install utf8 file in /etc
my $command_5="cp $newfile_log_utf8 $target";
print "5) Install filtered utf8 file in /etc:\n";
print "   $command_5\n";
system($command_5);


# 6) delete newfile
if ($keep_file==0){
    my $command_6="rm $newfile_abs";
    print "6) Deleting newfile:\n";
    print "   $command_6\n";
    system($command_6);
} else {
    print "6) NOT deleting newfile because of option --keep-file:\n";
    print "   $newfile_abs\n";
}

print "\n";

&AD_unbind_admin($ldap);

&log_script_end(\@arguments,\%sophomorix_result,\%sophomorix_config,$json);



################################################################################
# Sub
################################################################################

sub print_valid_filenames {
    my @list=();
    foreach my $key (keys %{$sophomorix_config{'FILES'}{'USER_FILE'}}) {
	push @list, $key;
    }
    @list=sort @list;
    print "   * Valid filenames on this server are:\n";
    foreach my $file (@list){
        print "      * $file\n";
    }
    print "\n";
}
