/* * get_file_names.c - Part of AFD, an automatic file distribution program. * Copyright (c) 1996 - 2022 Holger Kiehl * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; either version 2 of the License, or * (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. */ #include "afddefs.h" DESCR__S_M3 /* ** NAME ** get_file_names - gets the name of all files in a directory ** ** SYNOPSIS ** int get_file_names(char *file_path, off_t *file_size_to_send) ** ** DESCRIPTION ** ** RETURN VALUES ** The number of files, total file size it has found in the directory ** and the directory where the files have been found. If all files ** are deleted due to age limit, it will return -1. Otherwise if ** an error occurs it will exit. ** ** AUTHOR ** H.Kiehl ** ** HISTORY ** 31.12.1996 H.Kiehl Created ** 31.01.1997 H.Kiehl Support for age limit. ** 14.10.1998 H.Kiehl Free unused memory. ** 03.02.2001 H.Kiehl Sort the files by date. ** 21.08.2001 H.Kiehl Return -1 if all files have been deleted due ** to age limit. ** 20.12.2004 H.Kiehl Addition of delete file name buffer. ** 14.02.2009 H.Kiehl Log to OUTPUT_LOG when we delete a file. ** */ DESCR__E_M3 #include /* NULL */ #include /* strcpy(), strcat(), strcmp(), */ /* strerror() */ #include /* realloc(), malloc(), free() */ #include /* time() */ #include #include #include /* opendir(), closedir(), DIR, */ /* readdir(), struct dirent */ #include #include /* unlink(), getpid() */ #include #include "fddefs.h" /* Global variables. */ extern int counter_fd, files_to_delete, fsa_fd, *unique_counter; #ifdef HAVE_HW_CRC32 extern int have_hw_crc32; #endif #ifdef _OUTPUT_LOG extern int ol_fd; # ifdef WITHOUT_FIFO_RW_SUPPORT extern int ol_readfd; # endif extern unsigned int *ol_job_number, *ol_retries; extern char *ol_data, *ol_file_name, *ol_output_type; extern unsigned short *ol_archive_name_length, *ol_file_name_length, *ol_unl; extern off_t *ol_file_size; extern size_t ol_size, ol_real_size; extern clock_t *ol_transfer_time; #endif extern off_t *file_size_buffer; extern time_t *file_mtime_buffer; extern char *p_work_dir, tr_hostname[], *del_file_name_buffer, *file_name_buffer; extern struct filetransfer_status *fsa; extern struct job db; #ifdef WITH_DUP_CHECK extern struct rule *rule; #endif #ifdef _DELETE_LOG extern struct delete_log dl; #endif #if defined (_DELETE_LOG) || defined (_OUTPUT_LOG) /* Local function prototypes. */ static void log_data(char *, #ifdef HAVE_STATX struct statx *, #else struct stat *, #endif # ifdef WITH_DUP_CHECK int, # endif # ifdef _OUTPUT_LOG char, # endif time_t); #endif /*########################## get_file_names() ###########################*/ int get_file_names(char *file_path, off_t *file_size_to_send) { #ifdef WITH_DUP_CHECK int dup_counter = 0; off_t dup_counter_size = 0; #endif time_t diff_time, now; int files_not_send = 0, files_to_send = 0, new_size, offset; off_t file_size_not_send = 0, *p_file_size; time_t *p_file_mtime; char fullname[MAX_PATH_LENGTH], *p_del_file_name, *p_file_name, *p_source_file; #ifdef HAVE_STATX struct statx stat_buf; #else struct stat stat_buf; #endif struct dirent *p_dir; DIR *dp; /* * Create directory name in which we can find the files for this job. */ (void)strcpy(file_path, p_work_dir); (void)strcat(file_path, AFD_FILE_DIR); (void)strcat(file_path, OUTGOING_DIR); (void)strcat(file_path, "/"); (void)strcat(file_path, db.msg_name); db.p_unique_name = db.msg_name; #ifdef MULTI_FS_SUPPORT while ((*db.p_unique_name != '/') && (*db.p_unique_name != '\0')) { db.p_unique_name++; /* Away with the filesystem ID. */ } if (*db.p_unique_name == '/') { db.p_unique_name++; #endif while ((*db.p_unique_name != '/') && (*db.p_unique_name != '\0')) { db.p_unique_name++; /* Away with the job ID. */ } if (*db.p_unique_name == '/') { db.p_unique_name++; while ((*db.p_unique_name != '/') && (*db.p_unique_name != '\0')) { db.p_unique_name++; /* Away with the dir number. */ } if (*db.p_unique_name == '/') { int i; char str_num[MAX_TIME_T_HEX_LENGTH + 1]; db.p_unique_name++; i = 0; while ((i < MAX_TIME_T_HEX_LENGTH) && (*(db.p_unique_name + i) != '_') && (*(db.p_unique_name + i) != '\0')) { str_num[i] = *(db.p_unique_name + i); i++; } if ((*(db.p_unique_name + i) != '_') || (i == 0)) { system_log(ERROR_SIGN, __FILE__, __LINE__, "Could not determine message name from `%s'. #%x", db.msg_name, db.id.job); exit(SYNTAX_ERROR); } str_num[i] = '\0'; db.creation_time = (time_t)str2timet(str_num, NULL, 16); db.unl = i + 1; i = 0; while ((i < MAX_INT_HEX_LENGTH) && (*(db.p_unique_name + db.unl + i) != '_') && (*(db.p_unique_name + db.unl + i) != '\0')) { str_num[i] = *(db.p_unique_name + db.unl + i); i++; } if ((*(db.p_unique_name + db.unl + i) != '_') || (i == 0)) { system_log(ERROR_SIGN, __FILE__, __LINE__, "Could not determine message name from `%s'. #%x", db.msg_name, db.id.job); exit(SYNTAX_ERROR); } str_num[i] = '\0'; db.unique_number = (unsigned int)strtoul(str_num, NULL, 16); db.unl = db.unl + i + 1; i = 0; while ((i < MAX_INT_HEX_LENGTH) && (*(db.p_unique_name + db.unl + i) != '\0')) { str_num[i] = *(db.p_unique_name + db.unl + i); i++; } if (i == 0) { system_log(ERROR_SIGN, __FILE__, __LINE__, "Could not determine message name from `%s'. #%x", db.msg_name, db.id.job); exit(SYNTAX_ERROR); } str_num[i] = '\0'; db.split_job_counter = (unsigned int)strtoul(str_num, NULL, 16); db.unl = db.unl + i; if (db.unl == 0) { system_log(ERROR_SIGN, __FILE__, __LINE__, "Could not determine message name from `%s'. #%x", db.msg_name, db.id.job); exit(SYNTAX_ERROR); } } else { system_log(ERROR_SIGN, __FILE__, __LINE__, "Could not determine message name from `%s'. #%x", db.msg_name, db.id.job); exit(SYNTAX_ERROR); } } else { system_log(ERROR_SIGN, __FILE__, __LINE__, "Could not determine message name from `%s'. #%x", db.msg_name, db.id.job); exit(SYNTAX_ERROR); } #ifdef MULTI_FS_SUPPORT } else { system_log(ERROR_SIGN, __FILE__, __LINE__, "Could not determine message name from `%s'. #%x", db.msg_name, db.id.job); exit(SYNTAX_ERROR); } #endif if ((dp = opendir(file_path)) == NULL) { system_log((errno != ENOENT) ? ERROR_SIGN : WARN_SIGN, __FILE__, __LINE__, "Could not opendir() %s [%s %d] : %s #%x", file_path, db.host_alias, (int)db.job_no, strerror(errno), db.id.job); exit(OPEN_FILE_DIR_ERROR); } /* Prepare pointers and directory name. */ (void)strcpy(fullname, file_path); p_source_file = fullname + strlen(fullname); *p_source_file++ = '/'; if (file_name_buffer != NULL) { free(file_name_buffer); file_name_buffer = NULL; } p_file_name = file_name_buffer; if (file_size_buffer != NULL) { free(file_size_buffer); file_size_buffer = NULL; } p_file_size = file_size_buffer; if (file_mtime_buffer != NULL) { free(file_mtime_buffer); file_mtime_buffer = NULL; } p_file_mtime = file_mtime_buffer; if (del_file_name_buffer != NULL) { free(del_file_name_buffer); del_file_name_buffer = NULL; } p_del_file_name = del_file_name_buffer; files_to_delete = 0; now = time(NULL); /* * Now let's determine the number of files that have to be * transmitted and the total size. */ errno = 0; while ((p_dir = readdir(dp)) != NULL) { if ( #ifdef LINUX (p_dir->d_type != DT_REG) || #endif ((p_dir->d_name[0] == '.') && (p_dir->d_name[1] == '\0')) || ((p_dir->d_name[0] == '.') && (p_dir->d_name[1] == '.') && (p_dir->d_name[2] == '\0'))) { continue; } (void)strcpy(p_source_file, p_dir->d_name); #ifdef HAVE_STATX if (statx(0, fullname, AT_STATX_SYNC_AS_STAT, # ifndef LINUX STATX_MODE | # endif STATX_SIZE | STATX_MTIME, &stat_buf) == -1) #else if (stat(fullname, &stat_buf) == -1) #endif { system_log(WARN_SIGN, __FILE__, __LINE__, #ifdef HAVE_STATX "Can't statx() file `%s' : %s #%x", #else "Can't stat() file `%s' : %s #%x", #endif fullname, strerror(errno), db.id.job); continue; } #ifndef LINUX /* Sure it is a normal file? */ # ifdef HAVE_STATX if (S_ISREG(stat_buf.stx_mode)) # else if (S_ISREG(stat_buf.st_mode)) # endif #endif { #ifdef WITH_DUP_CHECK int is_duplicate = NO; #endif int remove_file = NO; #ifdef HAVE_STATX if (now < stat_buf.stx_mtime.tv_sec) #else if (now < stat_buf.st_mtime) #endif { diff_time = 0L; } else { #ifdef HAVE_STATX diff_time = now - stat_buf.stx_mtime.tv_sec; #else diff_time = now - stat_buf.st_mtime; #endif } /* Don't send files older then age_limit! */ if ((db.age_limit > 0) && ((fsa->host_status & DO_NOT_DELETE_DATA) == 0) && (diff_time > db.age_limit)) { remove_file = YES; } else { #ifdef WITH_DUP_CHECK if (((db.age_limit > 0) && ((fsa->host_status & DO_NOT_DELETE_DATA) == 0) && (diff_time > db.age_limit)) || ((db.dup_check_timeout > 0) && # ifdef FAST_SF_DUPCHECK ((db.special_flag & OLD_ERROR_JOB) == 0) && (((is_duplicate = isdup(fullname, p_dir->d_name, # ifdef HAVE_STATX stat_buf.stx_size, # else stat_buf.st_size, # endif db.crc_id, db.dup_check_timeout, db.dup_check_flag, NO, # ifdef HAVE_HW_CRC32 have_hw_crc32, # endif YES, YES)) == YES) && ((db.dup_check_flag & DC_DELETE) || (db.dup_check_flag & DC_STORE))) # else (((is_duplicate = isdup(fullname, p_dir->d_name, # ifdef HAVE_STATX stat_buf.stx_size, # else stat_buf.st_size, # endif db.crc_id, 0, db.dup_check_flag, YES, # ifdef HAVE_HW_CRC32 have_hw_crc32, # endif YES, YES)) == YES) && ((db.dup_check_flag & DC_DELETE) || (db.dup_check_flag & DC_STORE))) # endif )) { remove_file = YES; } else { if ((db.trans_dup_check_timeout > 0) && ((db.special_flag & OLD_ERROR_JOB) == 0)) { register int k; char tmp_filename[MAX_PATH_LENGTH]; tmp_filename[0] = '\0'; for (k = 0; k < rule[db.trans_rule_pos].no_of_rules; k++) { if (pmatch(rule[db.trans_rule_pos].filter[k], p_dir->d_name, NULL) == 0) { change_name(p_dir->d_name, rule[db.trans_rule_pos].filter[k], rule[db.trans_rule_pos].rename_to[k], tmp_filename, MAX_PATH_LENGTH, &counter_fd, &unique_counter, db.id.job); break; } } if (tmp_filename[0] != '\0') { if (((is_duplicate = isdup(fullname, tmp_filename, # ifdef HAVE_STATX stat_buf.stx_size, # else stat_buf.st_size, # endif db.crc_id, db.trans_dup_check_timeout, db.trans_dup_check_flag, NO, # ifdef HAVE_HW_CRC32 have_hw_crc32, # endif YES, YES)) == YES) && ((db.trans_dup_check_flag & DC_DELETE) || (db.trans_dup_check_flag & DC_STORE))) { remove_file = YES; } } } } #endif } if (remove_file == YES) { char file_to_remove[MAX_FILENAME_LENGTH]; file_to_remove[0] = '\0'; if (db.no_of_restart_files > 0) { int ii; char initial_filename[MAX_FILENAME_LENGTH]; if ((db.lock == DOT) || (db.lock == DOT_VMS)) { (void)strcpy(initial_filename, db.lock_notation); (void)strcat(initial_filename, p_dir->d_name); } else { (void)strcpy(initial_filename, p_dir->d_name); } for (ii = 0; ii < db.no_of_restart_files; ii++) { if ((CHECK_STRCMP(db.restart_file[ii], initial_filename) == 0) && (append_compare(db.restart_file[ii], fullname) == YES)) { (void)strcpy(file_to_remove, db.restart_file[ii]); remove_append(db.id.job, db.restart_file[ii]); break; } } } #ifdef WITH_DUP_CHECK if (is_duplicate == YES) { dup_counter++; # ifdef HAVE_STATX dup_counter_size += stat_buf.stx_size; # else dup_counter_size += stat_buf.st_size; # endif if (db.dup_check_flag & DC_WARN) { trans_log(WARN_SIGN, __FILE__, __LINE__, NULL, NULL, "File `%s' is duplicate. #%x", p_dir->d_name, db.id.job); } } if ((is_duplicate == YES) && (db.dup_check_flag & DC_STORE)) { char *p_end, save_dir[MAX_PATH_LENGTH]; p_end = save_dir + snprintf(save_dir, MAX_PATH_LENGTH, "%s%s%s/%x/", p_work_dir, AFD_FILE_DIR, STORE_DIR, db.id.job); if ((mkdir(save_dir, DIR_MODE) == -1) && (errno != EEXIST)) { system_log(WARN_SIGN, __FILE__, __LINE__, "Failed to mkdir() `%s' : %s", save_dir, strerror(errno)); if (unlink(fullname) == -1) { system_log(WARN_SIGN, __FILE__, __LINE__, "Failed to unlink() file `%s' due to duplicate check : %s #%x", p_dir->d_name, strerror(errno), db.id.job); } # if defined (_DELETE_LOG) || defined (_OUTPUT_LOG) else { log_data(p_dir->d_name, &stat_buf, YES, # ifdef _OUTPUT_LOG OT_DUPLICATE_DELETE + '0', # endif now); } # endif } else { (void)strcpy(p_end, p_dir->d_name); if (rename(fullname, save_dir) == -1) { system_log(WARN_SIGN, __FILE__, __LINE__, "Failed to rename() `%s' to `%s' : %s #%x", fullname, save_dir, strerror(errno), db.id.job); if (unlink(fullname) == -1) { system_log(WARN_SIGN, __FILE__, __LINE__, "Failed to unlink() file `%s' due to duplicate check : %s #%x", p_dir->d_name, strerror(errno), db.id.job); } # if defined (_DELETE_LOG) || defined (_OUTPUT_LOG) else { log_data(p_dir->d_name, &stat_buf, YES, # ifdef _OUTPUT_LOG OT_DUPLICATE_STORED + '0', # endif now); } # endif } } files_not_send++; # ifdef HAVE_STATX file_size_not_send += stat_buf.stx_size; # else file_size_not_send += stat_buf.st_size; # endif } else { #endif /* WITH_DUP_CHECK */ if (unlink(fullname) == -1) { system_log(ERROR_SIGN, __FILE__, __LINE__, "Failed to unlink() file `%s' due to age : %s #%x", p_dir->d_name, strerror(errno), db.id.job); } else { #if defined (_DELETE_LOG) || defined (_OUTPUT_LOG) log_data(p_dir->d_name, &stat_buf, # ifdef WITH_DUP_CHECK is_duplicate, # ifdef _OUTPUT_LOG (is_duplicate == YES) ? (OT_DUPLICATE_DELETE + '0') : (OT_AGE_LIMIT_DELETE + '0'), # endif # else # ifdef _OUTPUT_LOG OT_AGE_LIMIT_DELETE + '0', # endif # endif now); #endif #ifndef _DELETE_LOG if ((db.protocol & FTP_FLAG) || (db.protocol & SFTP_FLAG)) { if (db.no_of_restart_files > 0) { int append_file_number = -1, ii; for (ii = 0; ii < db.no_of_restart_files; ii++) { if (CHECK_STRCMP(db.restart_file[ii], p_dir->d_name) == 0) { append_file_number = ii; break; } } if (append_file_number != -1) { remove_append(db.id.job, db.restart_file[append_file_number]); } } } #endif if (file_to_remove[0] != '\0') { if ((files_to_delete % 20) == 0) { /* Increase the space for the delete file name buffer. */ new_size = ((files_to_delete / 20) + 1) * 20 * MAX_FILENAME_LENGTH; offset = p_del_file_name - del_file_name_buffer; if ((del_file_name_buffer = realloc(del_file_name_buffer, new_size)) == NULL) { system_log(ERROR_SIGN, __FILE__, __LINE__, "Could not realloc() memory : %s #%x", strerror(errno), db.id.job); exit(ALLOC_ERROR); } p_del_file_name = del_file_name_buffer + offset; } (void)strcpy(p_del_file_name, file_to_remove); p_del_file_name += MAX_FILENAME_LENGTH; files_to_delete++; } files_not_send++; #ifdef HAVE_STATX file_size_not_send += stat_buf.stx_size; #else file_size_not_send += stat_buf.st_size; #endif } #ifdef WITH_DUP_CHECK } #endif } else { #ifdef WITH_DUP_CHECK if ((is_duplicate == YES) && (db.dup_check_flag & DC_WARN)) { trans_log(WARN_SIGN, __FILE__, __LINE__, NULL, NULL, "File `%s' is duplicate. #%x", p_dir->d_name, db.id.job); } #endif if ((files_to_send % 20) == 0) { /* Increase the space for the file name buffer. */ new_size = ((files_to_send / 20) + 1) * 20 * MAX_FILENAME_LENGTH; offset = p_file_name - file_name_buffer; if ((file_name_buffer = realloc(file_name_buffer, new_size)) == NULL) { system_log(ERROR_SIGN, __FILE__, __LINE__, "Could not realloc() memory : %s #%x", strerror(errno), db.id.job); exit(ALLOC_ERROR); } p_file_name = file_name_buffer + offset; /* Increase the space for the file size buffer. */ new_size = ((files_to_send / 20) + 1) * 20 * sizeof(off_t); offset = (char *)p_file_size - (char *)file_size_buffer; if ((file_size_buffer = realloc(file_size_buffer, new_size)) == NULL) { system_log(ERROR_SIGN, __FILE__, __LINE__, "Could not realloc() memory : %s #%x", strerror(errno), db.id.job); exit(ALLOC_ERROR); } p_file_size = (off_t *)((char *)file_size_buffer + offset); if ((fsa->protocol_options & SORT_FILE_NAMES) || #ifdef WITH_EUMETSAT_HEADERS (db.special_flag & ADD_EUMETSAT_HEADER) || #endif (fsa->protocol_options & KEEP_TIME_STAMP)) { /* Increase the space for the file mtime buffer. */ new_size = ((files_to_send / 20) + 1) * 20 * sizeof(time_t); offset = (char *)p_file_mtime - (char *)file_mtime_buffer; if ((file_mtime_buffer = realloc(file_mtime_buffer, new_size)) == NULL) { system_log(ERROR_SIGN, __FILE__, __LINE__, "Could not realloc() memory : %s #%x", strerror(errno), db.id.job); exit(ALLOC_ERROR); } p_file_mtime = (time_t *)((char *)file_mtime_buffer + offset); } } /* Sort the files, newest must be last (FIFO). */ if ((fsa->protocol_options & SORT_FILE_NAMES) && (files_to_send > 1) && #ifdef HAVE_STATX (*(p_file_mtime - 1) > stat_buf.stx_mtime.tv_sec) #else (*(p_file_mtime - 1) > stat_buf.st_mtime) #endif ) { int i; off_t *sp_file_size = p_file_size - 1; time_t *sp_mtime = p_file_mtime - 1; char *sp_file_name = p_file_name - MAX_FILENAME_LENGTH; for (i = files_to_send; i > 0; i--) { #ifdef HAVE_STATX if (*sp_mtime <= stat_buf.stx_mtime.tv_sec) #else if (*sp_mtime <= stat_buf.st_mtime) #endif { break; } sp_mtime--; sp_file_size--; sp_file_name -= MAX_FILENAME_LENGTH; } (void)memmove(sp_mtime + 2, sp_mtime + 1, (files_to_send - i) * sizeof(time_t)); #ifdef HAVE_STATX *(sp_mtime + 1) = stat_buf.stx_mtime.tv_sec; #else *(sp_mtime + 1) = stat_buf.st_mtime; #endif (void)memmove(sp_file_size + 2, sp_file_size + 1, (files_to_send - i) * sizeof(off_t)); #ifdef HAVE_STATX *(sp_file_size + 1) = stat_buf.stx_size; #else *(sp_file_size + 1) = stat_buf.st_size; #endif (void)memmove(sp_file_name + (2 * MAX_FILENAME_LENGTH), sp_file_name + MAX_FILENAME_LENGTH, (files_to_send - i) * MAX_FILENAME_LENGTH); (void)strcpy(sp_file_name + MAX_FILENAME_LENGTH, p_dir->d_name); } else { (void)strcpy(p_file_name, p_dir->d_name); #ifdef HAVE_STATX *p_file_size = stat_buf.stx_size; #else *p_file_size = stat_buf.st_size; #endif if (file_mtime_buffer != NULL) { #ifdef HAVE_STATX *p_file_mtime = stat_buf.stx_mtime.tv_sec; #else *p_file_mtime = stat_buf.st_mtime; #endif } } p_file_name += MAX_FILENAME_LENGTH; p_file_size++; p_file_mtime++; files_to_send++; #ifdef HAVE_STATX *file_size_to_send += stat_buf.stx_size; #else *file_size_to_send += stat_buf.st_size; #endif } } errno = 0; } /* while ((p_dir = readdir(dp)) != NULL) */ if (errno) { system_log(ERROR_SIGN, __FILE__, __LINE__, "Could not readdir() `%s' : %s #%x", file_path, strerror(errno), db.id.job); } #ifdef WITH_DUP_CHECK isdup_detach(); #endif if ((file_mtime_buffer != NULL) && #ifdef WITH_EUMETSAT_HEADERS ((db.special_flag & ADD_EUMETSAT_HEADER) == 0) && #endif ((fsa->protocol_options & KEEP_TIME_STAMP) == 0)) { free(file_mtime_buffer); file_mtime_buffer = NULL; } if (files_not_send > 0) { /* Total file counter. */ #ifdef LOCK_DEBUG lock_region_w(fsa_fd, db.lock_offset + LOCK_TFC, __FILE__, __LINE__); #else lock_region_w(fsa_fd, db.lock_offset + LOCK_TFC); #endif fsa->total_file_counter -= files_not_send; #ifdef _VERIFY_FSA if (fsa->total_file_counter < 0) { system_log(DEBUG_SIGN, __FILE__, __LINE__, "Total file counter for host `%s' less then zero. Correcting to 0. #%x", fsa->host_dsp_name, db.id.job); fsa->total_file_counter = 0; } #endif /* Total file size. */ fsa->total_file_size -= file_size_not_send; #ifdef _VERIFY_FSA if (fsa->total_file_size < 0) { system_log(DEBUG_SIGN, __FILE__, __LINE__, "Total file size for host `%s' overflowed. Correcting to 0. #%x", fsa->host_dsp_name, db.id.job); fsa->total_file_size = 0; } else if ((fsa->total_file_counter == 0) && (fsa->total_file_size > 0)) { system_log(DEBUG_SIGN, __FILE__, __LINE__, "fc for host `%s' is zero but fs is not zero. Correcting to 0. #%x", fsa->host_dsp_name, db.id.job); fsa->total_file_size = 0; } #endif if ((fsa->total_file_counter == 0) && (fsa->total_file_size == 0) && (fsa->error_counter > 0)) { int j; #ifdef LOCK_DEBUG lock_region_w(fsa_fd, db.lock_offset + LOCK_EC, __FILE__, __LINE__); #else lock_region_w(fsa_fd, db.lock_offset + LOCK_EC); #endif fsa->error_counter = 0; /* * Remove the error condition (NOT_WORKING) from all jobs * of this host. */ for (j = 0; j < fsa->allowed_transfers; j++) { if ((j != db.job_no) && (fsa->job_status[j].connect_status == NOT_WORKING)) { fsa->job_status[j].connect_status = DISCONNECT; } } fsa->error_history[0] = 0; fsa->error_history[1] = 0; #ifdef LOCK_DEBUG unlock_region(fsa_fd, db.lock_offset + LOCK_EC, __FILE__, __LINE__); #else unlock_region(fsa_fd, db.lock_offset + LOCK_EC); #endif } #ifdef LOCK_DEBUG unlock_region(fsa_fd, db.lock_offset + LOCK_TFC, __FILE__, __LINE__); #else unlock_region(fsa_fd, db.lock_offset + LOCK_TFC); #endif #ifdef WITH_DUP_CHECK if (dup_counter > 0) { trans_log(INFO_SIGN, NULL, 0, NULL, NULL, # if SIZEOF_OFF_T == 4 "Deleted %d duplicate file(s) (%ld bytes). #%x", # else "Deleted %d duplicate file(s) (%lld bytes). #%x", # endif dup_counter, (pri_off_t)dup_counter_size, db.id.job); } if ((files_not_send - dup_counter) > 0) { trans_log(INFO_SIGN, NULL, 0, NULL, NULL, # if SIZEOF_OFF_T == 4 "Deleted %d file(s) (%ld bytes) due to age. #%x", # else "Deleted %d file(s) (%lld bytes) due to age. #%x", # endif files_not_send - dup_counter, (pri_off_t)(file_size_not_send - dup_counter_size), db.id.job); } #else trans_log(INFO_SIGN, NULL, 0, NULL, NULL, # if SIZEOF_OFF_T == 4 "Deleted %d file(s) (%ld bytes) due to age. #%x", # else "Deleted %d file(s) (%lld bytes) due to age. #%x", # endif files_not_send, (pri_off_t)file_size_not_send, db.id.job); #endif #ifdef WITH_ERROR_QUEUE if ((files_to_send == 0) && (fsa->host_status & ERROR_QUEUE_SET) && (check_error_queue(db.id.job, -1, 0, 0) == YES)) { (void)remove_from_error_queue(db.id.job, fsa, db.fsa_pos, fsa_fd); } #endif } if (closedir(dp) < 0) { system_log(ERROR_SIGN, __FILE__, __LINE__, "Could not closedir() `%s' : %s #%x", file_path, strerror(errno), db.id.job); exit(OPEN_FILE_DIR_ERROR); } /* * If we just return zero here when all files have been deleted * due to age and sf_xxx is bursting it does not know if this * is an error situation or not. So return -1 if we deleted all * files. */ if ((files_to_send == 0) && (files_not_send > 0)) { files_to_send = -1; } return(files_to_send); } #if defined (_DELETE_LOG) || defined (_OUTPUT_LOG) /*++++++++++++++++++++++++++++++ log_data() +++++++++++++++++++++++++++++*/ static void log_data(char *d_name, #ifdef HAVE_STATX struct statx *stat_buf, #else struct stat *stat_buf, #endif # ifdef WITH_DUP_CHECK int is_duplicate, # endif # ifdef _OUTPUT_LOG char output_type, # endif time_t now) { # ifdef _DELETE_LOG int prog_name_length; size_t dl_real_size; char str_diff_time[2 + MAX_LONG_LENGTH + 6 + MAX_LONG_LENGTH + 12 + MAX_LONG_LENGTH + 30 + 1]; # endif # ifdef _OUTPUT_LOG if (db.output_log == YES) { if (ol_fd == -2) { # ifdef WITHOUT_FIFO_RW_SUPPORT output_log_fd(&ol_fd, &ol_readfd, &db.output_log); # else output_log_fd(&ol_fd, &db.output_log); # endif } if (ol_fd > -1) { if (ol_data == NULL) { int current_toggle, protocol; if (db.protocol & FTP_FLAG) { # ifdef WITH_SSL if (db.tls_auth == NO) { protocol = FTP; } else { protocol = FTPS; } # else protocol = FTP; # endif } else if (db.protocol & LOC_FLAG) { protocol = LOC; } else if (db.protocol & EXEC_FLAG) { protocol = EXEC; } else if (db.protocol & HTTP_FLAG) { # ifdef WITH_SSL if (db.tls_auth == NO) { protocol = HTTP; } else { protocol = HTTPS; } # else protocol = HTTP; # endif } else if (db.protocol & SFTP_FLAG) { protocol = SFTP; } # ifdef _WITH_SCP_SUPPORT else if (db.protocol & SCP_FLAG) { protocol = SCP; } # endif # ifdef _WITH_WMO_SUPPORT else if (db.protocol & WMO_FLAG) { protocol = WMO; } # endif # ifdef _WITH_MAP_SUPPORT else if (db.protocol & MAP_FLAG) { protocol = MAP; } # endif # ifdef _WITH_DFAX_SUPPORT else if (db.protocol & DFAX_FLAG) { protocol = DFAX; } # endif else if (db.protocol & SMTP_FLAG) { protocol = SMTP; } # ifdef _WITH_DE_MAIL_SUPPORT else if (db.protocol & DE_MAIL_FLAG) { protocol = DE_MAIL; } # endif else { system_log(DEBUG_SIGN, __FILE__, __LINE__, "Unknown protocol flag %d, setting FTP. #%x", db.protocol, db.id.job); protocol = FTP; } if (fsa->real_hostname[1][0] == '\0') { current_toggle = HOST_ONE; } else { if (db.toggle_host == YES) { if (fsa->host_toggle == HOST_ONE) { current_toggle = HOST_TWO; } else { current_toggle = HOST_ONE; } } else { current_toggle = (int)fsa->host_toggle; } } output_log_ptrs(&ol_retries, &ol_job_number, &ol_data, /* Pointer to buffer. */ &ol_file_name, &ol_file_name_length, &ol_archive_name_length, &ol_file_size, &ol_unl, &ol_size, &ol_transfer_time, &ol_output_type, db.host_alias, (current_toggle - 1), protocol, &db.output_log); } (void)memcpy(ol_file_name, db.p_unique_name, db.unl); (void)strcpy(ol_file_name + db.unl, d_name); *ol_file_name_length = (unsigned short)strlen(ol_file_name); ol_file_name[*ol_file_name_length] = SEPARATOR_CHAR; ol_file_name[*ol_file_name_length + 1] = '\0'; (*ol_file_name_length)++; # ifdef HAVE_STATX *ol_file_size = stat_buf->stx_size; # else *ol_file_size = stat_buf->st_size; # endif *ol_job_number = db.id.job; *ol_retries = db.retries; *ol_unl = db.unl; *ol_transfer_time = 0L; *ol_archive_name_length = 0; *ol_output_type = output_type; ol_real_size = *ol_file_name_length + ol_size; if (write(ol_fd, ol_data, ol_real_size) != ol_real_size) { system_log(ERROR_SIGN, __FILE__, __LINE__, "write() error : %s #%x", strerror(errno), db.id.job); } } } # endif # ifdef _DELETE_LOG if (dl.fd == -1) { delete_log_ptrs(&dl); } (void)strcpy(dl.file_name, d_name); (void)snprintf(dl.host_name, MAX_HOSTNAME_LENGTH + 4 + 1, "%-*s %03x", MAX_HOSTNAME_LENGTH, fsa->host_alias, # ifdef WITH_DUP_CHECK (is_duplicate == YES) ? DUP_OUTPUT : AGE_OUTPUT); # else AGE_OUTPUT); # endif # ifdef HAVE_STATX *dl.file_size = stat_buf->stx_size; # else *dl.file_size = stat_buf->st_size; # endif *dl.job_id = db.id.job; *dl.dir_id = 0; *dl.input_time = db.creation_time; *dl.split_job_counter = db.split_job_counter; *dl.unique_number = db.unique_number; *dl.file_name_length = strlen(d_name); # ifdef WITH_DUP_CHECK if (is_duplicate == YES) { str_diff_time[0] = '\0'; } else { time_t diff_time; # ifdef HAVE_STATX if (now < stat_buf->stx_mtime.tv_sec) # else if (now < stat_buf->st_mtime) # endif { diff_time = 0L; } else { # ifdef HAVE_STATX diff_time = now - stat_buf->stx_mtime.tv_sec; # else diff_time = now - stat_buf->st_mtime; # endif } # endif (void)snprintf(str_diff_time, 2 + MAX_LONG_LENGTH + 6 + MAX_LONG_LENGTH + 12 + MAX_LONG_LENGTH + 30 + 1, # if SIZEOF_TIME_T == 4 "%c>%ld [now=%ld file_mtime=%ld] (%s %d)", # else "%c>%lld [now=%lld file_mtime=%lld] (%s %d)", # endif SEPARATOR_CHAR, (pri_time_t)diff_time, (pri_time_t)now, # ifdef HAVE_STATX (pri_time_t)stat_buf->stx_mtime.tv_sec, # else (pri_time_t)stat_buf->st_mtime, # endif __FILE__, __LINE__); # ifdef WITH_DUP_CHECK } # endif if (db.protocol & FTP_FLAG) { if (db.no_of_restart_files > 0) { int append_file_number = -1, ii; for (ii = 0; ii < db.no_of_restart_files; ii++) { if (CHECK_STRCMP(db.restart_file[ii], d_name) == 0) { append_file_number = ii; break; } } if (append_file_number != -1) { remove_append(db.id.job, db.restart_file[append_file_number]); } } prog_name_length = snprintf((dl.file_name + *dl.file_name_length + 1), MAX_FILENAME_LENGTH + 1, "%s%s", SEND_FILE_FTP, str_diff_time); } else if (db.protocol & LOC_FLAG) { prog_name_length = snprintf((dl.file_name + *dl.file_name_length + 1), MAX_FILENAME_LENGTH + 1, "%s%s", SEND_FILE_LOC, str_diff_time); } else if (db.protocol & EXEC_FLAG) { prog_name_length = snprintf((dl.file_name + *dl.file_name_length + 1), MAX_FILENAME_LENGTH + 1, "%s%s", SEND_FILE_EXEC, str_diff_time); } else if (db.protocol & HTTP_FLAG) { prog_name_length = snprintf((dl.file_name + *dl.file_name_length + 1), MAX_FILENAME_LENGTH + 1, "%s%s", SEND_FILE_HTTP, str_diff_time); } else if (db.protocol & SFTP_FLAG) { if (db.no_of_restart_files > 0) { int append_file_number = -1, ii; for (ii = 0; ii < db.no_of_restart_files; ii++) { if (CHECK_STRCMP(db.restart_file[ii], d_name) == 0) { append_file_number = ii; break; } } if (append_file_number != -1) { remove_append(db.id.job, db.restart_file[append_file_number]); } } prog_name_length = snprintf((dl.file_name + *dl.file_name_length + 1), MAX_FILENAME_LENGTH + 1, "%s%s", SEND_FILE_SFTP, str_diff_time); } # ifdef _WITH_SCP_SUPPORT else if (db.protocol & SCP_FLAG) { prog_name_length = snprintf((dl.file_name + *dl.file_name_length + 1), MAX_FILENAME_LENGTH + 1, "%s%s", SEND_FILE_SCP, str_diff_time); } # endif # ifdef _WITH_WMO_SUPPORT else if (db.protocol & WMO_FLAG) { prog_name_length = snprintf((dl.file_name + *dl.file_name_length + 1), MAX_FILENAME_LENGTH + 1, "%s%s", SEND_FILE_WMO, str_diff_time); } # endif # ifdef _WITH_MAP_SUPPORT else if (db.protocol & MAP_FLAG) { prog_name_length = snprintf((dl.file_name + *dl.file_name_length + 1), MAX_FILENAME_LENGTH + 1, "%s%s", SEND_FILE_MAP, str_diff_time); } # endif # ifdef _WITH_DFAX_SUPPORT else if (db.protocol & DFAX_FLAG) { prog_name_length = snprintf((dl.file_name + *dl.file_name_length + 1), MAX_FILENAME_LENGTH + 1, "%s%s", SEND_FILE_DFAX, str_diff_time); } # endif # ifdef _WITH_DE_MAIL_SUPPORT else if ((db.protocol & SMTP_FLAG) || (db.protocol & DE_MAIL_FLAG)) # else else if (db.protocol & SMTP_FLAG) # endif { prog_name_length = snprintf((dl.file_name + *dl.file_name_length + 1), MAX_FILENAME_LENGTH + 1, "%s%s", SEND_FILE_SMTP, str_diff_time); } else { prog_name_length = snprintf((dl.file_name + *dl.file_name_length + 1), MAX_FILENAME_LENGTH + 1, "sf_???%s", str_diff_time); } dl_real_size = *dl.file_name_length + dl.size + prog_name_length; if (write(dl.fd, dl.data, dl_real_size) != dl_real_size) { system_log(ERROR_SIGN, __FILE__, __LINE__, "write() error : %s #%x", strerror(errno), db.id.job); } # endif return; } #endif