--- ../../netmist/netmist.c 2017-11-29 10:28:00.982368118 -0600 +++ netmist/netmist.c 2018-05-22 15:37:07.073845895 -0500 @@ -126,7 +126,7 @@ #else #define MALLOC(x) my_malloc(x) #endif -#define THISVERSION " $Revision: 1699 $" +#define THISVERSION " $Revision: 1699dns $" char version[]=THISVERSION; #if defined(Windows) || defined(_linux_) || defined (_bsd_)||defined(WIN32) || defined(_solaris_) || defined(_macos_) @@ -451,7 +451,6 @@ void hist_insert(double); void child_dump_hist(FILE *); void dump_hist(void); - int validate_all_tokens(int); int parse_all_tokens(char * ); int parse_token(char *, char *, char *); @@ -518,6 +517,7 @@ int in_validate = 0; int init_heartbeats = 0; int licensed = 0; +unsigned int prime_addr_buf[16]; long long file_accesses; long long fd_accesses; long long fd_cache_misses; @@ -716,6 +716,8 @@ char pit_service[8]; char time_string[30]; int flush_flag = 1; +int pn_option = 0; +char *subarg; int bad_clock = 0; long long avg_start_comm_go_latency = 0LL; long long time_skew; @@ -899,6 +901,7 @@ void child_wait(void); void tell_master_ready(int); void print_results(void); +void print_results_in_csv(void); void tell_master_pct_complete(int,int,int); void tell_master_heartbeat(int,int,double); void tell_master_phase(int,int); @@ -1096,6 +1099,7 @@ }; void master_dump_hist(struct result_object *, FILE *); +void master_dump_hist_in_csv(struct result_object *, FILE *); /* * The client object contains all of the properties and attributes of * a given client worker thread/proc. @@ -1603,9 +1607,11 @@ long long fsize_per_client; long long total_set_size; int picked; + int m_retry_count = 0; #if defined(RLIMIT_NPROC) struct rlimit rlimit_struct; #endif + char tmp_buf[MAXHNAME]; int cret; /* @@ -1682,7 +1688,7 @@ strcpy(pit_service,"PIT"); /* Default */ - while((cret = getopt(argc,argv,"zNKYXFGDLvkimhux65432:7:c:o:O:Q:T:H:l:S:n:r:s:p:e:g:f:j:t:M:R:U:b:B:d:P:w:W:EI:a:y:J:V:A:Z:q: ")) != EOF){ + while((cret = getopt(argc,argv,"zNKYXFGDLvkimhux65432:7:c:o:O:Q:T:H:l:S:n:r:s:p:e:g:f:j:t:M:R:U:b:B:d:P:w:W:EI:a:y:J:V:A:Z:q:+: ")) != EOF){ switch(cret){ case 'F': /* Disable flushes */ flush_flag = 0; @@ -2009,6 +2015,18 @@ case 'S': /* Argument is port of the PIT */ strcpy(pit_service,optarg); break; + case '+': + /* printf("Plus option = >%s<\n",optarg);*/ + switch (*((char *)optarg)){ + + case 'h': /* Prime name encode option */ + subarg=argv[optind++]; + if(subarg!=(char *)0) + pn_option=atoi(subarg); + break; + } + break; + } } @@ -2394,6 +2412,23 @@ strcpy(master_name,tmp_buf); */ + /* BUT NOW, we have to try to make this work because our lab's don't have DNS caching enabled + * and *that* causes the DNS servers to get hammered and start failing under high loads. + */ + + if(pn_option == PN_IP_OPTION) + { + memset(tmp_buf,0,MAXHNAME); +m_retry: + mret=get_host_ip_address(master_name, tmp_buf); + if(mret != 0) + { + if(m_retry_count++ < 5) + goto m_retry; + } + strcpy(master_name,tmp_buf); + } + /* * Calculate the current aggregate data set size. At this point * it should be large enough to use all client memory but may @@ -2982,6 +3017,7 @@ master_stop_comm(num_clients); print_results(); + print_results_in_csv(); #if defined(WIN32) /* stop network */ win32_close(); @@ -3394,6 +3430,492 @@ } } +/** + * @brief Gather up all of the results associated with each + * client proc and generate comma seperated CSV file. +*/ +/* + * __doc__ + * __doc__ Function : void print_results_in_csv(void) + * __doc__ Arguments: void + * __doc__ Performs : Gather up all of the results associated with each + * __doc__ client proc and generate comma seperated CSV file. + * __doc__ + */ +void +print_results_in_csv (void) +{ + struct result_object *here; + int i; + char ch_filename[400]; + FILE *ch_out; + long long timestamp = (long long)time(NULL); + +#if defined(WIN32) + sprintf (ch_filename, "%s\\%s.csv", master_results_dir, + "Clients"); +#else + sprintf (ch_filename, "%s/%s.csv", master_results_dir, + "Clients"); +#endif + + ch_out = fopen (ch_filename, "a"); + if (ch_out < (FILE *) 0) + { + printf("Error opening .csv file: %s\n", + strerror (errno)); + exit (1); + } + + fprintf (ch_out, "Timestamp, Name,Id,workdir,Workload,Runtime,Ops/sec,Avg latency (ms)," + "Total file ops,Read throughput (KiBytes/s),Read KiBytes," + "Write throughput (KiBytes/s),Write KiBytes," + "Native_read throughput (KiBytes/s),Native read KiBytes," + "Native_write throughput (KiBytes/s),Native_write KiBytes," + "Meta_write throughput (KiBytes/s),Meta_write KiBytes," + "Meta_read throughput (KiBytes/s),Meta_read KiBytes," + "Initialized dirs,Initialized files," + "Initialized files with space,Per proc file space," + "Min Direct I/O size," + "Write ops,Write latency (s)," + "Write_file ops,Write_file latency (s)," + "Mmap_write ops,Mmap_write latency (s)," + "Mmap_read ops,Mmap_read latency (s)," + "Read ops,Read latency (s)," + "Read_file ops,Read_file latency (s)," + "Mkdir ops,Mkdir latency (s)," + "Rmdir ops,Rmdir latency (s)," + "Unlink ops,Unlink latency (s)," + "Unlink2 ops,Unlink2 latency (s)," + "Create ops,Create latency (s)," + "Stat ops,Stat latency (s)," + "Append ops,Append latency (s)," + "Lock ops,Lock latency (s)," + "Access ops,Access latency (s)," + "Chmod ops,Chmod latency (s)," + "Readdir ops,Readdir latency (s)," + "Random_write ops,Random_write latency (s)," + "Random_read ops,Random_read latency (s)," + "Read_modify_write ops,Read_modify_write latency (s)," + "Open file ops,Open file latency (s)," + "Close file ops,Close file latency (s)," + "Copyfile ops,Copyfile latency (s)," + "Rename ops,Rename latency (s)," + "Statfs ops,Statfs latency (s)," + "Pathconf ops,Pathconf latency (s)," + "Custom1 ops,Custom1 latency (s)," + "Custom2 ops,Custom2 latency (s)," + "Band1 20us,Band1 40us,Band1 60us,Band1 80us,Band1 100us," + "Band2 200us,Band2 400us,Band2 600us,Band2 800us,Band2 1ms," + "Band3 2ms,Band3 4ms,Band3 6ms,Band3 8ms,Band1 10ms," + "Band4 12ms,Band4 14ms,Band4 16ms,Band4 18ms,Band4 20ms," + "Band5 40ms,Band5 60ms,Band5 80ms,Band5 100ms," + "Band6 200ms,Band6 400ms,Band6 600ms,Band6 800ms," + "Band7 2s,Band7 4s,Band7 6s,Band7 8s,Band7 10s," + "Band8 20s,Band8 40s,Band8 60s,Band8 80s,Band8 120s," + "Band 9 120+s" + "\n"); + + for (i = 0; i < num_clients; i++) + { + here = &client_obj[i].results; + + fprintf (ch_out, "%lld,", timestamp); + fprintf (ch_out, "%s,%d,", here->client_name, here->client_id); + fprintf (ch_out, "%s,%s,", client_obj[i].client_workdir, + here->work_obj_name); + fprintf (ch_out, "%ld,", here->run_time); + fprintf (ch_out, "%.2f,", here->ops_per_second); + fprintf (ch_out, "%.3f,",here->average_latency); + + fprintf (ch_out, "%f,", here->total_file_ops); + fprintf (ch_out,"%.3f,", here->read_throughput / 1024); + fprintf (ch_out, "%.3f,", here->read_kbytes / 1024); + + fprintf (ch_out,"%.3f,", here->write_throughput / 1024); + fprintf (ch_out, "%.3f,", here->write_kbytes / 1024); + + fprintf (ch_out,"%.3f,", here->Nread_throughput / 1024); + fprintf (ch_out, "%.3f,", here->Nread_kbytes / 1024); + + fprintf (ch_out, "%.3f,", here->Nwrite_throughput / 1024); + fprintf (ch_out, " %.3f,", here->Nwrite_kbytes / 1024); + + fprintf (ch_out,"%.3f,",(here->meta_w_kbytes / 1024) / here->run_time); + fprintf (ch_out, "%.3f,", here->meta_w_kbytes / 1024); + + fprintf (ch_out, "%.3f,", (here->meta_r_kbytes / 1024) / here->run_time); + fprintf (ch_out, "%.3f,", here->meta_r_kbytes / 1024); + + fprintf (ch_out, "%d,", here->init_dirs); + fprintf (ch_out, "%d,", here->init_files); + + fprintf (ch_out, "%d,", here->init_files_ws); + fprintf (ch_out, " ~%.0f,", here->file_space_mb); + + fprintf (ch_out, "%ld,", here->min_direct_size); + + /* + * Here are the outputs for all of the standard workloads. + */ + + /* Write */ + fprintf (ch_out, "%ld,", here->write_count); + if (Op_lat_flag && (here->write_count != 0) + && (here->write_time != 0.0)) + { + fprintf (ch_out, "%.6f,", + here->write_time / (double) here->write_count); + } + else + { + fprintf (ch_out, " ,"); + } + + /* Write file */ + fprintf (ch_out, "%ld,", here->write_file_count); + if (Op_lat_flag && (here->write_file_count != 0) + && (here->write_file_time != 0.0)) + { + fprintf (ch_out, "%.6f,", + here->write_file_time / (double) here->write_file_count); + } + else + { + fprintf (ch_out, " ,"); + } + + /* mmap write */ + fprintf (ch_out, "%ld,", here->mmap_write_count); + if (Op_lat_flag && (here->mmap_write_count != 0) + && (here->mmap_write_time != 0.0)) + { + fprintf (ch_out, "%.6f,", + here->mmap_write_time / (double) here->mmap_write_count); + } + else + { + fprintf (ch_out, " ,"); + } + + /* mmap read */ + fprintf (ch_out, "%ld,", here->mmap_read_count); + if (Op_lat_flag && (here->mmap_read_count != 0) + && (here->mmap_read_time != 0.0)) + { + fprintf (ch_out, "%.6f,", + here->mmap_read_time / (double) here->mmap_read_count); + } + else + { + fprintf (ch_out, " ,"); + } + + /* read */ + fprintf (ch_out, "%ld,", here->read_count); + if (Op_lat_flag && (here->read_count != 0) + && (here->read_time != 0.0)) + { + fprintf (ch_out, "%.6f,", + here->read_time / (double) here->read_count); + } + else + { + fprintf (ch_out, " ,"); + } + + /* read file */ + fprintf (ch_out, "%ld,", here->read_file_count); + if (Op_lat_flag && (here->read_file_count != 0) + && (here->read_file_time != 0.0)) + { + fprintf (ch_out, "%.6f,", + here->read_file_time / (double) here->read_file_count); + } + else + { + fprintf (ch_out, " ,"); + } + + /* mkdir */ + fprintf (ch_out, "%ld,", here->mkdir_count); + if (Op_lat_flag && (here->mkdir_count != 0) + && (here->mkdir_time != 0.0)) + { + fprintf (ch_out, "%.6f,", + here->mkdir_time / (double) here->mkdir_count); + } + else + { + fprintf (ch_out, " ,"); + } + + /* rmdir */ + fprintf (ch_out, "%ld,", here->rmdir_count); + if (Op_lat_flag && (here->rmdir_count != 0) + && (here->rmdir_time != 0.0)) + { + fprintf (ch_out, "%.6f,", + here->rmdir_time / (double) here->rmdir_count); + } + else + { + fprintf (ch_out, " ,"); + } + + /* unlink */ + fprintf (ch_out, "%ld,", here->unlink_count); + if (Op_lat_flag && (here->unlink_count != 0) + && (here->unlink_time != 0.0)) + { + fprintf (ch_out, "%6f,", + here->unlink_time / (double) here->unlink_count); + } + else + { + fprintf (ch_out, " ,"); + } + + /* unlink2 */ + fprintf (ch_out, "%ld,", here->unlink2_count); + if (Op_lat_flag && (here->unlink2_count != 0) + && (here->unlink2_time != 0.0)) + { + fprintf (ch_out, "%.6f,", + here->unlink2_time / (double) here->unlink2_count); + } + else + { + fprintf (ch_out, " ,"); + } + + /* create */ + fprintf (ch_out, "%ld,", here->create_count); + if (Op_lat_flag && (here->create_count != 0) + && (here->create_time != 0.0)) + { + fprintf (ch_out, "%.6f,", + here->create_time / (double) here->create_count); + } + else + { + fprintf (ch_out, " ,"); + } + + /* stat */ + fprintf (ch_out, "%ld,", here->stat_count); + if (Op_lat_flag && (here->stat_count != 0) + && (here->stat_time != 0.0)) + { + fprintf (ch_out, "%.6f,", + here->stat_time / (double) here->stat_count); + } + else + { + fprintf (ch_out, " ,"); + } + + /* append */ + fprintf (ch_out, "%ld,", here->append_count); + if (Op_lat_flag && (here->append_count != 0) + && (here->append_time != 0.0)) + { + fprintf (ch_out, "%.6f,", + here->append_time / (double) here->append_count); + } + else + { + fprintf (ch_out, " ,"); + } + + /* Lock */ + fprintf (ch_out, "%ld,", here->lock_count); + if (Op_lat_flag && (here->lock_count != 0) + && (here->lock_time != 0.0)) + { + fprintf (ch_out, "%.6f,", + here->lock_time / (double) here->lock_count); + } + else + { + fprintf (ch_out, " ,"); + } + + /* append */ + fprintf (ch_out, "%ld,", here->access_count); + if (Op_lat_flag && (here->access_count != 0) + && (here->access_time != 0.0)) + { + fprintf (ch_out, "%.6f,", + here->access_time / (double) here->access_count); + } + else + { + fprintf (ch_out, " ,"); + } + + /* chmod */ + fprintf (ch_out, "%ld,", here->chmod_count); + if (Op_lat_flag && (here->chmod_count != 0) + && (here->chmod_time != 0.0)) + { + fprintf (ch_out, "%.6f,", + here->chmod_time / (double) here->chmod_count); + } + else + { + fprintf (ch_out, " ,"); + } + + /* readdir */ + fprintf (ch_out, "%ld,", here->readdir_count); + if (Op_lat_flag && (here->readdir_count != 0) + && (here->readdir_time != 0.0)) + { + fprintf (ch_out, "%.6f,", + here->readdir_time / (double) here->readdir_count); + } + else + { + fprintf (ch_out, " ,"); + } + + /* write rand */ + fprintf (ch_out, "%ld,", here->write_rand_count); + if (Op_lat_flag && (here->write_rand_count != 0) + && (here->write_rand_time != 0.0)) + { + fprintf (ch_out, "%.6f,", + here->write_rand_time / (double) here->write_rand_count); + } + else + { + fprintf (ch_out, " ,"); + } + + /* Read rand */ + fprintf (ch_out, "%ld,", here->read_rand_count); + if (Op_lat_flag && (here->read_rand_count != 0) + && (here->read_rand_time != 0.0)) + { + fprintf (ch_out, "%.6f,", + here->read_rand_time / (double) here->read_rand_count); + } + else + { + fprintf (ch_out, " ,"); + } + + /* rmw */ + fprintf (ch_out, "%ld,", here->rmw_count); + if (Op_lat_flag && (here->rmw_count != 0) + && (here->rmw_time != 0.0)) + { + fprintf (ch_out, "%.6f,", + here->rmw_time / (double) here->rmw_count); + } + else + { + fprintf (ch_out, " ,"); + } + + fprintf (ch_out, "%ld,", here->open_count); + fprintf (ch_out, " ,"); + + fprintf (ch_out, "%ld,", here->close_count); + fprintf (ch_out, " ,"); + + /* Copyfile */ + fprintf (ch_out, "%ld,", here->copyfile_count); + if (Op_lat_flag && (here->copyfile_count != 0) + && (here->copyfile_time != 0.0)) + { + fprintf (ch_out, "%.6f,", + here->copyfile_time / (double) here->copyfile_count); + } + else + { + fprintf (ch_out, " ,"); + } + + /* Rename */ + fprintf (ch_out, "%ld,", here->rename_count); + if (Op_lat_flag && (here->rename_count != 0) + && (here->rename_time != 0.0)) + { + fprintf (ch_out, "%.6f,", + here->rename_time / (double) here->rename_count); + } + else + { + fprintf (ch_out, " ,"); + } + + /* Statfs */ + fprintf (ch_out, "%ld,", here->statfs_count); + if (Op_lat_flag && (here->statfs_count != 0) + && (here->statfs_time != 0.0)) + { + fprintf (ch_out, "%.6f,", + here->statfs_time / (double) here->statfs_count); + } + else + { + fprintf (ch_out, " ,"); + } + + /* Pathconf */ + fprintf (ch_out, "%ld,", here->pathconf_count); + if (Op_lat_flag && (here->pathconf_count != 0) + && (here->pathconf_time != 0.0)) + { + fprintf (ch_out, "%.6f,", + here->pathconf_time / (double) here->pathconf_count); + } + else + { + fprintf (ch_out, " ,"); + } + + /* custom1 */ + fprintf (ch_out, "%ld,", here->custom1_count); + if (Op_lat_flag && (here->custom1_count != 0) + && (here->custom1_time != 0.0)) + { + fprintf (ch_out, "%.6f,", + here->custom1_time / (double) here->custom1_count); + } + else + { + fprintf (ch_out, " ,"); + } + + /* custom2 */ + fprintf (ch_out, "%ld,", here->custom2_count); + if (Op_lat_flag && (here->custom2_count != 0) + && (here->custom2_time != 0.0)) + { + fprintf (ch_out, "%.6f,", + here->custom2_time / (double) here->custom2_count); + } + else + { + fprintf (ch_out, " ,"); + } + + if (Op_lat_flag) + { + master_dump_hist_in_csv (here, ch_out); + } + + fprintf (ch_out, "\n"); + fflush (ch_out); + } + + fprintf (ch_out, "\n\n\n\n\n"); + fclose (ch_out); +} + /* * This dumps the internal workloads to an ascii file, for editing * and later import. @@ -3570,17 +4092,17 @@ else strcpy(remsh, UNIX_REMOTE_FROM_WINDOWS); strcpy(remote_command,remsh); - sprintf(command,"%s %s -l %s %s -A %s -Z %s -Q %d -T %d -m -d %d -O %.2f -s %lld -n %d -t %d -p %s -e %s -M %s -R %s -U %s -b %lld -B %lld %s -P %d -w %d -W %s -y %d -r %d %s", + sprintf(command,"%s %s -l %s %s -A %s -Z %s -Q %d -T %d -m -d %d -O %.2f -s %lld -n %d -t %d -p %s -e %s -M %s -R %s -U %s -b %lld -B %lld %s -P %d -w %d -W %s -y %d -r %d -+h %d %s", remote_command,client_obj[i].client_name, client_obj[i].client_dom_user, client_obj[i].client_execdir, client_obj[i].client_password, client_obj[i].client_dom_user, client_files, client_dirs, ppc, my_oprate, fsize, client_obj[i].client_id, run_time, client_obj[i].client_workdir,client_obj[i].client_execdir,master_name, master_results_dir,client_log_dir,client_memsize_mb, agg_set_memsize_mb, var_opt,master_listen_port,warm_time, - client_obj[i].client_workload_name,ipv6_enable,setfsize,option_buf); + client_obj[i].client_workload_name,ipv6_enable,setfsize,pn_option,option_buf); } else { strcpy(remote_command,WINDOWS_REMOTE_FROM_WINDOWS); /* Destination is Windows */ - sprintf(command,"%s /node:\"%s\" /user:\"%s\" /password:\"%s\" process call create \"%s -A %s -Z %s -Q %d -T %d -m -d %d -O %.2f -s %lld -n %d -t %d -p %s -e %s -M %s -R %s -U %s -b %lld -B %lld %s -P %d -w %d -W %s -y %d -r %d %s\"", + sprintf(command,"%s /node:\"%s\" /user:\"%s\" /password:\"%s\" process call create \"%s -A %s -Z %s -Q %d -T %d -m -d %d -O %.2f -s %lld -n %d -t %d -p %s -e %s -M %s -R %s -U %s -b %lld -B %lld %s -P %d -w %d -W %s -y %d -r %d -+h %d %s\"", remote_command,client_obj[i].client_name, client_obj[i].client_dom_user, client_obj[i].client_password, client_obj[i].client_execdir, client_obj[i].client_password, client_obj[i].client_dom_user, client_files, client_dirs, ppc, my_oprate, fsize, client_obj[i].client_id, run_time, client_obj[i].client_workdir, client_obj[i].client_execdir,master_name, @@ -3596,48 +4118,40 @@ else strcpy(remsh, UNIX_REMOTE_FROM_UNIX); strcpy(remote_command,remsh); - sprintf(command,"%s %s -l %s %s -A %s -Z %s -Q %d -T %d -m -d %d -O %.2f -s %lld -n %d -t %d -p %s -e %s -M %s -R %s -U %s -b %lld -B %lld %s -P %d -w %d -W %s -y %d -r %d %s", + sprintf(command,"%s %s -l %s %s -A %s -Z %s -Q %d -T %d -m -d %d -O %.2f -s %lld -n %d -t %d -p %s -e %s -M %s -R %s -U %s -b %lld -B %lld %s -P %d -w %d -W %s -y %d -r %d -+h %d %s", remote_command,client_obj[i].client_name, client_obj[i].client_dom_user, client_obj[i].client_execdir, client_obj[i].client_password, client_obj[i].client_dom_user, client_files, client_dirs, ppc, my_oprate, fsize, client_obj[i].client_id, run_time, client_obj[i].client_workdir,client_obj[i].client_execdir,master_name, master_results_dir,client_log_dir,client_memsize_mb, agg_set_memsize_mb, var_opt,master_listen_port,warm_time, - client_obj[i].client_workload_name,ipv6_enable,setfsize,option_buf); + client_obj[i].client_workload_name,ipv6_enable,setfsize,pn_option,option_buf); } else { strcpy(remote_command,WINDOWS_REMOTE_FROM_UNIX); /* Destination is Windows box. Assumes "winexe" format. */ - sprintf(command,"%s -U %s%s%s //%s \'%s -A %s -Z %s -Q %d -T %d -m -d %d -O %.2f -s %lld -n %d -t %d -p %s -e %s -M %s -R %s -U %s -b %lld -B %lld %s -P %d -w %d -W %s -y %d -r %d %s\' &", + sprintf(command,"%s -U %s%s%s //%s \'cmd /C start /b %s -A %s -Z %s -Q %d -T %d -m -d %d -O %.2f -s %lld -n %d -t %d -p %s -e %s -M %s -R %s -U %s -b %lld -B %lld %s -P %d -w %d -W %s -y %d -r %d -+h %d %s\' ", remote_command, client_obj[i].client_dom_user,"%",client_obj[i].client_password,client_obj[i].client_name, client_obj[i].client_execdir, client_obj[i].client_password,client_obj[i].client_dom_user, client_files, client_dirs, ppc, my_oprate, fsize, client_obj[i].client_id, run_time, client_obj[i].client_workdir, client_obj[i].client_execdir,master_name, master_results_dir,client_log_dir,client_memsize_mb, agg_set_memsize_mb, var_opt,master_listen_port,warm_time, - client_obj[i].client_workload_name,ipv6_enable,setfsize,option_buf); -#ifdef HHH - strcpy(remote_command,WINDOWS_REMOTE_FROM_UNIX); /* Destination is Windows box. Assumes "winexe" format. */ - sprintf(command,"%s -U %s --password=%s //%s \'%s -A %s -Z %s -Q %d -T %d -m -d %d -O %.2f -s %lld -n %d -t %d -p %s -e %s -M %s -R %s -U %s -b %lld -B %lld %s -P %d -w %d -W %s -y %d -r %d %s\' &", - remote_command, client_obj[i].client_dom_user,client_obj[i].client_password, client_obj[i].client_name, client_obj[i].client_execdir, - client_obj[i].client_password,client_obj[i].client_dom_user, client_files, client_dirs, ppc, my_oprate, - fsize, client_obj[i].client_id, run_time, client_obj[i].client_workdir, client_obj[i].client_execdir,master_name, - master_results_dir,client_log_dir,client_memsize_mb, agg_set_memsize_mb, var_opt,master_listen_port,warm_time, - client_obj[i].client_workload_name,ipv6_enable,setfsize,option_buf); -#endif + client_obj[i].client_workload_name,ipv6_enable,setfsize,pn_option,option_buf); } #endif } else + /* The bloody LOCAL path. No remote shells */ { #if defined(WIN32) - sprintf(command,"start %s -A %s -Z %s -Q %d -T %d -m -d %d -O %.2f -s %lld -n %d -t %d -p %s -e %s -M %s -R %s -U %s -b %lld -B %lld %s -P %d -w %d -W %s -z -y %d -r %d %s", + sprintf(command,"start %s -A %s -Z %s -Q %d -T %d -m -d %d -O %.2f -s %lld -n %d -t %d -p %s -e %s -M %s -R %s -U %s -b %lld -B %lld %s -P %d -w %d -W %s -z -y %d -r %d -+h %d %s", client_obj[i].client_execdir, client_obj[i].client_password, client_obj[i].client_dom_user, client_files, client_dirs, ppc, my_oprate, fsize, client_obj[i].client_id, run_time, client_obj[i].client_workdir, client_obj[i].client_execdir,master_name, master_results_dir,client_log_dir,client_memsize_mb, - agg_set_memsize_mb, var_opt,master_listen_port,warm_time,client_obj[i].client_workload_name,ipv6_enable,setfsize,option_buf); + agg_set_memsize_mb, var_opt,master_listen_port,warm_time,client_obj[i].client_workload_name,ipv6_enable,setfsize,pn_option,option_buf); #else - sprintf(command,"%s -A %s -Z %s -Q %d -T %d -m -d %d -O %.2f -s %lld -n %d -t %d -p %s -e %s -M %s -R %s -U %s -b %lld -B %lld %s -P %d -w %d -W %s -z -y %d -r %d %s", + sprintf(command,"%s -A %s -Z %s -Q %d -T %d -m -d %d -O %.2f -s %lld -n %d -t %d -p %s -e %s -M %s -R %s -U %s -b %lld -B %lld %s -P %d -w %d -W %s -z -y %d -r %d -+h %d %s", client_obj[i].client_execdir, client_obj[i].client_password, client_obj[i].client_dom_user, client_files, client_dirs, ppc, my_oprate, fsize, client_obj[i].client_id, run_time, client_obj[i].client_workdir, client_obj[i].client_execdir,master_name, master_results_dir,client_log_dir,client_memsize_mb, - agg_set_memsize_mb, var_opt,master_listen_port,warm_time,client_obj[i].client_workload_name,ipv6_enable,setfsize, option_buf); + agg_set_memsize_mb, var_opt,master_listen_port,warm_time,client_obj[i].client_workload_name,ipv6_enable,setfsize, pn_option, option_buf); #endif } if(cdebug) @@ -3693,20 +4207,20 @@ else strcpy(remsh, UNIX_REMOTE_FROM_WINDOWS); strcpy(remote_command,remsh); - sprintf(command,"%s %s -l %s %s -A %s -Z %s -k -m -s %lld -n %d -t %d -p %s -e %s -M %s -R %s -U %s -W %s -y %d -V %d -r %d %s", + sprintf(command,"%s %s -l %s %s -A %s -Z %s -k -m -s %lld -n %d -t %d -p %s -e %s -M %s -R %s -U %s -W %s -y %d -V %d -r %d -+h %d %s", remote_command, client_obj[i].client_name, client_obj[i].client_dom_user, client_obj[i].client_execdir, client_obj[i].client_password, client_obj[i].client_dom_user, fsize, client_obj[i].client_id, run_time, client_obj[i].client_workdir, client_obj[i].client_execdir, master_name, master_results_dir, - client_log_dir,client_obj[i].client_workload_name,ipv6_enable,child_pid[i],setfsize,option_buf); + client_log_dir,client_obj[i].client_workload_name,ipv6_enable,child_pid[i],setfsize,pn_option, option_buf); } else { strcpy(remote_command,WINDOWS_REMOTE_FROM_WINDOWS); /* Destinatino is Windows */ - sprintf(command,"%s /node:\"%s\" /user:\"%s\" /password:\"%s\" process call create \"%s -A %s -Z %s -k -m -s %lld -n %d -t %d -p %s -e %s -M %s -R %s -U %s -W %s -y %d -V %d -r %d %s\"", + sprintf(command,"%s /node:\"%s\" /user:\"%s\" /password:\"%s\" process call create \"%s -A %s -Z %s -k -m -s %lld -n %d -t %d -p %s -e %s -M %s -R %s -U %s -W %s -y %d -V %d -r %d -+h %d %s\"", remote_command,client_obj[i].client_name, client_obj[i].client_dom_user, client_obj[i].client_password, client_obj[i].client_execdir, client_obj[i].client_password, client_obj[i].client_dom_user, fsize, client_obj[i].client_id, run_time, client_obj[i].client_workdir, client_obj[i].client_execdir, master_name, master_results_dir, client_log_dir, - client_obj[i].client_workload_name,ipv6_enable,child_pid[i],setfsize,option_buf); + client_obj[i].client_workload_name,ipv6_enable,child_pid[i],setfsize,pn_option, option_buf); } #else if(client_obj[i].client_type == TYPE_UNIX) /* Destination is UNIX */ @@ -3716,26 +4230,26 @@ else strcpy(remsh, UNIX_REMOTE_FROM_UNIX); strcpy(remote_command,remsh); - sprintf(command,"%s %s -l %s %s -A %s -Z %s -k -m -s %lld -n %d -t %d -p %s -e %s -M %s -R %s -U %s -W %s -y %d -V %d -r %d %s", + sprintf(command,"%s %s -l %s %s -A %s -Z %s -k -m -s %lld -n %d -t %d -p %s -e %s -M %s -R %s -U %s -W %s -y %d -V %d -r %d -+h %d %s", remote_command, client_obj[i].client_name, client_obj[i].client_dom_user, client_obj[i].client_execdir, client_obj[i].client_password, client_obj[i].client_dom_user, fsize, client_obj[i].client_id, run_time, client_obj[i].client_workdir, client_obj[i].client_execdir, master_name, master_results_dir, - client_log_dir,client_obj[i].client_workload_name,ipv6_enable,child_pid[i],setfsize,option_buf); + client_log_dir,client_obj[i].client_workload_name,ipv6_enable,child_pid[i],setfsize,pn_option,option_buf); } else { strcpy(remote_command,WINDOWS_REMOTE_FROM_UNIX); /* Destination is Windows box. Assumes "winexe" format. */ - sprintf(command,"%s -U %s%s%s //%s \'%s -A %s -Z %s -k -m -s %lld -n %d -t %d -p %s -e %s -M %s -R %s -U %s -W %s -y %d -V %d -r %d %s\'", + sprintf(command,"%s -U %s%s%s //%s \'%s -A %s -Z %s -k -m -s %lld -n %d -t %d -p %s -e %s -M %s -R %s -U %s -W %s -y %d -V %d -r %d -+h %d %s\'", remote_command, client_obj[i].client_dom_user,"%",client_obj[i].client_password,client_obj[i].client_name, client_obj[i].client_execdir, client_obj[i].client_password, client_obj[i].client_dom_user,fsize, client_obj[i].client_id, run_time, client_obj[i].client_workdir, client_obj[i].client_execdir, master_name, master_results_dir, client_log_dir, - client_obj[i].client_workload_name,ipv6_enable,child_pid[i],setfsize,option_buf); + client_obj[i].client_workload_name,ipv6_enable,child_pid[i],setfsize,pn_option, option_buf); #ifdef HHH - sprintf(command,"%s -U %s --password=%s //%s \'%s -A %s -Z %s -k -m -s %lld -n %d -t %d -p %s -e %s -M %s -R %s -U %s -W %s -y %d -V %d -r %d %s\'", + sprintf(command,"%s -U %s --password=%s //%s \'%s -A %s -Z %s -k -m -s %lld -n %d -t %d -p %s -e %s -M %s -R %s -U %s -W %s -y %d -V %d -r %d -+h %d %s\'", remote_command, client_obj[i].client_dom_user, client_obj[i].client_password, client_obj[i].client_name, client_obj[i].client_execdir, client_obj[i].client_password, client_obj[i].client_dom_user, fsize, client_obj[i].client_id, run_time, client_obj[i].client_workdir, client_obj[i].client_execdir, master_name, master_results_dir, client_log_dir, - client_obj[i].client_workload_name,ipv6_enable,child_pid[i],setfsize,option_buf); + client_obj[i].client_workload_name,ipv6_enable,child_pid[i],setfsize,pn_option,option_buf); #endif } @@ -3743,10 +4257,10 @@ } else { - sprintf(command,"%s -A %s -Z %s -k -m -s %lld -n %d -t %d -p %s -e %s -M %s -R %s -U %s -W %s -z -y %d -V %d -r %d %s", + sprintf(command,"%s -A %s -Z %s -k -m -s %lld -n %d -t %d -p %s -e %s -M %s -R %s -U %s -W %s -z -y %d -V %d -r %d -+h %d %s", client_obj[i].client_execdir, client_obj[i].client_password, client_obj[i].client_dom_user, fsize, client_obj[i].client_id, run_time, client_obj[i].client_workdir, client_obj[i].client_execdir,master_name, - master_results_dir, client_log_dir, client_obj[i].client_workload_name, ipv6_enable,child_pid[i],setfsize, option_buf); + master_results_dir, client_log_dir, client_obj[i].client_workload_name, ipv6_enable,child_pid[i],setfsize, pn_option, option_buf); } printf("Killing test on -> %s <- Client ID %5d PID %5d\n",client_obj[i].client_name,client_obj[i].client_id,child_pid[i]); fflush(stdout); @@ -19749,6 +20263,7 @@ Child_send(char *controlling_host_name, char *send_buffer) { int rc,xx,retry_send=0; + int retry_count = 0; /* struct hostent *he; */ int tmp_port; /*struct in_addr *ip;*/ @@ -19785,16 +20300,51 @@ } ip = (struct in_addr *)he->h_addr_list[0]; */ + + /* + * If the pn_option == 1 then the controlling_host_name contains + * an IP address as a string. + * So, we need to convert it to a network address format. + */ + if(pn_option == PN_IP_OPTION) + { + if (ipv6_enable) { + inet_pton(AF_INET6, (void*)controlling_host_name, addr_buf); + } + else + { + inet_pton(AF_INET, (void*)controlling_host_name, addr_buf); + } + goto overL; + } + + /* Now use that cached IP address for the prime. No need to hammer DNS */ + if(prime_addr_buf[0] != 0) + { + memcpy(addr_buf, prime_addr_buf,sizeof(addr_buf)); + goto overL; + } +retry_dns: xx=get_host_address(controlling_host_name, addr_buf, name_buf); if(xx != 0) { + if(retry_count++ < 5) + { + nap(10); + goto retry_dns; + } c_eflag = 1; c_eval = BAD_DNS; R_exit(BAD_DNS); } overL: + /* Cache the IP address of the prime. No need to hammer DNS */ + if(prime_addr_buf[0] == 0) + { + memcpy(prime_addr_buf,addr_buf,sizeof(addr_buf)); + } if(ipv6_enable) { cs_raddr6.sin6_family = AF_INET6; @@ -20455,6 +21005,7 @@ Master_send(char *child_host_name, int child_port, char *send_buffer) { int rc,master_socket_val; + int retry_count=0; /*struct hostent *he; */ int tmp_port; /*struct in_addr *ip; */ @@ -20490,9 +21041,15 @@ start_master_send = gettime(); memset(name_buf,0,MAXHNAME); +retry_dns: xx=get_host_address(child_host_name, addr_buf, name_buf); if(xx != 0) { + if(retry_count++ < 5) + { + nap(10); + goto retry_dns; + } printf("DNS problem. Unable to determine the IP address for the child: %s\n",child_host_name); fflush(stdout); exit(BAD_DNS); @@ -22374,6 +22931,129 @@ } } +/** + * @brief The prime dumping its histograms from all of the + * children into csv. + * + * @param here : Result object for results. + * @param out : Output file + */ +/* + * __doc__ + * __doc__ Function : void master_dump_hist_in_csv (struct result_object *here, + * __doc__ FILE *out) + * __doc__ Arguments: Result object for results. + * __doc__ FILE *: Output file + * __doc__ Returns : void + * __doc__ Performs : The prime dumping its histograms from all of the + * __doc__ children. + * __doc__ + */ +void +master_dump_hist_in_csv (struct result_object *here, FILE * out) +{ + +#ifndef NO_PRINT_LLD + fprintf (out, "%lld,", here->bands[0]); + fprintf (out, "%lld,", here->bands[1]); + fprintf (out, "%lld,", here->bands[2]); + fprintf (out, "%lld,", here->bands[3]); + fprintf (out, "%lld,", here->bands[4]); + + fprintf (out, "%lld,", here->bands[5]); + fprintf (out, "%lld,", here->bands[6]); + fprintf (out, "%lld,", here->bands[7]); + fprintf (out, "%lld,", here->bands[8]); + fprintf (out, "%lld,", here->bands[9]); + + fprintf (out, "%lld,", here->bands[10]); + fprintf (out, "%lld,", here->bands[11]); + fprintf (out, "%lld,", here->bands[12]); + fprintf (out, "%lld,", here->bands[13]); + fprintf (out, "%lld,", here->bands[14]); + + fprintf (out, "%lld,", here->bands[15]); + fprintf (out, "%lld,", here->bands[16]); + fprintf (out, "%lld,", here->bands[17]); + fprintf (out, "%lld,", here->bands[18]); + fprintf (out, "%lld,", here->bands[19]); + + fprintf (out, "%lld,", here->bands[20]); + fprintf (out, "%lld,", here->bands[21]); + fprintf (out, "%lld,", here->bands[22]); + fprintf (out, "%lld,", here->bands[23]); + + fprintf (out, "%lld,", here->bands[24]); + fprintf (out, "%lld,", here->bands[25]); + fprintf (out, "%lld,", here->bands[26]); + fprintf (out, "%lld,", here->bands[27]); + fprintf (out, "%lld,", here->bands[28]); + + fprintf (out, "%lld,", here->bands[29]); + fprintf (out, "%lld,", here->bands[30]); + fprintf (out, "%lld,", here->bands[31]); + fprintf (out, "%lld,", here->bands[32]); + fprintf (out, "%lld,", here->bands[33]); + + fprintf (out, "%lld,", here->bands[34]); + fprintf (out, "%lld,", here->bands[35]); + fprintf (out, "%lld,", here->bands[36]); + fprintf (out, "%lld,", here->bands[37]); + fprintf (out, "%lld", here->bands[38]); + + fprintf (out, "%lld", here->bands[39]); +#else + fprintf (out, "%ld,", here->bands[0]); + fprintf (out, "%ld,", here->bands[1]); + fprintf (out, "%ld,", here->bands[2]); + fprintf (out, "%ld,", here->bands[3]); + fprintf (out, "%ld,", here->bands[4]); + + fprintf (out, "%ld,", here->bands[5]); + fprintf (out, "%ld,", here->bands[6]); + fprintf (out, "%ld,", here->bands[7]); + fprintf (out, "%ld,", here->bands[8]); + fprintf (out, "%ld,", here->bands[9]); + + fprintf (out, "%ld,", here->bands[10]); + fprintf (out, "%ld,", here->bands[11]); + fprintf (out, "%ld,", here->bands[12]); + fprintf (out, "%ld,", here->bands[13]); + fprintf (out, "%ld,", here->bands[14]); + + fprintf (out, "%ld,", here->bands[15]); + fprintf (out, "%ld,", here->bands[16]); + fprintf (out, "%ld,", here->bands[17]); + fprintf (out, "%ld,", here->bands[18]); + fprintf (out, "%ld,", here->bands[19]); + + fprintf (out, "%ld,", here->bands[20]); + fprintf (out, "%ld,", here->bands[21]); + fprintf (out, "%ld,", here->bands[22]); + fprintf (out, "%ld,", here->bands[23]); + + fprintf (out, "%ld,", here->bands[24]); + fprintf (out, "%ld,", here->bands[25]); + fprintf (out, "%ld,", here->bands[26]); + fprintf (out, "%ld,", here->bands[27]); + fprintf (out, "%ld,", here->bands[28]); + + fprintf (out, "%ld,", here->bands[29]); + fprintf (out, "%ld,", here->bands[30]); + fprintf (out, "%ld,", here->bands[31]); + fprintf (out, "%ld,", here->bands[32]); + fprintf (out, "%ld,", here->bands[33]); + + fprintf (out, "%ld,", here->bands[34]); + fprintf (out, "%ld,", here->bands[35]); + fprintf (out, "%ld,", here->bands[36]); + fprintf (out, "%ld,", here->bands[37]); + fprintf (out, "%ld,", here->bands[38]); + + fprintf (out, "%ld", here->bands[39]); +#endif +} + #define DEDUP_CHUNK 512 /* * Function that generates data layout that implements the patterns needed @@ -23760,7 +24440,7 @@ error = getaddrinfo(host, NULL, &ai_hints, &server_addr); if (error != 0) { (void) fprintf(newstdout, - "%s could not be located by getaddrinfo\n", host); + "%s could not be located by getaddrinfo. Error: %s\n", host, gai_strerror(error)); fflush(newstdout); return (1); } @@ -23773,12 +24453,12 @@ if (error != 0) { if(!mflag) { - (void) fprintf(newstdout,"%s could not be located by getaddrinfo\n",host); + (void) fprintf(newstdout,"%s could not be located by getaddrinfo. Error: %s\n",host, gai_strerror(error)); fflush(newstdout); } else { - (void) printf("\t%s could not be located by getaddrinfo\n",host); + (void) printf("\t%s could not be located by getaddrinfo. Error: %s\n",host,gai_strerror(error)); } return (1); } @@ -23824,7 +24504,7 @@ error = getaddrinfo(host, NULL, &ai_hints, &server_addr); if (error != 0) { (void) fprintf(stdout, - "%s could not be located by getaddrinfo\n", host); + "%s could not be located by getaddrinfo. Error: %s\n", host,gai_strerror(error)); fflush(stdout); return (1); } @@ -23835,7 +24515,7 @@ error = getaddrinfo(host, NULL, &ai_hints, &server_addr); if (error != 0) { (void) fprintf(stdout, - "%s could not be located by getaddrinfo\n",host); + "%s could not be located by getaddrinfo. Error: %s\n",host,gai_strerror(error)); fflush(stdout); return (1); } --- ../../netmist/dist/netmist.h 2017-11-29 10:28:00.367366363 -0600 +++ netmist/dist/netmist.h 2018-05-22 09:57:32.525115658 -0500 @@ -16,6 +16,7 @@ #define NETMIST_LIC_FILE "netmist_license_key" #endif +#define PN_IP_OPTION 1 #define MAXNAME 256 #define MAXHNAME 100 #define MAXFSNAME 20 --- ../../bin.in/dist_pro/SfsManager 2017-11-29 10:28:39.045479764 -0600 +++ bin.in/dist_pro/SfsManager 2018-05-22 15:21:30.402032237 -0500 @@ -183,6 +183,8 @@ netmist_cmd += " -u" if rc.parms["NO_OP_VALIDATE"] == 1: netmist_cmd += " -3" + if rc.parms["PRIME_IP_OPTION"] == 1: + netmist_cmd += " -+h 1" if rc.parms["NO_SHARED_BUCKETS"] == 1: netmist_cmd += " -4" if rc.parms["UNLINK2_NO_RECREATE"] == 1: @@ -795,6 +797,7 @@ "TRACEDEBUG": 0, "INIT_RATE": 0.0, "NO_OP_VALIDATE": 0, + "PRIME_IP_OPTION": 0, "NO_SHARED_BUCKETS": 0, "UNLINK2_NO_RECREATE": 0, "SHARING_MODE": 0, @@ -903,7 +906,7 @@ elif parm in ['IPV6_ENABLE', 'UNLINK_FILES', 'DISABLE_FSYNCS', \ 'USE_RSHRCP', 'LOCAL_ONLY', 'HEARTBEAT_NOTIFICATIONS', \ 'LATENCY_GRAPH', 'FILE_ACCESS_LIST', 'SHARING_MODE', \ - 'SOCK_DEBUG','NO_OP_VALIDATE', 'NO_SHARED_BUCKETS', \ + 'SOCK_DEBUG','NO_OP_VALIDATE', 'PRIME_IP_OPTION', 'NO_SHARED_BUCKETS', \ 'UNLINK2_NO_RECREATE']: if value != "": retval = check_string_bool(value)