parcsv.c

Go to the documentation of this file.

 
/*****************************************************************************/
#include "tpcclibConfig.h"
/*****************************************************************************/
#include <stdio.h>
#include <stdlib.h>
#include <math.h>
#include <time.h>
#include <string.h>
/*****************************************************************************/
#include "tpcextensions.h"
#include "tpccsv.h"
#include "tpcift.h"
/*****************************************************************************/
#include "tpcpar.h"
/*****************************************************************************/
 
/*****************************************************************************/
int parWriteCSV(
  PAR *par,
  FILE *fp,
  parformat format,
  int extra,
  TPCSTATUS *status
) {
  int verbose=0; if(status!=NULL) verbose=status->verbose;
  if(fp==NULL) {
    statusSet(status, __func__, __FILE__, __LINE__, TPCERROR_CANNOT_WRITE);
    return TPCERROR_CANNOT_WRITE;
  }
  if(verbose>0) {printf("%s():\n", __func__); fflush(stdout);}
  if(par==NULL || par->tacNr<1 || par->parNr<1) {
    statusSet(status, __func__, __FILE__, __LINE__, TPCERROR_NO_DATA);
    return TPCERROR_NO_DATA;
  }
 
  /* Determine and verify the write format */
  if(format<=PAR_FORMAT_UNKNOWN || format>=PAR_FORMAT_LAST) format=par->format;
  /* Set decimal and item separators */
  int tointl=0;
  char separator;
  if(format==PAR_FORMAT_CSV_INT) {
    tointl=1; separator=';';
  } else if(format==PAR_FORMAT_CSV_UK) {
    tointl=0; separator=',';
  } else if(format==PAR_FORMAT_TSV_INT) {
    tointl=1; separator='\t';
  } else if(format==PAR_FORMAT_TSV_UK) {
    tointl=0; separator='\t';
  } else {
    statusSet(status, __func__, __FILE__, __LINE__, TPCERROR_INVALID_FORMAT);
    return TPCERROR_INVALID_FORMAT;
  }
  if(verbose>0) {
    printf("writing %d parameters from %d TACs in %s format\n",
      par->parNr, par->tacNr, parFormattxt(format));
  }
 
  int ret;
 
  /* Write header if requested */
  if(extra) {
    if(verbose>1) printf("  add common info into header\n");
    char tmp[128];
    if(parIsModel(par)==1) { // same model in each TAC
      iftDeleteKey(&par->h, "model");
      sprintf(tmp, "%s", modelCode(par->r[0].model));
      iftPut(&par->h, "model", tmp, 1, NULL);
    }
    if(parIsDataNr(par)==1) {
      iftDeleteKey(&par->h, "dataNr");
      sprintf(tmp, "%d", par->r[0].dataNr);
      iftPut(&par->h, "dataNr", tmp, 1, NULL);
    }
    if(parIsFitNr(par)==1) {
      iftDeleteKey(&par->h, "fitNr");
      sprintf(tmp, "%d", par->r[0].fitNr);
      iftPut(&par->h, "fitNr", tmp, 1, NULL);
    }
    if(parIsFitRange(par)==1) {
      char unittext[60];
      int tunit=UNIT_UNKNOWN;
      int i=iftFindKey(&par->h, "timeunit", 0);
      if(i<0) i=iftFindKey(&par->h, "time_unit", 0);
      if(i>=0) tunit=unitIdentify(par->h.item[i].value);
      if(unitIsTime(tunit)) sprintf(unittext, " %s", unitName(tunit)); else strcpy(unittext, "");
      iftDeleteKey(&par->h, "fit_start");
      sprintf(tmp, "%g%s", par->r[0].start, unittext);
      iftPut(&par->h, "fit_start", tmp, 1, NULL);
      iftDeleteKey(&par->h, "fit_end");
      sprintf(tmp, "%g%s", par->r[0].end, unittext);
      iftPut(&par->h, "fit_end", tmp, 1, NULL);
    }
    if(verbose>1) {printf("  writing parameter header\n"); fflush(stdout);}
    for(int i=0; i<par->h.keyNr; i++) par->h.item[i].comment=1;
    ret=iftWrite(&par->h, fp, status); if(ret!=TPCERROR_OK) return ret;
    //fprintf(fp, "\n");
    fflush(fp);
  }
 
  /* Allocate initial space for CSV structure; more will be added later when necessary */
  CSV csv; csvInit(&csv);
  ret=csvAllocate(&csv, (par->parNr+1)*(par->tacNr+1));
  if(ret!=TPCERROR_OK) {
    if(verbose>1) {fprintf(stderr, "Error: cannot allocate memory for CSV.\n"); fflush(stderr);}
    statusSet(status, __func__, __FILE__, __LINE__, ret);
    return ret;
  }
  csv.separator=separator;
 
  /* Write parameter table into CSV */
  if(verbose>1) {printf("  writing parameter table\n"); fflush(stdout);}
  int pi, ri;
  /* Parameter names */
  if(verbose>2) {printf("  writing parameter names\n"); fflush(stdout);}
  char pstr[MAX_PARNAME_LEN+MAX_UNITS_LEN+3];
  ret=csvPutString(&csv, "Parameters", 0);
  /* Column for model id, if different models fitted to TACs */
  if(parIsModel(par)==2 && ret==TPCERROR_OK) 
    ret=csvPutString(&csv, "Model", 0);
  for(pi=0; pi<par->parNr && ret==TPCERROR_OK; pi++) {
    if(par->n[pi].unit==UNIT_UNKNOWN) {
      ret=csvPutString(&csv, par->n[pi].name, 0);
    } else {
      sprintf(pstr, "%s[%s]", par->n[pi].name, unitName(par->n[pi].unit));
      ret=csvPutString(&csv, pstr, 0);
    }
    if(ret==TPCERROR_OK && parSDWithPar(par, pi)) {
      ret=csvPutString(&csv, "SD", 0);
    }
    if(ret==TPCERROR_OK && parCLWithPar(par, pi)) {
      ret=csvPutString(&csv, "95%CL1", 0);
      if(ret==TPCERROR_OK) ret=csvPutString(&csv, "95%CL2", 0);
    }
  }
  if(ret==TPCERROR_OK && parIsWSS(par))
    ret=csvPutString(&csv, "WSS", 0);
  if(ret==TPCERROR_OK && parIsFitRange(par)==2) {
    ret=csvPutString(&csv, "Start[min]", 0);
    if(ret==TPCERROR_OK) ret=csvPutString(&csv, "End[min]", 0);
  }
  if(ret==TPCERROR_OK && parIsDataNr(par)==2)
    ret=csvPutString(&csv, "DataNr", 0);
  if(ret==TPCERROR_OK && parIsFitNr(par)==2)
    ret=csvPutString(&csv, "FitNr", 0);
  if(ret!=TPCERROR_OK) {
    statusSet(status, __func__, __FILE__, __LINE__, ret);
    csvFree(&csv); return ret;
  }
  /* Data table */
  if(verbose>2) {printf("  writing parameter values\n"); fflush(stdout);}
  for(ri=0; ri<par->tacNr && ret==TPCERROR_OK; ri++) {
    if(verbose>3) {printf("    for region %d\n", 1+ri); fflush(stdout);}
    /* Region names and parameter values */
    if(strlen(par->r[ri].name)>0) {
      ret=csvPutString(&csv, par->r[ri].name, 1);
    } else {
      char buf[16]; sprintf(buf, "tac%d", 1+ri);
      ret=csvPutString(&csv, buf, 1);
    }
    if(parIsModel(par)==2 && ret==TPCERROR_OK)
      ret=csvPutString(&csv, modelCode(par->r[ri].model), 0);
    for(pi=0; pi<par->parNr && ret==TPCERROR_OK; pi++) {
      ret=csvPutDouble(&csv, par->r[ri].p[pi], 0, tointl);
      if(ret==TPCERROR_OK && parSDWithPar(par, pi))
        ret=csvPutDouble(&csv, par->r[ri].sd[pi], 0, tointl);
      if(ret==TPCERROR_OK && parCLWithPar(par, pi)) {
        ret=csvPutDouble(&csv, par->r[ri].cl1[pi], 0, tointl);
        if(ret==TPCERROR_OK)
          ret=csvPutDouble(&csv, par->r[ri].cl2[pi], 0, tointl);
      }
    }
    /* WSS etc */
    if(ret==TPCERROR_OK && parIsWSS(par))
      ret=csvPutDouble(&csv, par->r[ri].wss, 0, tointl);
    if(ret==TPCERROR_OK && parIsFitRange(par)==2) {
      ret=csvPutDouble(&csv, par->r[ri].start, 0, tointl);
      if(ret==TPCERROR_OK) ret=csvPutDouble(&csv, par->r[ri].end, 0, tointl);
    }
    if(ret==TPCERROR_OK && parIsDataNr(par)==2)
      ret=csvPutInt(&csv, par->r[ri].dataNr, 0);
    if(ret==TPCERROR_OK && parIsFitNr(par)==2)
      ret=csvPutInt(&csv, par->r[ri].fitNr, 0);
  }
  if(ret!=TPCERROR_OK) {
    statusSet(status, __func__, __FILE__, __LINE__, ret);
    csvFree(&csv); return ret;
  }
 
  /* Write CSV into file pointer */
  if(verbose>2) {printf("  actually writing CSV in file\n"); fflush(stdout);}
  ret=csvWrite(&csv, 0, fp, status);
  csvFree(&csv); fflush(fp);
  if(ret!=TPCERROR_OK) {csvFree(&csv); return ret;}
  
  /* Quit */
  statusSet(status, __func__, __FILE__, __LINE__, ret);
  return(ret);
}
/*****************************************************************************/
 
/*****************************************************************************/
int parReadCSV(
  PAR *par,
  CSV *csv,
  IFT *hdr,
  TPCSTATUS *status
) {
  int verbose=0; if(status!=NULL) verbose=status->verbose;
  if(par==NULL) {
    statusSet(status, __func__, __FILE__, __LINE__, TPCERROR_FAIL);
    return TPCERROR_FAIL;
  }
 
  parFree(par);
  if(csv==NULL || csv->row_nr<1 || csv->col_nr<1) {
    statusSet(status, __func__, __FILE__, __LINE__, TPCERROR_NO_DATA);
    return TPCERROR_NO_DATA;
  }  
  if(verbose>0) printf("%s()\n", __func__);
 
  /* Check that this is valid CSV or TSV parameter file */
  if(strcasecmp(csv->c[0].content, "Parameters")) {
    statusSet(status, __func__, __FILE__, __LINE__, TPCERROR_UNSUPPORTED);
    return TPCERROR_UNSUPPORTED;
  }
 
  /* Remove initial and trailing space characters from CSV.
     Those may exist in manually made files. */
  csvCleanSpaces(csv);
 
  /* Calculate the number of parameters from the title line */
  int colNr=1, parNr=0;
  for(int i=1; i<csv->nr; i++) {
    if(csv->c[i].row==0) { // title line
      colNr++;
      /* do not count special columns */
      if(strncasecmp(csv->c[i].content, "SD", 2)==0) continue;
      if(strncasecmp(csv->c[i].content, "95%CL", 5)==0) continue;
      if(strcasecmp(csv->c[i].content, "WSS")==0) continue;
      if(strcasecmp(csv->c[i].content, "Model")==0) continue;
      if(strcasecmp(csv->c[i].content, "Start[min]")==0) continue;
      if(strcasecmp(csv->c[i].content, "End[min]")==0) continue;
      if(strcasecmp(csv->c[i].content, "dataNr")==0) continue;
      if(strcasecmp(csv->c[i].content, "fitNr")==0) continue;
      parNr++;
    } else break; // this is not more in title line
  }
  if(parNr<1 || csv->row_nr<2) {
    statusSet(status, __func__, __FILE__, __LINE__, TPCERROR_INVALID_FORMAT);
    return TPCERROR_INVALID_FORMAT;
  }
  if(verbose>2) printf("  parNr := %d\n  colNr := %d\n", parNr, colNr);
 
  /* Allocate memory for parameters */
  int ret;
  ret=parAllocate(par, parNr, csv->row_nr-1);
  if(ret!=TPCERROR_OK) {statusSet(status, __func__, __FILE__, __LINE__, ret); return ret;}
  par->parNr=parNr; par->tacNr=csv->row_nr-1;
 
 
  /* Read CSV contents into PAR struct */
  int decimal_comma=-1;
  ret=0; if(csv->separator==',') decimal_comma=0;
  /* Read TAC names from column 1 */
  for(int ri=1; ri<csv->row_nr; ri++) {
    int i=ri*colNr; if(i>=csv->nr || csv->c[i].col!=0) {ret++; break;}
    strlcpy(par->r[ri-1].name, csv->c[i].content, MAX_TACNAME_LEN);
  }
 
  /* Read one column at a time */
  parNr=0;
  for(int ci=1; ci<colNr && ret==0; ci++) {
    //printf("parNr=%d / %d\n", parNr, par->parNr);
    if(parNr>par->parNr) {ret++; break;}
    int ri=0; int i=ri*colNr+ci;
    if(i>=csv->nr || csv->c[i].col!=ci) {ret++; break;}
    if(strcasecmp(csv->c[i].content, "Model")==0) {
      for(ri=1; ri<csv->row_nr; ri++) {
        int i=ri*colNr+ci;
        if(i>=csv->nr || csv->c[i].col!=ci) {ret++; break;}
        par->r[ri-1].model=modelCodeIndex(csv->c[i].content);
      }
      continue;
    }
    if(strncasecmp(csv->c[i].content, "SD", 2)==0) {
      if(parNr==0) {ret++; break;}
      for(ri=1; ri<csv->row_nr; ri++) {
        i=ri*colNr+ci;
        if(i>=csv->nr || csv->c[i].col!=ci) {ret++; break;}
        par->r[ri-1].sd[parNr-1]=atofVerified(csv->c[i].content);
      }
      continue;
    }
    if(strncasecmp(csv->c[i].content, "95%CL1", 6)==0) {
      if(parNr==0) {ret++; break;}
      for(ri=1; ri<csv->row_nr; ri++) {
        i=ri*colNr+ci;
        if(i>=csv->nr || csv->c[i].col!=ci) {ret++; break;}
        par->r[ri-1].cl1[parNr-1]=atofVerified(csv->c[i].content);
      }
      continue;
    }
    if(strncasecmp(csv->c[i].content, "95%CL2", 6)==0) {
      if(parNr==0) {ret++; break;}
      for(ri=1; ri<csv->row_nr; ri++) {
        i=ri*colNr+ci;
        if(i>=csv->nr || csv->c[i].col!=ci) {ret++; break;}
        par->r[ri-1].cl2[parNr-1]=atofVerified(csv->c[i].content);
      }
      continue;
    }
    if(strcasecmp(csv->c[i].content, "WSS")==0 || strcasecmp(csv->c[i].content, "SS")==0) {
      for(ri=1; ri<csv->row_nr; ri++) {
        i=ri*colNr+ci;
        if(i>=csv->nr || csv->c[i].col!=ci) {ret++; break;}
        par->r[ri-1].wss=atofVerified(csv->c[i].content);
      }
      continue;
    }
    if(strncasecmp(csv->c[i].content, "Start", 5)==0) {
      for(ri=1; ri<csv->row_nr; ri++) {
        i=ri*colNr+ci;
        if(i>=csv->nr || csv->c[i].col!=ci) {ret++; break;}
        par->r[ri-1].start=atofVerified(csv->c[i].content);
      }
      continue;
    }
    if(strncasecmp(csv->c[i].content, "End", 3)==0) {
      for(ri=1; ri<csv->row_nr; ri++) {
        i=ri*colNr+ci;
        if(i>=csv->nr || csv->c[i].col!=ci) {ret++; break;}
        par->r[ri-1].end=atofVerified(csv->c[i].content);
      }
      continue;
    }
    if(strcasecmp(csv->c[i].content, "dataNr")==0) {
      for(ri=1; ri<csv->row_nr; ri++) {
        i=ri*colNr+ci;
        if(i>=csv->nr || csv->c[i].col!=ci) {ret++; break;}
        par->r[ri-1].dataNr=atoi(csv->c[i].content);
      }
      continue;
    }
    if(strcasecmp(csv->c[i].content, "fitNr")==0) {
      for(ri=1; ri<csv->row_nr; ri++) {
        i=ri*colNr+ci;
        if(i>=csv->nr || csv->c[i].col!=ci) {ret++; break;}
        par->r[ri-1].fitNr=atoi(csv->c[i].content);
      }
      continue;
    }
    /* then this column is a parameter */
    char tmp[128];
    /* parameter name may contain unit */
    if(strTokenNCpy(csv->c[i].content, "[] \t", 1, par->n[parNr].name, MAX_PARNAME_LEN)==0) 
      {ret++; break;}
    if(strTokenNCpy(csv->c[i].content, "[] \t", 2, tmp, 128)>0)
      par->n[parNr].unit=unitIdentify(tmp);
    //strncpy(par->n[parNr].name, csv->c[i].content, MAX_PARNAME_LEN);
    //par->n[parNr].name[MAX_PARNAME_LEN]='\0';
    /* Read parameter values */
    for(ri=1; ri<csv->row_nr; ri++) {
      i=ri*colNr+ci;
      if(i>=csv->nr || csv->c[i].col!=ci) {ret++; break;}
      par->r[ri-1].p[parNr]=atofVerified(csv->c[i].content);
      if(decimal_comma<1) {
        /* Try to determine whether file format is UK or INT */
        if(strchr(csv->c[i].content, '.')!=NULL) decimal_comma=0;
        else if(strchr(csv->c[i].content, ',')!=NULL) decimal_comma=1;
      }
    }
    parNr++;
  }
  if(ret>0) {
    parFree(par);
    statusSet(status, __func__, __FILE__, __LINE__, TPCERROR_INVALID_FORMAT);
    return TPCERROR_INVALID_FORMAT;
  }
 
 
  /* If one 'parameter' is named WSS or SS, then move its contents to
     correct place inside the struct */
  {
    int ci=-1;
    for(int i=0; i<par->parNr; i++) {
      if(!strcasecmp(par->n[i].name, "WSS") || !strcasecmp(par->n[i].name, "SS")) ci=i;
    }
    if(ci>=0 && ci<par->parNr && par->parNr>1) {
      if(verbose>4) {
        printf("column %d contains WSS, moving to correct place\n", 1+ci); fflush(stdout);}
      for(int i=0; i<par->tacNr; i++) par->r[i].wss=par->r[i].p[ci];
      /* delete the parameter column */
      parDeletePar(par, ci);
    }
  }
 
 
  /* Set format in struct */
  if(csv->separator==';') {
    if(decimal_comma==1) par->format=PAR_FORMAT_CSV_INT;
    else par->format=PAR_FORMAT_CSV_UK;
  } else if(csv->separator==',') {
    par->format=PAR_FORMAT_CSV_UK;
  } else if(csv->separator=='\t' || csv->separator==' ') {
    if(decimal_comma==1) par->format=PAR_FORMAT_TSV_INT;
    else par->format=PAR_FORMAT_TSV_UK;
  } else {
    if(decimal_comma==1) par->format=PAR_FORMAT_TSV_INT;
    else par->format=PAR_FORMAT_TSV_UK;
  }
 
 
 
  /* Copy header */
  if(hdr!=NULL && hdr->keyNr>0) {
    if(verbose>4) {
       printf("copying header contents (%d items) to PAR struct\n", hdr->keyNr); fflush(stdout);}
    if(iftDuplicate(hdr, &par->h)!=TPCERROR_OK) {
      if(verbose>0) {printf("  error in iftDuplicate()\n"); fflush(stdout);}
      parFree(par);
      statusSet(status, __func__, __FILE__, __LINE__, TPCERROR_INVALID_HEADER);
      return TPCERROR_INVALID_HEADER;
    }
    /* Move standard fields from header to struct */
    double v, f;
    int i, n, u;
    char key[32];
 
    strcpy(key, "fit_start");
    if(!iftGetDoubleWithUnit(&par->h, iftFindKey(&par->h, key, 0), &v, &u)) {
      f=unitConversionFactor(u, UNIT_MIN); if(!isnan(f)) v*=f;
      for(i=0; i<par->tacNr; i++) if(isnan(par->r[i].start)) par->r[i].start=v;
    }
    strcpy(key, "fit_end");
    if(!iftGetDoubleWithUnit(&par->h, iftFindKey(&par->h, key, 0), &v, &u)) {
      f=unitConversionFactor(u, UNIT_MIN); if(!isnan(f)) v*=f;
      for(i=0; i<par->tacNr; i++) if(isnan(par->r[i].end)) par->r[i].end=v;
    }
    strcpy(key, "dataNr");
    if(!iftGetInt(&par->h, iftFindKey(&par->h, key, 0), &n)) {
      for(i=0; i<par->tacNr; i++) if(par->r[i].dataNr<=0) par->r[i].dataNr=n;
    }
    strcpy(key, "model");
    i=iftFindKey(&par->h, key, 0);
    if(i>=0) {
      unsigned int n=modelCodeIndex(par->h.item[i].value);
      iftDelete(&par->h, i);
      for(i=0; i<par->tacNr; i++) par->r[i].model=n;
    }
  }
 
  /* Quit */
  statusSet(status, __func__, __FILE__, __LINE__, TPCERROR_OK);
  return(TPCERROR_OK);
}
/*****************************************************************************/
 
/*****************************************************************************/