cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/berlinModel/in/BerlinModelReferenceImport.java

   1 /**
   2 * Copyright (C) 2007 EDIT
   3 * European Distributed Institute of Taxonomy
   4 * http://www.e-taxonomy.eu
   5 *
   6 * The contents of this file are subject to the Mozilla Public License Version 1.1
   7 * See LICENSE.TXT at the top of this package for the full license terms.
   8 */
   9
  10 package eu.etaxonomy.cdm.io.berlinModel.in;
  11
  12 import static eu.etaxonomy.cdm.io.berlinModel.BerlinModelTransformer.REF_ARTICLE;
  13 import static eu.etaxonomy.cdm.io.berlinModel.BerlinModelTransformer.REF_BOOK;
  14 import static eu.etaxonomy.cdm.io.berlinModel.BerlinModelTransformer.REF_CONFERENCE_PROCEEDINGS;
  15 import static eu.etaxonomy.cdm.io.berlinModel.BerlinModelTransformer.REF_DATABASE;
  16 import static eu.etaxonomy.cdm.io.berlinModel.BerlinModelTransformer.REF_INFORMAL;
  17 import static eu.etaxonomy.cdm.io.berlinModel.BerlinModelTransformer.REF_JOURNAL;
  18 import static eu.etaxonomy.cdm.io.berlinModel.BerlinModelTransformer.REF_JOURNAL_VOLUME;
  19 import static eu.etaxonomy.cdm.io.berlinModel.BerlinModelTransformer.REF_PART_OF_OTHER_TITLE;
  20 import static eu.etaxonomy.cdm.io.berlinModel.BerlinModelTransformer.REF_PRINT_SERIES;
  21 import static eu.etaxonomy.cdm.io.berlinModel.BerlinModelTransformer.REF_UNKNOWN;
  22 import static eu.etaxonomy.cdm.io.berlinModel.BerlinModelTransformer.REF_WEBSITE;
  23 import static eu.etaxonomy.cdm.io.common.IImportConfigurator.DO_REFERENCES.ALL;
  24 import static eu.etaxonomy.cdm.io.common.IImportConfigurator.DO_REFERENCES.CONCEPT_REFERENCES;
  25 import static eu.etaxonomy.cdm.io.common.IImportConfigurator.DO_REFERENCES.NOMENCLATURAL;
  26 import static eu.etaxonomy.cdm.io.common.ImportHelper.NO_OVERWRITE;
  27 import static eu.etaxonomy.cdm.io.common.ImportHelper.OBLIGATORY;
  28 import static eu.etaxonomy.cdm.io.common.ImportHelper.OVERWRITE;
  29
  30 import java.net.URI;
  31 import java.net.URISyntaxException;
  32 import java.sql.ResultSet;
  33 import java.sql.SQLException;
  34 import java.util.ArrayList;
  35 import java.util.Arrays;
  36 import java.util.HashMap;
  37 import java.util.HashSet;
  38 import java.util.List;
  39 import java.util.Map;
  40 import java.util.Set;
  41 import java.util.UUID;
  42
  43 import org.apache.log4j.Logger;
  44 import org.springframework.stereotype.Component;
  45
  46 import eu.etaxonomy.cdm.common.CdmUtils;
  47 import eu.etaxonomy.cdm.io.berlinModel.CdmOneToManyMapper;
  48 import eu.etaxonomy.cdm.io.berlinModel.CdmStringMapper;
  49 import eu.etaxonomy.cdm.io.berlinModel.CdmUriMapper;
  50 import eu.etaxonomy.cdm.io.berlinModel.in.validation.BerlinModelReferenceImportValidator;
  51 import eu.etaxonomy.cdm.io.common.ICdmIO;
  52 import eu.etaxonomy.cdm.io.common.IImportConfigurator;
  53 import eu.etaxonomy.cdm.io.common.ImportHelper;
  54 import eu.etaxonomy.cdm.io.common.ResultSetPartitioner;
  55 import eu.etaxonomy.cdm.io.common.Source;
  56 import eu.etaxonomy.cdm.io.common.mapping.CdmAttributeMapperBase;
  57 import eu.etaxonomy.cdm.io.common.mapping.CdmIoMapping;
  58 import eu.etaxonomy.cdm.io.common.mapping.CdmSingleAttributeMapperBase;
  59 import eu.etaxonomy.cdm.io.common.mapping.DbImportExtensionMapper;
  60 import eu.etaxonomy.cdm.io.common.mapping.DbImportMarkerMapper;
  61 import eu.etaxonomy.cdm.model.agent.Team;
  62 import eu.etaxonomy.cdm.model.agent.TeamOrPersonBase;
  63 import eu.etaxonomy.cdm.model.common.CdmBase;
  64 import eu.etaxonomy.cdm.model.common.ExtensionType;
  65 import eu.etaxonomy.cdm.model.common.IdentifiableSource;
  66 import eu.etaxonomy.cdm.model.common.Marker;
  67 import eu.etaxonomy.cdm.model.common.MarkerType;
  68 import eu.etaxonomy.cdm.model.reference.IArticle;
  69 import eu.etaxonomy.cdm.model.reference.IBook;
  70 import eu.etaxonomy.cdm.model.reference.IBookSection;
  71 import eu.etaxonomy.cdm.model.reference.IJournal;
  72 import eu.etaxonomy.cdm.model.reference.IPrintSeries;
  73 import eu.etaxonomy.cdm.model.reference.Reference;
  74 import eu.etaxonomy.cdm.model.reference.ReferenceFactory;
  75 import eu.etaxonomy.cdm.model.reference.ReferenceType;
  76
  77 /**
  78  * @author a.mueller
  79  * @created 20.03.2008
  80  * @version 1.0
  81  */
  82 @Component
  83 public class BerlinModelReferenceImport extends BerlinModelImportBase {
  84         private static final Logger logger = Logger.getLogger(BerlinModelReferenceImport.class);
  85
  86         public static final String NOM_REFERENCE_NAMESPACE = "NomReference";
  87         public static final String BIBLIO_REFERENCE_NAMESPACE = "BiblioReference";
  88
  89         public static final UUID REF_DEPOSITED_AT_UUID = UUID.fromString("23ca88c7-ce73-41b2-8ca3-2cb22f013beb");
  90         public static final UUID REF_SOURCE_UUID = UUID.fromString("d6432582-2216-4b08-b0db-76f6c1013141");
  91         public static final UUID DATE_STRING_UUID = UUID.fromString("e4130eae-606e-4b0c-be4f-e93dc161be7d");
  92         public static final UUID IS_PAPER_UUID = UUID.fromString("8a326129-d0d0-4f9d-bbdf-8d86b037c65e");
  93
  94
  95         private int modCount = 1000;
  96         private static final String pluralString = "references";
  97         private static final String dbTableName = "reference";
  98
  99
 100         public BerlinModelReferenceImport(){
 101                 super();
 102         }
 103
 104         protected void initializeMappers(BerlinModelImportState state){
 105                 for (CdmAttributeMapperBase mapper: classMappers){
 106                         if (mapper instanceof DbImportExtensionMapper){
 107                                 ((DbImportExtensionMapper)mapper).initialize(state, Reference.class);
 108                         }
 109                 }
 110                 return;
 111         }
 112
 113         protected static CdmAttributeMapperBase[] classMappers = new CdmAttributeMapperBase[]{
 114                 new CdmStringMapper("edition", "edition"),
 115                 new CdmStringMapper("volume", "volume"),
 116                 new CdmStringMapper("publisher", "publisher"),
 117                 new CdmStringMapper("publicationTown", "placePublished"),
 118                 new CdmStringMapper("isbn", "isbn"),
 119                 new CdmStringMapper("isbn", "isbn"),
 120                 new CdmStringMapper("pageString", "pages"),
 121                 new CdmStringMapper("series", "series"),
 122                 new CdmStringMapper("issn", "issn"),
 123                 new CdmUriMapper("url", "uri"),
 124                 DbImportExtensionMapper.NewInstance("NomStandard", ExtensionType.NOMENCLATURAL_STANDARD()),
 125                 DbImportExtensionMapper.NewInstance("DateString", DATE_STRING_UUID, "Date String", "Date String", "dates"),
 126                 DbImportExtensionMapper.NewInstance("RefDepositedAt", REF_DEPOSITED_AT_UUID, "RefDepositedAt", "reference is deposited at", "at"),
 127                 DbImportExtensionMapper.NewInstance("RefSource", REF_SOURCE_UUID, "RefSource", "reference source", "source"),
 128                 DbImportMarkerMapper.NewInstance("isPaper", IS_PAPER_UUID, "is paper", "is paper", "paper", false)
 129         };
 130
 131
 132         protected static String[] operationalAttributes = new String[]{
 133                 "refId", "refCache", "nomRefCache", "preliminaryFlag", "inRefFk", "title", "nomTitleAbbrev",
 134                 "refAuthorString", "nomAuthorTeamFk",
 135                 "refCategoryFk", "thesisFlag", "informalRefCategory", "idInSource"
 136         };
 137
 138         protected static String[] createdAndNotesAttributes = new String[]{
 139                         "created_When", "updated_When", "created_Who", "updated_Who", "notes"
 140         };
 141
 142         protected static String[] unclearMappers = new String[]{
 143                         /*"isPaper",*/ "exportDate",
 144         };
 145
 146         //TODO isPaper
 147         //
 148
 149
 150
 151         //type to count the references nomReferences that have been created and saved
 152         private class RefCounter{
 153                 RefCounter() {nomRefCount = 0; referenceCount = 0;};
 154                 int nomRefCount;
 155                 int referenceCount;
 156                 public String toString(){return String.valueOf(nomRefCount) + "," +String.valueOf(referenceCount);};
 157         }
 158
 159
 160         /* (non-Javadoc)
 161          * @see eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelImportBase#getRecordQuery(eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelImportConfigurator)
 162          */
 163         @Override
 164         protected String getRecordQuery(BerlinModelImportConfigurator config) {
 165                 return null;  //not needed
 166         }
 167
 168
 169 //      @Override
 170 //      protected void doInvoke_old(BerlinModelImportState state){
 171 //              logger.info("start make " + getPluralString() + " ...");
 172 //
 173 //              boolean success = true;
 174 //              initializeMappers(state);
 175 //              BerlinModelImportConfigurator config = state.getConfig();
 176 //              Source source = config.getSource();
 177 //
 178 //              String strSelectId = " SELECT Reference.RefId as refId ";
 179 //              String strSelectFull =
 180 //                      " SELECT Reference.* , InReference.RefId as InRefId, InReference.RefCategoryFk as InRefCategoryFk,  " +
 181 //                      " InInReference.RefId as InInRefId, InInReference.RefCategoryFk as InInRefCategoryFk, " +
 182 //                      " InReference.InRefFk AS InRefInRefFk, InInReference.InRefFk AS InInRefInRefFk, RefSource.RefSource " ;
 183 //              String strFrom =  " FROM Reference AS InInReference " +
 184 //                      " RIGHT OUTER JOIN Reference AS InReference ON InInReference.RefId = InReference.InRefFk " +
 185 //                      " RIGHT OUTER JOIN %s ON InReference.RefId = Reference.InRefFk " +
 186 //                      " LEFT OUTER JOIN RefSource ON Reference.RefSourceFk = RefSource.RefSourceId " +
 187 //                      " WHERE (1=1) ";
 188 //              String strWherePartitioned = " AND (Reference.refId IN ("+ ID_LIST_TOKEN + ") ) ";
 189 //
 190 //              String referenceTable = CdmUtils.Nz(state.getConfig().getReferenceIdTable());
 191 //              referenceTable = referenceTable.isEmpty() ? " Reference"  : referenceTable + " as Reference ";
 192 //              String strIdFrom = String.format(strFrom, referenceTable );
 193 //
 194 //
 195 //              //test max number of recursions
 196 //              String strQueryTestMaxRecursion = strSelectId + strIdFrom +
 197 //                      " AND (Reference.InRefFk is NOT NULL) AND (InReference.InRefFk is NOT NULL) AND (InInReference.InRefFk is NOT NULL) ";
 198 //              ResultSet testMaxRecursionResultSet = source.getResultSet(strQueryTestMaxRecursion);
 199 //              try {
 200 //                      if (testMaxRecursionResultSet.next() == true){
 201 //                              logger.error("Maximum allowed InReference recursions exceeded in Berlin Model. Maximum recursion level is 2.");
 202 //                              state.setUnsuccessfull();
 203 //                              return;
 204 //                      }
 205 //              } catch (SQLException e1) {
 206 //                      e1.printStackTrace();
 207 //                      logger.error("There are references with more then 2 in-reference recursion. Maximum number of allowed recursions is 2. Records will not be stored.");
 208 //                      success = false;
 209 //              }
 210 //
 211 //              String strSelectIdBase = strSelectId + strIdFrom;
 212 //
 213 //              String referenceFilter = CdmUtils.Nz(state.getConfig().getReferenceIdTable());
 214 //              if (! referenceFilter.isEmpty()){
 215 //                      referenceFilter = " AND " + referenceFilter + " ";
 216 //              }
 217 //              referenceFilter = "";  //don't use it for now
 218 //
 219 //              String strIdQueryNoInRef = strSelectIdBase +
 220 //                      " AND (Reference.InRefFk is NULL) " +  referenceFilter;
 221 //              String strIdQuery1InRef = strSelectIdBase +
 222 //                      " AND (Reference.InRefFk is NOT NULL) AND (InReference.InRefFk is NULL) "  +  referenceFilter;
 223 //              String strIdQuery2InRefs = strSelectIdBase +
 224 //                      " AND (Reference.InRefFk is NOT NULL) AND (InReference.InRefFk is NOT NULL) AND (InInReference.InRefFk is NULL) "  +  referenceFilter;
 225 //
 226 //              if (config.getDoReferences() == CONCEPT_REFERENCES){
 227 //                      strIdQueryNoInRef += " AND ( Reference.refId IN ( SELECT ptRefFk FROM PTaxon) ) " + referenceFilter;
 228 //              }
 229 //
 230 //              String strRecordQuery = strSelectFull + String.format(strFrom, " Reference ") + strWherePartitioned;
 231 //
 232 //              int recordsPerTransaction = config.getRecordsPerTransaction();
 233 //              try{
 234 //                      //NoInRefs
 235 //                      ResultSetPartitioner partitioner = ResultSetPartitioner.NewInstance(source, strIdQueryNoInRef, strRecordQuery, recordsPerTransaction);
 236 //                      while (partitioner.nextPartition()){
 237 //                              partitioner.doPartition(this, state);
 238 //                      }
 239 //                      logger.info("end make references with no in-references ... " + getSuccessString(success));
 240 //
 241 //                      if (config.getDoReferences() == ALL || config.getDoReferences() == NOMENCLATURAL){
 242 //
 243 //                              //1InRef
 244 //                              partitioner = ResultSetPartitioner.NewInstance(source, strIdQuery1InRef, strRecordQuery, recordsPerTransaction);
 245 //                              while (partitioner.nextPartition()){
 246 //                                      partitioner.doPartition(this, state);
 247 //                              }
 248 //                              logger.info("end make references with no 1 in-reference ... " + getSuccessString(success));
 249 //
 250 //                              //2InRefs
 251 //                              partitioner = ResultSetPartitioner.NewInstance(source, strIdQuery2InRefs, strRecordQuery, recordsPerTransaction);
 252 //                              while (partitioner.nextPartition()){
 253 //                                      partitioner.doPartition(this, state);
 254 //                              }
 255 //                              logger.info("end make references with no 2 in-reference ... " + getSuccessString(success));
 256 //                      }
 257 //
 258 //              } catch (SQLException e) {
 259 //                      logger.error("SQLException:" +  e);
 260 //                      state.setUnsuccessfull();
 261 //                      return;
 262 //              }
 263 //              logger.info("end make " + getPluralString() + " ... " + getSuccessString(success));
 264 //              if (! success){
 265 //                      state.setUnsuccessfull();
 266 //              }
 267 //              return;
 268 //      }
 269
 270         @Override
 271         protected void doInvoke(BerlinModelImportState state){
 272                 logger.info("start make " + getPluralString() + " ...");
 273
 274                 boolean success = true;
 275                 initializeMappers(state);
 276                 BerlinModelImportConfigurator config = state.getConfig();
 277                 Source source = config.getSource();
 278
 279                 String strSelectId = " SELECT Reference.RefId as refId ";
 280                 String strSelectFull =
 281                         " SELECT Reference.* ,InReference.RefCategoryFk as InRefCategoryFk, RefSource.RefSource " ;
 282                 String strFrom =  " FROM %s  " +
 283                         " LEFT OUTER JOIN Reference as InReference ON InReference.refId = Reference.inRefFk " +
 284                                 " LEFT OUTER JOIN RefSource ON Reference.RefSourceFk = RefSource.RefSourceId " +
 285                         " WHERE (1=1) ";
 286                 String strWherePartitioned = " AND (Reference.refId IN ("+ ID_LIST_TOKEN + ") ) ";
 287
 288                 String referenceTable = CdmUtils.Nz(state.getConfig().getReferenceIdTable());
 289                 referenceTable = referenceTable.isEmpty() ? " Reference"  : referenceTable + " as Reference ";
 290                 String strIdFrom = String.format(strFrom, referenceTable );
 291
 292
 293                 //test max number of recursions
 294 //              String strQueryTestMaxRecursion = strSelectId + strIdFrom +
 295 //                      " AND (Reference.InRefFk is NOT NULL) AND (InReference.InRefFk is NOT NULL) AND (InInReference.InRefFk is NOT NULL) ";
 296 //              ResultSet testMaxRecursionResultSet = source.getResultSet(strQueryTestMaxRecursion);
 297 //              try {
 298 //                      if (testMaxRecursionResultSet.next() == true){
 299 //                              logger.error("Maximum allowed InReference recursions exceeded in Berlin Model. Maximum recursion level is 2.");
 300 //                              state.setUnsuccessfull();
 301 //                              return;
 302 //                      }
 303 //              } catch (SQLException e1) {
 304 //                      e1.printStackTrace();
 305 //                      logger.error("There are references with more then 2 in-reference recursion. Maximum number of allowed recursions is 2. Records will not be stored.");
 306 //                      success = false;
 307 //              }
 308
 309                 String strSelectIdBase = strSelectId + strIdFrom;
 310
 311                 String referenceFilter = CdmUtils.Nz(state.getConfig().getReferenceIdTable());
 312                 if (! referenceFilter.isEmpty()){
 313                         referenceFilter = " AND " + referenceFilter + " ";
 314                 }
 315                 referenceFilter = "";  //don't use it for now
 316
 317                 String strIdQueryFirstPath = strSelectId + strIdFrom ;
 318                 String strIdQuerySecondPath = strSelectId + strIdFrom + " AND (Reference.InRefFk is NOT NULL) ";
 319
 320 //              if (config.getDoReferences() == CONCEPT_REFERENCES){
 321 //                      strIdQueryNoInRef += " AND ( Reference.refId IN ( SELECT ptRefFk FROM PTaxon) ) " + referenceFilter;
 322 //              }
 323
 324                 String strRecordQuery = strSelectFull + String.format(strFrom, " Reference ") + strWherePartitioned;
 325
 326                 int recordsPerTransaction = config.getRecordsPerTransaction();
 327                 try{
 328                         //firstPath
 329                         ResultSetPartitioner partitioner = ResultSetPartitioner.NewInstance(source, strIdQueryFirstPath, strRecordQuery, recordsPerTransaction);
 330                         while (partitioner.nextPartition()){
 331                                 partitioner.doPartition(this, state);
 332                         }
 333                         logger.info("end make references without in-references ... " + getSuccessString(success));
 334                         state.setReferenceSecondPath(true);
 335
 336 //                      if (config.getDoReferences() == ALL || config.getDoReferences() == NOMENCLATURAL){
 337
 338                         //secondPath
 339                         partitioner = ResultSetPartitioner.NewInstance(source, strIdQuerySecondPath, strRecordQuery, recordsPerTransaction);
 340                         while (partitioner.nextPartition()){
 341                                 partitioner.doPartition(this, state);
 342                         }
 343                         logger.info("end make references with no 1 in-reference ... " + getSuccessString(success));
 344                         state.setReferenceSecondPath(false);
 345
 346 //                      }
 347
 348                 } catch (SQLException e) {
 349                         logger.error("SQLException:" +  e);
 350                         state.setUnsuccessfull();
 351                         return;
 352                 }
 353                 logger.info("end make " + getPluralString() + " ... " + getSuccessString(success));
 354                 if (! success){
 355                         state.setUnsuccessfull();
 356                 }
 357                 return;
 358         }
 359
 360
 361
 362
 363         /* (non-Javadoc)
 364          * @see eu.etaxonomy.cdm.io.berlinModel.in.IPartitionedIO#doPartition(eu.etaxonomy.cdm.io.berlinModel.in.ResultSetPartitioner, eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelImportState)
 365          */
 366         public boolean doPartition(ResultSetPartitioner partitioner, BerlinModelImportState state) {
 367                 if (state.isReferenceSecondPath()){
 368                         return doPartitionSecondPath(partitioner, state);
 369                 }
 370                 boolean success = true;
 371
 372                 Map<Integer, Reference> nomRefToSave = new HashMap<Integer, Reference>();
 373                 Map<Integer, Reference> biblioRefToSave = new HashMap<Integer, Reference>();
 374
 375                 Map<String, Reference> relatedNomReferences = partitioner.getObjectMap(NOM_REFERENCE_NAMESPACE);
 376                 Map<String, Reference> relatedBiblioReferences = partitioner.getObjectMap(BIBLIO_REFERENCE_NAMESPACE);
 377
 378                 BerlinModelImportConfigurator config = state.getConfig();
 379
 380                 try {
 381
 382                                 int i = 0;
 383                                 RefCounter refCounter  = new RefCounter();
 384
 385                                 ResultSet rs = partitioner.getResultSet();
 386
 387                                 //for each resultset
 388                                 while (rs.next()){
 389                                         if ((i++ % modCount) == 0 && i!= 1 ){ logger.info("References handled: " + (i-1) + " in round -" );}
 390
 391                                         success &= makeSingleReferenceRecord(rs, state, partitioner, biblioRefToSave, nomRefToSave, relatedBiblioReferences, relatedNomReferences, refCounter);
 392                                 } // end resultSet
 393
 394                                 //for the concept reference a fixed uuid may be needed -> change uuid
 395                                 Integer sourceSecId = (Integer)config.getSourceSecId();
 396                                 Reference<?> sec = biblioRefToSave.get(sourceSecId);
 397                                 if (sec == null){
 398                                         sec = nomRefToSave.get(sourceSecId);
 399                                 }
 400                                 if (sec != null){
 401                                         sec.setUuid(config.getSecUuid());
 402                                         logger.info("SecUuid changed to: " + config.getSecUuid());
 403                                 }
 404
 405                                 //save and store in map
 406                                 logger.info("Save nomenclatural references (" + refCounter.nomRefCount + ")");
 407                                 getReferenceService().save(nomRefToSave.values());
 408                                 logger.info("Save bibliographical references (" + refCounter.referenceCount +")");
 409                                 getReferenceService().save(biblioRefToSave.values());
 410
 411 //                      }//end resultSetList
 412
 413                         logger.info("end makeReferences ..." + getSuccessString(success));;
 414                         return success;
 415                 } catch (SQLException e) {
 416                         logger.error("SQLException:" +  e);
 417                         return false;
 418                 }
 419         }
 420
 421
 422
 423         /**
 424          * Adds the inReference to the according references.
 425          * @param partitioner
 426          * @param state
 427          * @return
 428          */
 429         private boolean doPartitionSecondPath(ResultSetPartitioner partitioner, BerlinModelImportState state) {
 430                 boolean success = true;
 431
 432                 Map<Integer, Reference> nomRefToSave = new HashMap<Integer, Reference>();
 433                 Map<Integer, Reference> biblioRefToSave = new HashMap<Integer, Reference>();
 434
 435                 Map<String, Reference> relatedNomReferences = partitioner.getObjectMap(NOM_REFERENCE_NAMESPACE);
 436                 Map<String, Reference> relatedBiblioReferences = partitioner.getObjectMap(BIBLIO_REFERENCE_NAMESPACE);
 437
 438                 try {
 439                                 int i = 0;
 440                                 RefCounter refCounter  = new RefCounter();
 441
 442                                 ResultSet rs = partitioner.getResultSet();
 443                                 //for each resultset
 444                                 while (rs.next()){
 445                                         if ((i++ % modCount) == 0 && i!= 1 ){ logger.info("References handled: " + (i-1) + " in round -" );}
 446
 447                                         Integer refId = rs.getInt("refId");
 448                                         Integer inRefFk = rs.getInt("inRefFk");
 449
 450                                         if (inRefFk != null){
 451                                                 Reference<?> thisNomRef = getReferenceOnlyFromMaps(relatedNomReferences, relatedBiblioReferences, String.valueOf(refId));
 452                                                 Reference<?> thisBiblioRef = getReferenceOnlyFromMaps(relatedBiblioReferences, relatedNomReferences, String.valueOf(refId));
 453
 454                                                 Reference<?> nomInReference = relatedNomReferences.get(String.valueOf(inRefFk));
 455                                                 Reference<?> biblioInReference = relatedBiblioReferences.get(String.valueOf(inRefFk));
 456                                                 boolean inRefExists = false;
 457                                                 if (thisNomRef != null){
 458                                                         Reference<?> inRef = (nomInReference != null)? nomInReference : biblioInReference;
 459                                                         thisNomRef.setInReference(inRef);
 460                                                         nomRefToSave.put(refId, thisNomRef);
 461                                                         //remember that an in reference exists
 462                                                         inRefExists |= (inRef != null);
 463                                                 }
 464                                                 if (thisBiblioRef != null){
 465                                                         Reference<?> inRef = (biblioInReference != null)? biblioInReference : nomInReference;
 466                                                         thisBiblioRef.setInReference(inRef);
 467                                                         biblioRefToSave.put(refId, thisBiblioRef);
 468                                                         //remember that an in reference exists
 469                                                         inRefExists |= (inRef != null);
 470                                                 }
 471                                                 if (inRefExists == false){
 472                                                         logger.warn("No in reference was saved though an 'inRefFk' is available. RefId " + refId);
 473                                                 }
 474                                         }
 475
 476                                 } // end resultSet
 477
 478                                 //save and store in map
 479                                 logger.info("Save nomenclatural references (" + refCounter.nomRefCount + ")");
 480                                 getReferenceService().save(nomRefToSave.values());
 481                                 logger.info("Save bibliographical references (" + refCounter.referenceCount +")");
 482                                 getReferenceService().save(biblioRefToSave.values());
 483
 484 //                      }//end resultSetList
 485
 486                         logger.info("end makeReferences ..." + getSuccessString(success));;
 487                         return success;
 488                 } catch (SQLException e) {
 489                         logger.error("SQLException:" +  e);
 490                         return false;
 491                 }
 492         }
 493
 494
 495
 496         /* (non-Javadoc)
 497          * @see eu.etaxonomy.cdm.io.berlinModel.in.IPartitionedIO#getRelatedObjectsForPartition(java.sql.ResultSet)
 498          */
 499         public Map<Object, Map<String, ? extends CdmBase>> getRelatedObjectsForPartition(ResultSet rs) {
 500                 String nameSpace;
 501                 Class cdmClass;
 502                 Set<String> idSet;
 503
 504                 Map<Object, Map<String, ? extends CdmBase>> result = new HashMap<Object, Map<String, ? extends CdmBase>>();
 505
 506                 try{
 507                         Set<String> teamIdSet = new HashSet<String>();
 508                         Set<String> referenceIdSet = new HashSet<String>();
 509
 510                         while (rs.next()){
 511                                 handleForeignKey(rs, teamIdSet, "NomAuthorTeamFk");
 512                                 handleForeignKey(rs, referenceIdSet, "InRefFk");
 513                                 //TODO only needed in second path but state not available here to check if state is second path
 514                                 handleForeignKey(rs, referenceIdSet, "refId");
 515                         }
 516
 517                         //team map
 518                         nameSpace = BerlinModelAuthorTeamImport.NAMESPACE;
 519                         cdmClass = Team.class;
 520                         idSet = teamIdSet;
 521                         Map<String, Team> teamMap = (Map<String, Team>)getCommonService().getSourcedObjectsByIdInSource(cdmClass, idSet, nameSpace);
 522                         result.put(nameSpace, teamMap);
 523
 524
 525                         //nom reference map
 526                         nameSpace = NOM_REFERENCE_NAMESPACE;
 527                         cdmClass = Reference.class;
 528                         idSet = referenceIdSet;
 529                         Map<String, Reference> nomRefMap = (Map<String, Reference>)getCommonService().getSourcedObjectsByIdInSource(cdmClass, idSet, nameSpace);
 530                         result.put(nameSpace, nomRefMap);
 531
 532                         //biblio reference map
 533                         nameSpace = BIBLIO_REFERENCE_NAMESPACE;
 534                         cdmClass = Reference.class;
 535                         idSet = referenceIdSet;
 536                         Map<String, Reference> biblioRefMap = (Map<String, Reference>)getCommonService().getSourcedObjectsByIdInSource(cdmClass, idSet, nameSpace);
 537                         result.put(nameSpace, biblioRefMap);
 538
 539                 } catch (SQLException e) {
 540                         throw new RuntimeException(e);
 541                 }
 542                 return result;
 543         }
 544
 545
 546         /**
 547          * Handles a single reference record
 548          * @param rs
 549          * @param state
 550          * @param biblioRefToSave
 551          * @param nomRefToSave
 552          * @param relatedBiblioReferences
 553          * @param relatedNomReferences
 554          * @param refCounter
 555          * @return
 556          */
 557         private boolean makeSingleReferenceRecord(
 558                                 ResultSet rs,
 559                                 BerlinModelImportState state,
 560                                 ResultSetPartitioner<BerlinModelImportState> partitioner,
 561                                 Map<Integer, Reference> biblioRefToSave,
 562                                 Map<Integer, Reference> nomRefToSave,
 563                                 Map<String, Reference> relatedBiblioReferences,
 564                                 Map<String, Reference> relatedNomReferences,
 565                                 RefCounter refCounter){
 566                 boolean success = true;
 567
 568                 Integer refId = null;
 569                 try {
 570                         Map<String, Object> valueMap = getValueMap(rs);
 571
 572                         Integer categoryFk = (Integer)valueMap.get("refCategoryFk".toLowerCase());
 573                         refId = (Integer)valueMap.get("refId".toLowerCase());
 574                         Boolean thesisFlag = (Boolean)valueMap.get("thesisFlag".toLowerCase());
 575
 576
 577                         Reference<?> referenceBase;
 578                         logger.debug("RefCategoryFk: " + categoryFk);
 579
 580                         if (thesisFlag){
 581                                 referenceBase = makeThesis(valueMap);
 582                         }else if (categoryFk == REF_JOURNAL){
 583                                 referenceBase = makeJournal(valueMap);
 584                         }else if(categoryFk == REF_BOOK){
 585                                 referenceBase = makeBook(valueMap, biblioRefToSave, nomRefToSave, relatedBiblioReferences, relatedNomReferences);
 586                         }else if(categoryFk == REF_DATABASE){
 587                                 referenceBase = makeDatabase(valueMap);
 588                         }else if(categoryFk == REF_INFORMAL){
 589                                 referenceBase = makeInformal(valueMap);
 590                         }else if(categoryFk == REF_WEBSITE){
 591                                 referenceBase = makeWebSite(valueMap);
 592                         }else if(categoryFk == REF_UNKNOWN){
 593                                 referenceBase = makeUnknown(valueMap);
 594                         }else if(categoryFk == REF_PRINT_SERIES){
 595                                 referenceBase = makePrintSeries(valueMap);
 596                         }else if(categoryFk == REF_CONFERENCE_PROCEEDINGS){
 597                                 referenceBase = makeProceedings(valueMap);
 598                         }else if(categoryFk == REF_ARTICLE){
 599                                 referenceBase = makeArticle(valueMap, biblioRefToSave, nomRefToSave, relatedBiblioReferences, relatedNomReferences);
 600                         }else if(categoryFk == REF_JOURNAL_VOLUME){
 601                                 referenceBase = makeJournalVolume(valueMap);
 602                         }else if(categoryFk == REF_PART_OF_OTHER_TITLE){
 603                                 referenceBase = makePartOfOtherTitle(valueMap, biblioRefToSave, nomRefToSave, relatedBiblioReferences, relatedNomReferences);
 604                         }else{
 605                                 logger.warn("Unknown categoryFk (" + categoryFk + "). Create 'Generic instead'");
 606                                 referenceBase = ReferenceFactory.newGeneric();
 607                                 success = false;
 608                         }
 609
 610                         //refYear
 611                         String refYear = (String)valueMap.get("refYear".toLowerCase());
 612                         referenceBase.setDatePublished(ImportHelper.getDatePublished(refYear));
 613
 614                         //created, updated, notes
 615                         doCreatedUpdatedNotes(state, referenceBase, rs);
 616
 617                         //idInSource
 618                         String idInSource = (String)valueMap.get("IdInSource".toLowerCase());
 619                         if (CdmUtils.isNotEmpty(idInSource)){
 620                                 IdentifiableSource source = IdentifiableSource.NewInstance(idInSource);
 621                                 source.setIdNamespace("import to Berlin Model");
 622                                 referenceBase.addSource(source);
 623                         }
 624
 625                         //nom&BiblioReference  - must be last because a clone is created
 626                         success &= makeNomAndBiblioReference(rs, state, partitioner, refId, referenceBase, refCounter,
 627                                         biblioRefToSave, nomRefToSave );
 628
 629
 630                 } catch (Exception e) {
 631                         logger.warn("Reference with BM refId '" + CdmUtils.Nz(refId) +  "' threw Exception and could not be saved");
 632                         e.printStackTrace();
 633                         success = false;
 634                 }
 635                 return success;
 636         }
 637
 638
 639         /**
 640          * Creates and saves a nom. reference and a biblio. reference after checking necessity
 641          * @param rs
 642          * @param refId
 643          * @param referenceBase
 644          * @param refCounter
 645          * @param biblioRefToSave
 646          * @param nomRefToSave
 647          * @param teamMap
 648          * @param stores
 649          * @return
 650          * @throws SQLException
 651          */
 652         private boolean makeNomAndBiblioReference(
 653                                 ResultSet rs,
 654                                 BerlinModelImportState state,
 655                                 ResultSetPartitioner partitioner,
 656                                 int refId,
 657                                 Reference<?> referenceBase,
 658                                 RefCounter refCounter,
 659                                 Map<Integer, Reference> biblioRefToSave,
 660                                 Map<Integer, Reference> nomRefToSave
 661                                 ) throws SQLException{
 662
 663                 Map<String, Team> teamMap = partitioner.getObjectMap(BerlinModelAuthorTeamImport.NAMESPACE);
 664
 665                 String refCache = rs.getString("refCache");
 666                 String nomRefCache = rs.getString("nomRefCache");
 667                 String title = rs.getString("title");
 668                 String nomTitleAbbrev = rs.getString("nomTitleAbbrev");
 669                 boolean isPreliminary = rs.getBoolean("PreliminaryFlag");
 670                 String refAuthorString = rs.getString("refAuthorString");
 671                 Integer nomAuthorTeamFk = rs.getInt("NomAuthorTeamFk");
 672                 String strNomAuthorTeamFk = String.valueOf(nomAuthorTeamFk);
 673                 TeamOrPersonBase<?> nomAuthor = teamMap.get(strNomAuthorTeamFk);
 674                 Reference nomReference = null;
 675
 676                 boolean hasNomRef = false;
 677                 boolean hasBiblioRef = false;
 678                 Reference sourceReference = state.getConfig().getSourceReference();
 679
 680                 //is Nomenclatural Reference
 681                 if ( (CdmUtils.isNotEmpty(nomRefCache) && isPreliminary) || (CdmUtils.isNotEmpty(nomTitleAbbrev) && ! isPreliminary) ){
 682                         referenceBase.setTitle(nomTitleAbbrev);
 683                         TeamOrPersonBase<?> author = getAuthorTeam(refAuthorString , nomAuthor, true);
 684                         referenceBase.setAuthorTeam(author);
 685                         //referenceBase.setNomenclaturallyRelevant(true);
 686                         if (isPreliminary){
 687                                 referenceBase.setTitleCache(nomRefCache, true);
 688                         }
 689                         if (! nomRefToSave.containsKey(refId)){
 690                                 if (referenceBase == null){
 691                                         logger.warn("refBase is null");
 692                                 }
 693                                 nomRefToSave.put(refId, referenceBase);
 694                         }else{
 695                                 logger.warn("Duplicate refId in Berlin Model database. Second reference was not imported !!");
 696                         }
 697
 698 //                      ???
 699 //                      nomRefToSave.put(refId, referenceBase);
 700                         hasNomRef = true;
 701                         nomReference = referenceBase;
 702                         refCounter.nomRefCount++;
 703                 }
 704                 //is bibliographical Reference
 705                 if ((CdmUtils.isNotEmpty(refCache) && isPreliminary && ! refCache.equalsIgnoreCase(nomRefCache))
 706                                 || (CdmUtils.isNotEmpty(title) && ! isPreliminary && ! title.equalsIgnoreCase(nomTitleAbbrev))
 707                                 || hasNomRef == false){
 708                         if (hasNomRef){
 709                                 referenceBase = (Reference)referenceBase.clone();
 710                                 copyCreatedUpdated(referenceBase, nomReference);
 711                         }
 712                         referenceBase.setTitle(title);
 713                         TeamOrPersonBase author = getAuthorTeam(refAuthorString , nomAuthor, false);
 714                         referenceBase.setAuthorTeam(author);
 715                         referenceBase.setNomenclaturallyRelevant(false);
 716                         if (isPreliminary){
 717                                 referenceBase.setTitleCache(refCache, true);
 718                         }
 719                         if (! biblioRefToSave.containsKey(refId)){
 720                                 biblioRefToSave.put(refId, referenceBase);
 721                         }else{
 722                                 logger.warn("Duplicate refId in Berlin Model database. Second reference was not imported !!");
 723                         }
 724                         hasBiblioRef = true;
 725
 726                         //??
 727                         //biblioRefToSave.put(refId, referenceBase);
 728                         refCounter.referenceCount++;
 729                 }
 730                 //refId
 731                 if (hasNomRef){
 732                         ImportHelper.setOriginalSource(nomReference, sourceReference, refId, NOM_REFERENCE_NAMESPACE);
 733                 }
 734                 if (hasBiblioRef){
 735                         ImportHelper.setOriginalSource(referenceBase, sourceReference, refId, BIBLIO_REFERENCE_NAMESPACE);
 736                 }
 737
 738                 return true;
 739
 740         }
 741
 742         /**
 743          * Copies the created and updated information from the nomReference to the cloned bibliographic reference
 744          * @param referenceBase
 745          * @param nomReference
 746          */
 747         private void copyCreatedUpdated(Reference<?> biblioReference, Reference nomReference) {
 748                 biblioReference.setCreatedBy(nomReference.getCreatedBy());
 749                 biblioReference.setCreated(nomReference.getCreated());
 750                 biblioReference.setUpdatedBy(nomReference.getUpdatedBy());
 751                 biblioReference.setUpdated(nomReference.getUpdated());
 752
 753         }
 754
 755         private Reference<?> makeArticle (Map<String, Object> valueMap, Map<Integer, Reference> biblioRefToSave, Map<Integer, Reference> nomRefToSave, Map<String, Reference> relatedBiblioReferences, Map<String, Reference> relatedNomReferences){
 756
 757                 IArticle article = ReferenceFactory.newArticle();
 758                 Object inRefFk = valueMap.get("inRefFk".toLowerCase());
 759                 Integer inRefCategoryFk = (Integer)valueMap.get("inRefCategoryFk".toLowerCase());
 760                 Integer refId = (Integer)valueMap.get("refId".toLowerCase());
 761
 762                 if (inRefFk != null){
 763                         if (inRefCategoryFk == REF_JOURNAL){
 764                                 int inRefFkInt = (Integer)inRefFk;
 765                                 if (true){
 766                                         //changed for first/second path implementation, if this is successful the following code can be deleted
 767                                 }else if (existsInMapOrToSave(inRefFkInt, biblioRefToSave, nomRefToSave, relatedBiblioReferences, relatedNomReferences)){
 768                                         Reference<?> inJournal = getReferenceFromMaps(inRefFkInt, nomRefToSave, relatedNomReferences);
 769                                         if (inJournal == null){
 770                                                 inJournal = getReferenceFromMaps(inRefFkInt, biblioRefToSave, relatedBiblioReferences);
 771                                                 logger.info("inJournal (" + inRefFkInt + ") found in referenceStore instead of nomRefStore.");
 772                                                 nomRefToSave.put(inRefFkInt, inJournal);
 773                                         }
 774                                         if (inJournal == null){
 775                                                 logger.warn("inJournal for " + inRefFkInt + " is null. "+
 776                                                         " InReference relation could not be set");
 777                                         //}else if (Reference.class.isAssignableFrom(inJournal.getClass())){
 778                                         }else if (inJournal.getType().equals(ReferenceType.Journal)){
 779                                                 article.setInJournal((IJournal)inJournal);
 780                                         }else{
 781                                                 logger.warn("InJournal is not of type journal but of type " + inJournal.getType() +
 782                                                         " Inreference relation could not be set");
 783                                         }
 784                                 }else{
 785                                         logger.error("Journal (refId = " + inRefFkInt + " ) for Article (refID = " + refId +") could not be found in nomRefStore. Inconsistency error. ");
 786                                         //success = false;;
 787                                 }
 788                         }else{
 789                                 logger.warn("Wrong inrefCategory for Article (refID = " + refId +"). Type must be 'Journal' but was not (RefCategoryFk=" + inRefCategoryFk + "))." +
 790                                         " InReference was not added to Article! ");
 791                         }
 792                 }
 793                 makeStandardMapper(valueMap, (Reference)article); //url, pages, series, volume
 794                 return (Reference)article;
 795         }
 796
 797         private Reference<?> makePartOfOtherTitle (Map<String, Object> valueMap, Map<Integer, Reference> biblioRefToSave, Map<Integer, Reference> nomRefToSave, Map<String, Reference> relatedBiblioReferences, Map<String, Reference> relatedNomReferences){
 798                 Reference<?> result;
 799                 Object inRefFk = valueMap.get("inRefFk".toLowerCase());
 800                 Integer inRefCategoryFk = (Integer)valueMap.get("inRefCategoryFk".toLowerCase());
 801                 Integer refId = (Integer)valueMap.get("refId".toLowerCase());
 802
 803                 if (inRefCategoryFk == null){
 804                         //null -> error
 805                         logger.warn("Part-Of-Other-Title has no inRefCategoryFk! RefId = " + refId + ". ReferenceType set to Generic.");
 806                         result = makeUnknown(valueMap);
 807                 }else if (inRefCategoryFk == REF_BOOK){
 808                         //BookSection
 809                         IBookSection bookSection = ReferenceFactory.newBookSection();
 810                         result = (Reference<?>)bookSection;
 811                         if (inRefFk != null && false){   //&& false added for first/second path implementation, following code can be deleted or moved if this is successful
 812                                 int inRefFkInt = (Integer) inRefFk;
 813                                 if (existsInMapOrToSave(inRefFkInt, biblioRefToSave, nomRefToSave, relatedBiblioReferences, relatedNomReferences)){
 814                                         Reference<?> inBook = getReferenceFromMaps(inRefFkInt, nomRefToSave, relatedNomReferences);
 815                                         if (inBook == null){
 816                                                 inBook = getReferenceFromMaps(inRefFkInt, biblioRefToSave, relatedBiblioReferences);
 817                                                 logger.info("inBook (" + inRefFkInt + ") found in referenceStore instead of nomRefStore.");
 818                                                 nomRefToSave.put(inRefFkInt, inBook);
 819                                         }
 820                                         if (inBook == null){
 821                                                 logger.warn("inBook for " + inRefFkInt + " is null. "+
 822                                                 " InReference relation could not be set");;
 823                                         //}else if (Book.class.isAssignableFrom(inBook.getClass())){
 824                                         }else if (inBook.getType().equals(ReferenceType.Book)){
 825                                                 bookSection.setInBook((IBook)inBook);
 826                                                 //TODO
 827                                         }else{
 828                                                 logger.warn("InBook is not of type book but of type " + inBook.getClass().getSimpleName() +
 829                                                                 " Inreference relation could not be set");
 830                                         }
 831                                 }else{
 832                                         logger.error("Book (refId = " + inRefFkInt + ") for part_of_other_title (refID = " + refId +") could not be found in nomRefStore. Inconsistency error. ");
 833                                         //success = false;
 834                                 }
 835                         }
 836                 }else if (inRefCategoryFk == REF_ARTICLE){
 837                         //Article
 838                         //TODO
 839                         logger.warn("Reference (refId = " + refId + ") of type 'part_of_other_title' is part of 'article'." +
 840                                         " This type is not implemented yet. Generic reference created instead") ;
 841                         result = ReferenceFactory.newGeneric();
 842                 }else if (inRefCategoryFk == REF_JOURNAL){
 843                         //TODO
 844                         logger.warn("Reference (refId = " + refId + ") of type 'part_of_other_title' has inReference of type 'journal'." +
 845                                         " This is not allowed! Generic reference created instead") ;
 846                         result = ReferenceFactory.newGeneric();
 847                         result.addMarker(Marker.NewInstance(MarkerType.TO_BE_CHECKED(), true));
 848                 }else{
 849                         logger.warn("InReference type (catFk = " + inRefCategoryFk + ") of part-of-reference not recognized for refId " + refId + "." +
 850                                 " Create 'Generic' reference instead");
 851                         result = ReferenceFactory.newGeneric();
 852                 }
 853                 makeStandardMapper(valueMap, result); //url, pages
 854                 return result;
 855         }
 856
 857
 858         /**
 859          * @param inRefFkInt
 860          * @param biblioRefToSave
 861          * @param nomRefToSave
 862          * @param relatedBiblioReferences
 863          * @param relatedNomReferences
 864          * @return
 865          */
 866         private boolean existsInMapOrToSave(Integer inRefFkInt, Map<Integer, Reference> biblioRefToSave, Map<Integer, Reference> nomRefToSave, Map<String, Reference> relatedBiblioReferences, Map<String, Reference> relatedNomReferences) {
 867                 boolean result = false;
 868                 if (inRefFkInt == null){
 869                         return false;
 870                 }
 871                 result |= nomRefToSave.containsKey(inRefFkInt);
 872                 result |= biblioRefToSave.containsKey(inRefFkInt);
 873                 result |= relatedBiblioReferences.containsKey(String.valueOf(inRefFkInt));
 874                 result |= relatedNomReferences.containsKey(String.valueOf(inRefFkInt));
 875                 return result;
 876         }
 877
 878         private Reference<?> makeWebSite(Map<String, Object> valueMap){
 879                 if (logger.isDebugEnabled()){logger.debug("RefType 'Website'");}
 880                 Reference<?> webPage = ReferenceFactory.newWebPage();
 881                 makeStandardMapper(valueMap, webPage); //placePublished, publisher
 882                 return webPage;
 883         }
 884
 885         private Reference<?> makeUnknown(Map<String, Object> valueMap){
 886                 if (logger.isDebugEnabled()){logger.debug("RefType 'Unknown'");}
 887                 Reference<?> generic = ReferenceFactory.newGeneric();
 888 //              generic.setSeries(series);
 889                 makeStandardMapper(valueMap, generic); //pages, placePublished, publisher, series, volume
 890                 return generic;
 891         }
 892
 893         private Reference<?> makeInformal(Map<String, Object> valueMap){
 894                 if (logger.isDebugEnabled()){logger.debug("RefType 'Informal'");}
 895                 Reference<?> generic = ReferenceFactory.newGeneric();
 896 //              informal.setSeries(series);
 897                 makeStandardMapper(valueMap, generic);//editor, pages, placePublished, publisher, series, volume
 898                 String informal = (String)valueMap.get("InformalRefCategory".toLowerCase());
 899                 if (CdmUtils.isNotEmpty(informal) ){
 900                         generic.addExtension(informal, ExtensionType.INFORMAL_CATEGORY());
 901                 }
 902                 return generic;
 903         }
 904
 905         private Reference<?> makeDatabase(Map<String, Object> valueMap){
 906                 if (logger.isDebugEnabled()){logger.debug("RefType 'Database'");}
 907                 Reference database =  ReferenceFactory.newDatabase();
 908                 makeStandardMapper(valueMap, database); //?
 909                 return database;
 910         }
 911
 912         private Reference<?> makeJournal(Map<String, Object> valueMap){
 913                 if (logger.isDebugEnabled()){logger.debug("RefType 'Journal'");}
 914                 Reference journal = ReferenceFactory.newJournal();
 915
 916                 Set<String> omitAttributes = new HashSet<String>();
 917                 String series = "series";
 918 //              omitAttributes.add(series);
 919
 920                 makeStandardMapper(valueMap, journal, omitAttributes); //issn,placePublished,publisher
 921 //              if (valueMap.get(series) != null){
 922 //                      logger.warn("Series not yet implemented for journal!");
 923 //              }
 924                 return journal;
 925         }
 926
 927         private Reference<?> makeBook(
 928                                 Map<String, Object> valueMap,
 929                                 Map<Integer, Reference> biblioRefToSave,
 930                                 Map<Integer, Reference> nomRefToSave,
 931                                 Map<String, Reference> relatedBiblioReferences,
 932                                 Map<String, Reference> relatedNomReferences){
 933                 if (logger.isDebugEnabled()){logger.debug("RefType 'Book'");}
 934                 Reference<?> book = ReferenceFactory.newBook();
 935                 Integer refId = (Integer)valueMap.get("refId".toLowerCase());
 936
 937                 //Set bookAttributes = new String[]{"edition", "isbn", "pages","publicationTown","publisher","volume"};
 938
 939                 Set<String> omitAttributes = new HashSet<String>();
 940                 String attrSeries = "series";
 941 //              omitAttributes.add(attrSeries);
 942
 943                 makeStandardMapper(valueMap, book, omitAttributes);
 944
 945                 //Series (as String)
 946                 IPrintSeries printSeries = null;
 947                 if (valueMap.get(attrSeries) != null){
 948                         String series = (String)valueMap.get("title".toLowerCase());
 949                         if (series == null){
 950                                 String nomTitle = (String)valueMap.get("nomTitleAbbrev".toLowerCase());
 951                                 series = nomTitle;
 952                         }
 953                         printSeries = ReferenceFactory.newPrintSeries(series);
 954                         logger.info("Implementation of printSeries is preliminary");
 955                 }
 956                 Object inRefFk = valueMap.get("inRefFk".toLowerCase());
 957                 //Series (as Reference)
 958                 if (inRefFk != null && false){  //&&false added for first/second path implementation, following code may be removed if this is successful
 959                         int inRefFkInt = (Integer)inRefFk;
 960                         if (existsInMapOrToSave(inRefFkInt, biblioRefToSave, nomRefToSave, relatedBiblioReferences, relatedNomReferences)){
 961                                 Reference<?> inSeries = getReferenceFromMaps(inRefFkInt, nomRefToSave, relatedNomReferences);
 962                                 if (inSeries == null){
 963                                         inSeries = getReferenceFromMaps(inRefFkInt, biblioRefToSave, relatedBiblioReferences);
 964                                         logger.info("inSeries (" + inRefFkInt + ") found in referenceStore instead of nomRefStore.");
 965                                         nomRefToSave.put(inRefFkInt, inSeries);
 966                                 }
 967                                 if (inSeries == null){
 968                                         logger.warn("inSeries for " + inRefFkInt + " is null. "+
 969                                         " InReference relation could not be set");;
 970                                 //}else if (PrintSeries.class.isAssignableFrom(inSeries.getClass())){
 971                                 }else if (inSeries.getType().equals(ReferenceType.PrintSeries)){
 972                                         book.setInSeries((IPrintSeries)inSeries);
 973                                         //TODO
 974                                 }else{
 975                                         logger.warn("inSeries is not of type PrintSeries but of type " + inSeries.getType().getMessage() +
 976                                                         ". In-reference relation could not be set for refId " + refId + " and inRefFk " + inRefFk);
 977                                 }
 978                         }else{
 979                                 logger.error("PrintSeries (refId = " + inRefFkInt + ") for book (refID = " + refId +") could not be found in nomRefStore. Inconsistency error. ");
 980                                 //success = false;
 981                         }
 982                 }
 983                 if (book.getInSeries() != null && printSeries != null){
 984                         logger.warn("Book has series string and inSeries reference. Can not take both. Series string neglected");
 985                 }else{
 986                         book.setInSeries(printSeries);
 987                 }
 988                 book.setEditor(null);
 989                 return book;
 990
 991         }
 992
 993         /**
 994          * Returns the requested object if it exists in one of both maps. Prefers the refToSaveMap in ambigious cases.
 995          * @param inRefFkInt
 996          * @param nomRefToSave
 997          * @param relatedNomReferences
 998          * @return
 999          */
1000         private Reference<?> getReferenceFromMaps(
1001                         int inRefFkInt,
1002                         Map<Integer, Reference> refToSaveMap,
1003                         Map<String, Reference> relatedRefMap) {
1004                 Reference<?> result = null;
1005                 result = refToSaveMap.get(inRefFkInt);
1006                 if (result == null){
1007                         result = relatedRefMap.get(String.valueOf(inRefFkInt));
1008                 }
1009                 return result;
1010         }
1011
1012         private Reference<?> makePrintSeries(Map<String, Object> valueMap){
1013                 if (logger.isDebugEnabled()){logger.debug("RefType 'PrintSeries'");}
1014                 Reference<?> printSeries = ReferenceFactory.newPrintSeries();
1015                 makeStandardMapper(valueMap, printSeries, null);
1016                 return printSeries;
1017         }
1018
1019         private Reference<?> makeProceedings(Map<String, Object> valueMap){
1020                 if (logger.isDebugEnabled()){logger.debug("RefType 'Proceedings'");}
1021                 Reference<?> proceedings = ReferenceFactory.newProceedings();
1022                 makeStandardMapper(valueMap, proceedings, null);
1023                 return proceedings;
1024         }
1025
1026         private Reference<?> makeThesis(Map<String, Object> valueMap){
1027                 if (logger.isDebugEnabled()){logger.debug("RefType 'Thesis'");}
1028                 Reference<?> thesis = ReferenceFactory.newThesis();
1029                 makeStandardMapper(valueMap, thesis, null);
1030                 return thesis;
1031         }
1032
1033
1034         private Reference<?> makeJournalVolume(Map<String, Object> valueMap){
1035                 if (logger.isDebugEnabled()){logger.debug("RefType 'JournalVolume'");}
1036                 //Proceedings proceedings = Proceedings.NewInstance();
1037                 Reference<?> journalVolume = ReferenceFactory.newGeneric();
1038                 makeStandardMapper(valueMap, journalVolume, null);
1039                 logger.warn("Journal volumes not yet implemented. Generic created instead but with errors");
1040                 return journalVolume;
1041         }
1042
1043         private boolean makeStandardMapper(Map<String, Object> valueMap, Reference<?> ref){
1044                 return makeStandardMapper(valueMap, ref, null);
1045         }
1046
1047
1048         private boolean makeStandardMapper(Map<String, Object> valueMap, CdmBase cdmBase, Set<String> omitAttributes){
1049                 boolean result = true;
1050                 for (CdmAttributeMapperBase mapper : classMappers){
1051                         if (mapper instanceof CdmSingleAttributeMapperBase){
1052                                 result &= makeStandardSingleMapper(valueMap, cdmBase, (CdmSingleAttributeMapperBase)mapper, omitAttributes);
1053                         }else if (mapper instanceof CdmOneToManyMapper){
1054                                 result &= makeMultipleValueAddMapper(valueMap, cdmBase, (CdmOneToManyMapper)mapper, omitAttributes);
1055                         }else{
1056                                 logger.error("Unknown mapper type");
1057                                 result = false;
1058                         }
1059                 }
1060                 return result;
1061         }
1062
1063         private boolean makeStandardSingleMapper(Map<String, Object> valueMap, CdmBase cdmBase, CdmSingleAttributeMapperBase mapper, Set<String> omitAttributes){
1064                 boolean result = true;
1065                 if (omitAttributes == null){
1066                         omitAttributes = new HashSet<String>();
1067                 }
1068                 if (mapper instanceof DbImportExtensionMapper){
1069                         result &= ((DbImportExtensionMapper)mapper).invoke(valueMap, cdmBase);
1070                 }else if (mapper instanceof DbImportMarkerMapper){
1071                         result &= ((DbImportMarkerMapper)mapper).invoke(valueMap, cdmBase);
1072                 }else{
1073                         String sourceAttribute = mapper.getSourceAttributeList().get(0).toLowerCase();
1074                         Object value = valueMap.get(sourceAttribute);
1075                         if (mapper instanceof CdmUriMapper && value != null){
1076                                 try {
1077                                         value = new URI (value.toString());
1078                                 } catch (URISyntaxException e) {
1079                                         logger.error("URI syntax exception: " + value.toString());
1080                                         value = null;
1081                                 }
1082                         }
1083                         if (value != null){
1084                                 String destinationAttribute = mapper.getDestinationAttribute();
1085                                 if (! omitAttributes.contains(destinationAttribute)){
1086                                         result &= ImportHelper.addValue(value, cdmBase, destinationAttribute, mapper.getTypeClass(), OVERWRITE, OBLIGATORY);
1087                                 }
1088                         }
1089                 }
1090                 return result;
1091         }
1092
1093
1094         private boolean makeMultipleValueAddMapper(Map<String, Object> valueMap, CdmBase cdmBase, CdmOneToManyMapper<CdmBase, CdmBase, CdmSingleAttributeMapperBase> mapper, Set<String> omitAttributes){
1095                 if (omitAttributes == null){
1096                         omitAttributes = new HashSet<String>();
1097                 }
1098                 boolean result = true;
1099                 String destinationAttribute = mapper.getSingleAttributeName();
1100                 List<Object> sourceValues = new ArrayList<Object>();
1101                 List<Class> classes = new ArrayList<Class>();
1102                 for (CdmSingleAttributeMapperBase singleMapper : mapper.getSingleMappers()){
1103                         String sourceAttribute = singleMapper.getSourceAttribute();
1104                         Object value = valueMap.get(sourceAttribute);
1105                         sourceValues.add(value);
1106                         Class<?> clazz = singleMapper.getTypeClass();
1107                         classes.add(clazz);
1108                 }
1109
1110                 result &= ImportHelper.addMultipleValues(sourceValues, cdmBase, destinationAttribute, classes, NO_OVERWRITE, OBLIGATORY);
1111 //              //only for testing
1112 //              if (cdmBase instanceof PublicationBase){
1113 //                      PublicationBase pub = ((PublicationBase)cdmBase);
1114 //                      pub.addPublisher("A new publisher for " + pub.getTitleCache(), "A nice place");
1115 //              }
1116                 return result;
1117         }
1118
1119
1120         private static TeamOrPersonBase<?> getAuthorTeam(String authorString, TeamOrPersonBase<?> nomAuthor, boolean preferNomeclaturalAuthor){
1121                 TeamOrPersonBase<?> result;
1122                 if (preferNomeclaturalAuthor){
1123                         if (nomAuthor != null){
1124                                 result = nomAuthor;
1125                         }else{
1126                                 if (CdmUtils.isEmpty(authorString)){
1127                                         result = null;
1128                                 }else{
1129                                         TeamOrPersonBase<?> team = Team.NewInstance();
1130                                         //TODO which one to use??
1131                                         team.setNomenclaturalTitle(authorString);
1132                                         team.setTitleCache(authorString, true);
1133                                         result = team;
1134                                 }
1135                         }
1136                 }else{ //prefer bibliographic
1137                         if (CdmUtils.isNotEmpty(authorString)){
1138                                 TeamOrPersonBase<?> team = Team.NewInstance();
1139                                 //TODO which one to use??
1140                                 team.setNomenclaturalTitle(authorString);
1141                                 team.setTitleCache(authorString, true);
1142                                 result = team;
1143                         }else{
1144                                 result = nomAuthor;
1145                         }
1146                 }
1147                 return result;
1148         }
1149
1150
1151         /**
1152          * @param lowerCase
1153          * @param config
1154          * @return
1155          */
1156         public Set<String> getObligatoryAttributes(boolean lowerCase, BerlinModelImportConfigurator config){
1157                 Set<String> result = new HashSet<String>();
1158                 Class<ICdmIO>[] ioClassList = config.getIoClassList();
1159                 logger.warn("getObligatoryAttributes has been commented because it still needs to be adapted to the new package structure");
1160                 result.addAll(Arrays.asList(unclearMappers));
1161                 result.addAll(Arrays.asList(createdAndNotesAttributes));
1162                 result.addAll(Arrays.asList(operationalAttributes));
1163                 CdmIoMapping mapping = new CdmIoMapping();
1164                 for (CdmAttributeMapperBase mapper : classMappers){
1165                         mapping.addMapper(mapper);
1166                 }
1167                 result.addAll(mapping.getSourceAttributes());
1168                 if (lowerCase){
1169                         Set<String> lowerCaseResult = new HashSet<String>();
1170                         for (String str : result){
1171                                 if (str != null){lowerCaseResult.add(str.toLowerCase());}
1172                         }
1173                         result = lowerCaseResult;
1174                 }
1175                 return result;
1176         }
1177
1178         /* (non-Javadoc)
1179          * @see eu.etaxonomy.cdm.io.common.CdmIoBase#doCheck(eu.etaxonomy.cdm.io.common.IoStateBase)
1180          */
1181         @Override
1182         protected boolean doCheck(BerlinModelImportState state){
1183                 BerlinModelReferenceImportValidator validator = new BerlinModelReferenceImportValidator();
1184                 return validator.validate(state, this);
1185         }
1186
1187
1188         /* (non-Javadoc)
1189          * @see eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelImportBase#getTableName()
1190          */
1191         @Override
1192         protected String getTableName() {
1193                 return dbTableName;
1194         }
1195
1196         /* (non-Javadoc)
1197          * @see eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelImportBase#getPluralString()
1198          */
1199         @Override
1200         public String getPluralString() {
1201                 return pluralString;
1202         }
1203
1204         /* (non-Javadoc)
1205          * @see eu.etaxonomy.cdm.io.common.CdmIoBase#isIgnore(eu.etaxonomy.cdm.io.common.IImportConfigurator)
1206          */
1207         protected boolean isIgnore(BerlinModelImportState state){
1208                 return (state.getConfig().getDoReferences() == IImportConfigurator.DO_REFERENCES.NONE);
1209         }
1210
1211
1212
1213
1214 }