cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/berlinModel/in/BerlinModelReferenceImport.java

   1 /**
   2 * Copyright (C) 2007 EDIT
   3 * European Distributed Institute of Taxonomy
   4 * http://www.e-taxonomy.eu
   5 *
   6 * The contents of this file are subject to the Mozilla Public License Version 1.1
   7 * See LICENSE.TXT at the top of this package for the full license terms.
   8 */
   9
  10 package eu.etaxonomy.cdm.io.berlinModel.in;
  11
  12 import static eu.etaxonomy.cdm.io.berlinModel.BerlinModelTransformer.REF_ARTICLE;
  13 import static eu.etaxonomy.cdm.io.berlinModel.BerlinModelTransformer.REF_BOOK;
  14 import static eu.etaxonomy.cdm.io.berlinModel.BerlinModelTransformer.REF_CONFERENCE_PROCEEDINGS;
  15 import static eu.etaxonomy.cdm.io.berlinModel.BerlinModelTransformer.REF_DATABASE;
  16 import static eu.etaxonomy.cdm.io.berlinModel.BerlinModelTransformer.REF_INFORMAL;
  17 import static eu.etaxonomy.cdm.io.berlinModel.BerlinModelTransformer.REF_JOURNAL;
  18 import static eu.etaxonomy.cdm.io.berlinModel.BerlinModelTransformer.REF_JOURNAL_VOLUME;
  19 import static eu.etaxonomy.cdm.io.berlinModel.BerlinModelTransformer.REF_PART_OF_OTHER_TITLE;
  20 import static eu.etaxonomy.cdm.io.berlinModel.BerlinModelTransformer.REF_PRINT_SERIES;
  21 import static eu.etaxonomy.cdm.io.berlinModel.BerlinModelTransformer.REF_UNKNOWN;
  22 import static eu.etaxonomy.cdm.io.berlinModel.BerlinModelTransformer.REF_WEBSITE;
  23 import static eu.etaxonomy.cdm.io.common.IImportConfigurator.DO_REFERENCES.ALL;
  24 import static eu.etaxonomy.cdm.io.common.IImportConfigurator.DO_REFERENCES.CONCEPT_REFERENCES;
  25 import static eu.etaxonomy.cdm.io.common.IImportConfigurator.DO_REFERENCES.NOMENCLATURAL;
  26 import static eu.etaxonomy.cdm.io.common.ImportHelper.NO_OVERWRITE;
  27 import static eu.etaxonomy.cdm.io.common.ImportHelper.OBLIGATORY;
  28 import static eu.etaxonomy.cdm.io.common.ImportHelper.OVERWRITE;
  29
  30 import java.net.URI;
  31 import java.net.URISyntaxException;
  32 import java.sql.ResultSet;
  33 import java.sql.SQLException;
  34 import java.util.ArrayList;
  35 import java.util.Arrays;
  36 import java.util.HashMap;
  37 import java.util.HashSet;
  38 import java.util.List;
  39 import java.util.Map;
  40 import java.util.Set;
  41 import java.util.UUID;
  42
  43 import org.apache.log4j.Logger;
  44 import org.springframework.stereotype.Component;
  45
  46 import eu.etaxonomy.cdm.common.CdmUtils;
  47 import eu.etaxonomy.cdm.io.berlinModel.CdmOneToManyMapper;
  48 import eu.etaxonomy.cdm.io.berlinModel.CdmStringMapper;
  49 import eu.etaxonomy.cdm.io.berlinModel.CdmUriMapper;
  50 import eu.etaxonomy.cdm.io.berlinModel.in.validation.BerlinModelReferenceImportValidator;
  51 import eu.etaxonomy.cdm.io.common.ICdmIO;
  52 import eu.etaxonomy.cdm.io.common.IImportConfigurator;
  53 import eu.etaxonomy.cdm.io.common.ImportHelper;
  54 import eu.etaxonomy.cdm.io.common.ResultSetPartitioner;
  55 import eu.etaxonomy.cdm.io.common.Source;
  56 import eu.etaxonomy.cdm.io.common.mapping.CdmAttributeMapperBase;
  57 import eu.etaxonomy.cdm.io.common.mapping.CdmIoMapping;
  58 import eu.etaxonomy.cdm.io.common.mapping.CdmSingleAttributeMapperBase;
  59 import eu.etaxonomy.cdm.io.common.mapping.DbImportExtensionMapper;
  60 import eu.etaxonomy.cdm.io.common.mapping.DbImportMarkerMapper;
  61 import eu.etaxonomy.cdm.model.agent.Team;
  62 import eu.etaxonomy.cdm.model.agent.TeamOrPersonBase;
  63 import eu.etaxonomy.cdm.model.common.CdmBase;
  64 import eu.etaxonomy.cdm.model.common.ExtensionType;
  65 import eu.etaxonomy.cdm.model.common.IdentifiableSource;
  66 import eu.etaxonomy.cdm.model.common.Marker;
  67 import eu.etaxonomy.cdm.model.common.MarkerType;
  68 import eu.etaxonomy.cdm.model.reference.IArticle;
  69 import eu.etaxonomy.cdm.model.reference.IBook;
  70 import eu.etaxonomy.cdm.model.reference.IBookSection;
  71 import eu.etaxonomy.cdm.model.reference.IJournal;
  72 import eu.etaxonomy.cdm.model.reference.IPrintSeries;
  73 import eu.etaxonomy.cdm.model.reference.Reference;
  74 import eu.etaxonomy.cdm.model.reference.ReferenceFactory;
  75 import eu.etaxonomy.cdm.model.reference.ReferenceType;
  76
  77 /**
  78  * @author a.mueller
  79  * @created 20.03.2008
  80  * @version 1.0
  81  */
  82 @Component
  83 public class BerlinModelReferenceImport extends BerlinModelImportBase {
  84         private static final Logger logger = Logger.getLogger(BerlinModelReferenceImport.class);
  85
  86         public static final String NOM_REFERENCE_NAMESPACE = "NomReference";
  87         public static final String BIBLIO_REFERENCE_NAMESPACE = "BiblioReference";
  88
  89         public static final UUID REF_DEPOSITED_AT_UUID = UUID.fromString("23ca88c7-ce73-41b2-8ca3-2cb22f013beb");
  90         public static final UUID REF_SOURCE_UUID = UUID.fromString("d6432582-2216-4b08-b0db-76f6c1013141");
  91         public static final UUID DATE_STRING_UUID = UUID.fromString("e4130eae-606e-4b0c-be4f-e93dc161be7d");
  92         public static final UUID IS_PAPER_UUID = UUID.fromString("8a326129-d0d0-4f9d-bbdf-8d86b037c65e");
  93
  94
  95         private int modCount = 1000;
  96         private static final String pluralString = "references";
  97         private static final String dbTableName = "reference";
  98
  99
 100         public BerlinModelReferenceImport(){
 101                 super();
 102         }
 103
 104         protected void initializeMappers(BerlinModelImportState state){
 105                 for (CdmAttributeMapperBase mapper: classMappers){
 106                         if (mapper instanceof DbImportExtensionMapper){
 107                                 ((DbImportExtensionMapper)mapper).initialize(state, Reference.class);
 108                         }
 109                 }
 110                 return;
 111         }
 112
 113         protected static CdmAttributeMapperBase[] classMappers = new CdmAttributeMapperBase[]{
 114                 new CdmStringMapper("edition", "edition"),
 115                 new CdmStringMapper("volume", "volume"),
 116                 new CdmStringMapper("publisher", "publisher"),
 117                 new CdmStringMapper("publicationTown", "placePublished"),
 118                 new CdmStringMapper("isbn", "isbn"),
 119                 new CdmStringMapper("isbn", "isbn"),
 120                 new CdmStringMapper("pageString", "pages"),
 121                 new CdmStringMapper("series", "series"),
 122                 new CdmStringMapper("issn", "issn"),
 123                 new CdmUriMapper("url", "uri"),
 124                 DbImportExtensionMapper.NewInstance("NomStandard", ExtensionType.NOMENCLATURAL_STANDARD()),
 125                 DbImportExtensionMapper.NewInstance("DateString", DATE_STRING_UUID, "Date String", "Date String", "dates"),
 126                 DbImportExtensionMapper.NewInstance("RefDepositedAt", REF_DEPOSITED_AT_UUID, "RefDepositedAt", "reference is deposited at", "at"),
 127                 DbImportExtensionMapper.NewInstance("RefSource", REF_SOURCE_UUID, "RefSource", "reference source", "source"),
 128                 DbImportMarkerMapper.NewInstance("isPaper", IS_PAPER_UUID, "is paper", "is paper", "paper", false)
 129         };
 130
 131
 132         protected static String[] operationalAttributes = new String[]{
 133                 "refId", "refCache", "nomRefCache", "preliminaryFlag", "inRefFk", "title", "nomTitleAbbrev",
 134                 "refAuthorString", "nomAuthorTeamFk",
 135                 "refCategoryFk", "thesisFlag", "informalRefCategory", "idInSource"
 136         };
 137
 138         protected static String[] createdAndNotesAttributes = new String[]{
 139                         "created_When", "updated_When", "created_Who", "updated_Who", "notes"
 140         };
 141
 142         protected static String[] unclearMappers = new String[]{
 143                         /*"isPaper",*/ "exportDate",
 144         };
 145
 146         //TODO isPaper
 147         //
 148
 149
 150
 151         //type to count the references nomReferences that have been created and saved
 152         private class RefCounter{
 153                 RefCounter() {nomRefCount = 0; referenceCount = 0;};
 154                 int nomRefCount;
 155                 int referenceCount;
 156                 public String toString(){return String.valueOf(nomRefCount) + "," +String.valueOf(referenceCount);};
 157         }
 158
 159
 160         /* (non-Javadoc)
 161          * @see eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelImportBase#getRecordQuery(eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelImportConfigurator)
 162          */
 163         @Override
 164         protected String getRecordQuery(BerlinModelImportConfigurator config) {
 165                 return null;  //not needed
 166         }
 167
 168         @Override
 169         protected void doInvoke(BerlinModelImportState state){
 170                 logger.info("start make " + getPluralString() + " ...");
 171
 172                 boolean success = true;
 173                 initializeMappers(state);
 174                 BerlinModelImportConfigurator config = state.getConfig();
 175                 Source source = config.getSource();
 176
 177                 String strSelectId = " SELECT Reference.RefId as refId ";
 178                 String strSelectFull =
 179                         " SELECT Reference.* ,InReference.RefCategoryFk as InRefCategoryFk, RefSource.RefSource " ;
 180                 String strFrom =  " FROM %s  " +
 181                         " LEFT OUTER JOIN Reference as InReference ON InReference.refId = Reference.inRefFk " +
 182                                 " LEFT OUTER JOIN RefSource ON Reference.RefSourceFk = RefSource.RefSourceId " +
 183                         " WHERE (1=1) ";
 184                 String strWherePartitioned = " AND (Reference.refId IN ("+ ID_LIST_TOKEN + ") ) ";
 185
 186                 String referenceTable = CdmUtils.Nz(state.getConfig().getReferenceIdTable());
 187                 referenceTable = referenceTable.isEmpty() ? " Reference"  : referenceTable + " as Reference ";
 188                 String strIdFrom = String.format(strFrom, referenceTable );
 189
 190                 String strSelectIdBase = strSelectId + strIdFrom;
 191
 192                 String referenceFilter = CdmUtils.Nz(state.getConfig().getReferenceIdTable());
 193                 if (! referenceFilter.isEmpty()){
 194                         referenceFilter = " AND " + referenceFilter + " ";
 195                 }
 196                 referenceFilter = "";  //don't use it for now
 197
 198                 String strIdQueryFirstPath = strSelectId + strIdFrom ;
 199                 String strIdQuerySecondPath = strSelectId + strIdFrom + " AND (Reference.InRefFk is NOT NULL) ";
 200
 201 //              if (config.getDoReferences() == CONCEPT_REFERENCES){
 202 //                      strIdQueryNoInRef += " AND ( Reference.refId IN ( SELECT ptRefFk FROM PTaxon) ) " + referenceFilter;
 203 //              }
 204
 205                 String strRecordQuery = strSelectFull + String.format(strFrom, " Reference ") + strWherePartitioned;
 206
 207                 int recordsPerTransaction = config.getRecordsPerTransaction();
 208                 try{
 209                         //firstPath
 210                         ResultSetPartitioner partitioner = ResultSetPartitioner.NewInstance(source, strIdQueryFirstPath, strRecordQuery, recordsPerTransaction);
 211                         while (partitioner.nextPartition()){
 212                                 partitioner.doPartition(this, state);
 213                         }
 214                         logger.info("end make references without in-references ... " + getSuccessString(success));
 215                         state.setReferenceSecondPath(true);
 216
 217 //                      if (config.getDoReferences() == ALL || config.getDoReferences() == NOMENCLATURAL){
 218
 219                         //secondPath
 220                         partitioner = ResultSetPartitioner.NewInstance(source, strIdQuerySecondPath, strRecordQuery, recordsPerTransaction);
 221                         while (partitioner.nextPartition()){
 222                                 partitioner.doPartition(this, state);
 223                         }
 224                         logger.info("end make references with no 1 in-reference ... " + getSuccessString(success));
 225                         state.setReferenceSecondPath(false);
 226
 227 //                      }
 228
 229                 } catch (SQLException e) {
 230                         logger.error("SQLException:" +  e);
 231                         state.setUnsuccessfull();
 232                         return;
 233                 }
 234                 logger.info("end make " + getPluralString() + " ... " + getSuccessString(success));
 235                 if (! success){
 236                         state.setUnsuccessfull();
 237                 }
 238                 return;
 239         }
 240
 241
 242
 243
 244         /* (non-Javadoc)
 245          * @see eu.etaxonomy.cdm.io.berlinModel.in.IPartitionedIO#doPartition(eu.etaxonomy.cdm.io.berlinModel.in.ResultSetPartitioner, eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelImportState)
 246          */
 247         public boolean doPartition(ResultSetPartitioner partitioner, BerlinModelImportState state) {
 248                 if (state.isReferenceSecondPath()){
 249                         return doPartitionSecondPath(partitioner, state);
 250                 }
 251                 boolean success = true;
 252
 253                 Map<Integer, Reference> nomRefToSave = new HashMap<Integer, Reference>();
 254                 Map<Integer, Reference> biblioRefToSave = new HashMap<Integer, Reference>();
 255
 256                 Map<String, Reference> relatedNomReferences = partitioner.getObjectMap(NOM_REFERENCE_NAMESPACE);
 257                 Map<String, Reference> relatedBiblioReferences = partitioner.getObjectMap(BIBLIO_REFERENCE_NAMESPACE);
 258
 259                 BerlinModelImportConfigurator config = state.getConfig();
 260
 261                 try {
 262
 263                                 int i = 0;
 264                                 RefCounter refCounter  = new RefCounter();
 265
 266                                 ResultSet rs = partitioner.getResultSet();
 267
 268                                 //for each resultset
 269                                 while (rs.next()){
 270                                         if ((i++ % modCount) == 0 && i!= 1 ){ logger.info("References handled: " + (i-1) + " in round -" );}
 271
 272                                         success &= makeSingleReferenceRecord(rs, state, partitioner, biblioRefToSave, nomRefToSave, relatedBiblioReferences, relatedNomReferences, refCounter);
 273                                 } // end resultSet
 274
 275                                 //for the concept reference a fixed uuid may be needed -> change uuid
 276                                 Integer sourceSecId = (Integer)config.getSourceSecId();
 277                                 Reference<?> sec = biblioRefToSave.get(sourceSecId);
 278                                 if (sec == null){
 279                                         sec = nomRefToSave.get(sourceSecId);
 280                                 }
 281                                 if (sec != null){
 282                                         sec.setUuid(config.getSecUuid());
 283                                         logger.info("SecUuid changed to: " + config.getSecUuid());
 284                                 }
 285
 286                                 //save and store in map
 287                                 logger.info("Save nomenclatural references (" + refCounter.nomRefCount + ")");
 288                                 getReferenceService().saveOrUpdate(nomRefToSave.values());
 289                                 logger.info("Save bibliographical references (" + refCounter.referenceCount +")");
 290                                 getReferenceService().saveOrUpdate(biblioRefToSave.values());
 291
 292 //                      }//end resultSetList
 293
 294                         logger.info("end makeReferences ..." + getSuccessString(success));;
 295                         return success;
 296                 } catch (SQLException e) {
 297                         logger.error("SQLException:" +  e);
 298                         return false;
 299                 }
 300         }
 301
 302
 303
 304         /**
 305          * Adds the inReference to the according references.
 306          * @param partitioner
 307          * @param state
 308          * @return
 309          */
 310         private boolean doPartitionSecondPath(ResultSetPartitioner partitioner, BerlinModelImportState state) {
 311                 boolean success = true;
 312
 313                 Map<Integer, Reference> nomRefToSave = new HashMap<Integer, Reference>();
 314                 Map<Integer, Reference> biblioRefToSave = new HashMap<Integer, Reference>();
 315
 316                 Map<String, Reference> relatedNomReferences = partitioner.getObjectMap(NOM_REFERENCE_NAMESPACE);
 317                 Map<String, Reference> relatedBiblioReferences = partitioner.getObjectMap(BIBLIO_REFERENCE_NAMESPACE);
 318
 319                 try {
 320                                 int i = 0;
 321                                 RefCounter refCounter  = new RefCounter();
 322
 323                                 ResultSet rs = partitioner.getResultSet();
 324                                 //for each resultset
 325                                 while (rs.next()){
 326                                         if ((i++ % modCount) == 0 && i!= 1 ){ logger.info("References handled: " + (i-1) + " in round -" );}
 327
 328                                         Integer refId = rs.getInt("refId");
 329                                         Integer inRefFk = rs.getInt("inRefFk");
 330
 331                                         if (inRefFk != null){
 332
 333                                                 Reference<?> thisNomRef = getReferenceOnlyFromMaps(relatedNomReferences, relatedBiblioReferences, String.valueOf(refId));
 334                                                 Reference<?> thisBiblioRef = getReferenceOnlyFromMaps(relatedBiblioReferences, relatedNomReferences, String.valueOf(refId));
 335
 336                                                 Reference<?> nomInReference = relatedNomReferences.get(String.valueOf(inRefFk));
 337                                                 Reference<?> biblioInReference = relatedBiblioReferences.get(String.valueOf(inRefFk));
 338                                                 boolean inRefExists = false;
 339                                                 if (thisNomRef != null){
 340                                                         Reference<?> inRef = (nomInReference != null)? nomInReference : biblioInReference;
 341                                                         if (inRef == null){
 342                                                                 logger.warn("No InRef found for nomRef: " + thisNomRef.getTitleCache() + "; RefId: " +  refId + "; inRefFK: " +  inRefFk);
 343                                                         }
 344                                                         thisNomRef.setInReference(inRef);
 345                                                         nomRefToSave.put(refId, thisNomRef);
 346                                                         //remember that an in reference exists
 347                                                         inRefExists |= (inRef != null);
 348                                                         thisNomRef.setTitleCache(null);
 349                                                         thisNomRef.getTitleCache();
 350                                                 }
 351                                                 if (thisBiblioRef != null){
 352                                                         Reference<?> inRef = (biblioInReference != null)? biblioInReference : nomInReference ;
 353                                                         if (inRef == null){
 354                                                                 logger.warn("No InRef found for biblioRef: " + thisBiblioRef.getTitleCache() + "; RefId: " +  refId + "; inRefFK: " +  inRefFk);
 355                                                         }
 356                                                         thisBiblioRef.setInReference(inRef);
 357                                                         biblioRefToSave.put(refId, thisBiblioRef);
 358                                                         //remember that an in reference exists
 359                                                         inRefExists |= (inRef != null);
 360                                                         thisBiblioRef.setTitleCache(null);
 361                                                         thisBiblioRef.getTitleCache();
 362                                                 }
 363                                                 if (inRefExists == false){
 364                                                         logger.warn("No in reference was saved though an 'inRefFk' is available. RefId " + refId);
 365                                                 }
 366
 367                                         }
 368
 369                                 } // end resultSet
 370
 371                                 //save and store in map
 372                                 logger.info("Save nomenclatural references (" + refCounter.nomRefCount + ")");
 373                                 getReferenceService().saveOrUpdate(nomRefToSave.values());
 374                                 logger.info("Save bibliographical references (" + refCounter.referenceCount +")");
 375                                 getReferenceService().saveOrUpdate(biblioRefToSave.values());
 376
 377 //                      }//end resultSetList
 378
 379                         logger.info("end makeReferences ..." + getSuccessString(success));;
 380                         return success;
 381                 } catch (SQLException e) {
 382                         logger.error("SQLException:" +  e);
 383                         return false;
 384                 }
 385         }
 386
 387
 388
 389         /* (non-Javadoc)
 390          * @see eu.etaxonomy.cdm.io.berlinModel.in.IPartitionedIO#getRelatedObjectsForPartition(java.sql.ResultSet)
 391          */
 392         public Map<Object, Map<String, ? extends CdmBase>> getRelatedObjectsForPartition(ResultSet rs) {
 393                 String nameSpace;
 394                 Class cdmClass;
 395                 Set<String> idSet;
 396
 397                 Map<Object, Map<String, ? extends CdmBase>> result = new HashMap<Object, Map<String, ? extends CdmBase>>();
 398
 399                 try{
 400                         Set<String> teamIdSet = new HashSet<String>();
 401                         Set<String> referenceIdSet = new HashSet<String>();
 402
 403                         while (rs.next()){
 404                                 handleForeignKey(rs, teamIdSet, "NomAuthorTeamFk");
 405                                 handleForeignKey(rs, referenceIdSet, "InRefFk");
 406                                 //TODO only needed in second path but state not available here to check if state is second path
 407                                 handleForeignKey(rs, referenceIdSet, "refId");
 408                         }
 409
 410                         //team map
 411                         nameSpace = BerlinModelAuthorTeamImport.NAMESPACE;
 412                         cdmClass = Team.class;
 413                         idSet = teamIdSet;
 414                         Map<String, Team> teamMap = (Map<String, Team>)getCommonService().getSourcedObjectsByIdInSource(cdmClass, idSet, nameSpace);
 415                         result.put(nameSpace, teamMap);
 416
 417
 418                         //nom reference map
 419                         nameSpace = NOM_REFERENCE_NAMESPACE;
 420                         cdmClass = Reference.class;
 421                         idSet = referenceIdSet;
 422                         Map<String, Reference> nomRefMap = (Map<String, Reference>)getCommonService().getSourcedObjectsByIdInSource(cdmClass, idSet, nameSpace);
 423                         result.put(nameSpace, nomRefMap);
 424
 425                         //biblio reference map
 426                         nameSpace = BIBLIO_REFERENCE_NAMESPACE;
 427                         cdmClass = Reference.class;
 428                         idSet = referenceIdSet;
 429                         Map<String, Reference> biblioRefMap = (Map<String, Reference>)getCommonService().getSourcedObjectsByIdInSource(cdmClass, idSet, nameSpace);
 430                         result.put(nameSpace, biblioRefMap);
 431
 432                 } catch (SQLException e) {
 433                         throw new RuntimeException(e);
 434                 }
 435                 return result;
 436         }
 437
 438
 439         /**
 440          * Handles a single reference record
 441          * @param rs
 442          * @param state
 443          * @param biblioRefToSave
 444          * @param nomRefToSave
 445          * @param relatedBiblioReferences
 446          * @param relatedNomReferences
 447          * @param refCounter
 448          * @return
 449          */
 450         private boolean makeSingleReferenceRecord(
 451                                 ResultSet rs,
 452                                 BerlinModelImportState state,
 453                                 ResultSetPartitioner<BerlinModelImportState> partitioner,
 454                                 Map<Integer, Reference> biblioRefToSave,
 455                                 Map<Integer, Reference> nomRefToSave,
 456                                 Map<String, Reference> relatedBiblioReferences,
 457                                 Map<String, Reference> relatedNomReferences,
 458                                 RefCounter refCounter){
 459                 boolean success = true;
 460
 461                 Integer refId = null;
 462                 try {
 463                         Map<String, Object> valueMap = getValueMap(rs);
 464
 465                         Integer categoryFk = (Integer)valueMap.get("refCategoryFk".toLowerCase());
 466                         refId = (Integer)valueMap.get("refId".toLowerCase());
 467                         Boolean thesisFlag = (Boolean)valueMap.get("thesisFlag".toLowerCase());
 468
 469
 470                         Reference<?> referenceBase;
 471                         logger.debug("RefCategoryFk: " + categoryFk);
 472
 473                         if (thesisFlag){
 474                                 referenceBase = makeThesis(valueMap);
 475                         }else if (categoryFk == REF_JOURNAL){
 476                                 referenceBase = makeJournal(valueMap);
 477                         }else if(categoryFk == REF_BOOK){
 478                                 referenceBase = makeBook(valueMap, biblioRefToSave, nomRefToSave, relatedBiblioReferences, relatedNomReferences);
 479                         }else if(categoryFk == REF_DATABASE){
 480                                 referenceBase = makeDatabase(valueMap);
 481                         }else if(categoryFk == REF_INFORMAL){
 482                                 referenceBase = makeInformal(valueMap);
 483                         }else if(categoryFk == REF_WEBSITE){
 484                                 referenceBase = makeWebSite(valueMap);
 485                         }else if(categoryFk == REF_UNKNOWN){
 486                                 referenceBase = makeUnknown(valueMap);
 487                         }else if(categoryFk == REF_PRINT_SERIES){
 488                                 referenceBase = makePrintSeries(valueMap);
 489                         }else if(categoryFk == REF_CONFERENCE_PROCEEDINGS){
 490                                 referenceBase = makeProceedings(valueMap);
 491                         }else if(categoryFk == REF_ARTICLE){
 492                                 referenceBase = makeArticle(valueMap, biblioRefToSave, nomRefToSave, relatedBiblioReferences, relatedNomReferences);
 493                         }else if(categoryFk == REF_JOURNAL_VOLUME){
 494                                 referenceBase = makeJournalVolume(valueMap);
 495                         }else if(categoryFk == REF_PART_OF_OTHER_TITLE){
 496                                 referenceBase = makePartOfOtherTitle(valueMap, biblioRefToSave, nomRefToSave, relatedBiblioReferences, relatedNomReferences);
 497                         }else{
 498                                 logger.warn("Unknown categoryFk (" + categoryFk + "). Create 'Generic instead'");
 499                                 referenceBase = ReferenceFactory.newGeneric();
 500                                 success = false;
 501                         }
 502
 503                         //refYear
 504                         String refYear = (String)valueMap.get("refYear".toLowerCase());
 505                         referenceBase.setDatePublished(ImportHelper.getDatePublished(refYear));
 506
 507                         //created, updated, notes
 508                         doCreatedUpdatedNotes(state, referenceBase, rs);
 509
 510                         //idInSource
 511                         String idInSource = (String)valueMap.get("IdInSource".toLowerCase());
 512                         if (CdmUtils.isNotEmpty(idInSource)){
 513                                 IdentifiableSource source = IdentifiableSource.NewInstance(idInSource);
 514                                 source.setIdNamespace("import to Berlin Model");
 515                                 referenceBase.addSource(source);
 516                         }
 517
 518                         //nom&BiblioReference  - must be last because a clone is created
 519                         success &= makeNomAndBiblioReference(rs, state, partitioner, refId, referenceBase, refCounter,
 520                                         biblioRefToSave, nomRefToSave );
 521
 522
 523                 } catch (Exception e) {
 524                         logger.warn("Reference with BM refId '" + CdmUtils.Nz(refId) +  "' threw Exception and could not be saved");
 525                         e.printStackTrace();
 526                         success = false;
 527                 }
 528                 return success;
 529         }
 530
 531
 532         /**
 533          * Creates and saves a nom. reference and a biblio. reference after checking necessity
 534          * @param rs
 535          * @param refId
 536          * @param referenceBase
 537          * @param refCounter
 538          * @param biblioRefToSave
 539          * @param nomRefToSave
 540          * @param teamMap
 541          * @param stores
 542          * @return
 543          * @throws SQLException
 544          */
 545         private boolean makeNomAndBiblioReference(
 546                                 ResultSet rs,
 547                                 BerlinModelImportState state,
 548                                 ResultSetPartitioner partitioner,
 549                                 int refId,
 550                                 Reference<?> referenceBase,
 551                                 RefCounter refCounter,
 552                                 Map<Integer, Reference> biblioRefToSave,
 553                                 Map<Integer, Reference> nomRefToSave
 554                                 ) throws SQLException{
 555
 556                 Map<String, Team> teamMap = partitioner.getObjectMap(BerlinModelAuthorTeamImport.NAMESPACE);
 557
 558                 String refCache = rs.getString("refCache");
 559                 String nomRefCache = rs.getString("nomRefCache");
 560                 String title = rs.getString("title");
 561                 String nomTitleAbbrev = rs.getString("nomTitleAbbrev");
 562                 boolean isPreliminary = rs.getBoolean("PreliminaryFlag");
 563                 String refAuthorString = rs.getString("refAuthorString");
 564                 Integer nomAuthorTeamFk = rs.getInt("NomAuthorTeamFk");
 565                 String strNomAuthorTeamFk = String.valueOf(nomAuthorTeamFk);
 566                 TeamOrPersonBase<?> nomAuthor = teamMap.get(strNomAuthorTeamFk);
 567                 Reference nomReference = null;
 568
 569                 boolean hasNomRef = false;
 570                 boolean hasBiblioRef = false;
 571                 Reference sourceReference = state.getTransactionalSourceReference();
 572
 573                 //is Nomenclatural Reference
 574                 if ( (CdmUtils.isNotEmpty(nomRefCache) && isPreliminary) || (CdmUtils.isNotEmpty(nomTitleAbbrev) && ! isPreliminary) ){
 575                         referenceBase.setTitle(nomTitleAbbrev);
 576                         TeamOrPersonBase<?> author = getAuthorTeam(refAuthorString , nomAuthor, true);
 577                         referenceBase.setAuthorTeam(author);
 578                         //referenceBase.setNomenclaturallyRelevant(true);
 579                         if (isPreliminary){
 580                                 referenceBase.setTitleCache(nomRefCache, true);
 581                         }
 582                         if (! nomRefToSave.containsKey(refId)){
 583                                 if (referenceBase == null){
 584                                         logger.warn("refBase is null");
 585                                 }
 586                                 nomRefToSave.put(refId, referenceBase);
 587                         }else{
 588                                 logger.warn("Duplicate refId in Berlin Model database. Second reference was not imported !!");
 589                         }
 590
 591 //                      ???
 592 //                      nomRefToSave.put(refId, referenceBase);
 593                         hasNomRef = true;
 594                         nomReference = referenceBase;
 595                         refCounter.nomRefCount++;
 596                 }
 597                 //is bibliographical Reference
 598                 if ((CdmUtils.isNotEmpty(refCache) && isPreliminary && ! refCache.equalsIgnoreCase(nomRefCache))
 599                                 || (CdmUtils.isNotEmpty(title) && ! isPreliminary && ! title.equalsIgnoreCase(nomTitleAbbrev))
 600                                 || hasNomRef == false){
 601                         if (hasNomRef){
 602                                 referenceBase = (Reference)referenceBase.clone();
 603                                 copyCreatedUpdated(referenceBase, nomReference);
 604                         }
 605                         referenceBase.setTitle(title);
 606                         TeamOrPersonBase author = getAuthorTeam(refAuthorString , nomAuthor, false);
 607                         referenceBase.setAuthorTeam(author);
 608                         referenceBase.setNomenclaturallyRelevant(false);
 609                         if (isPreliminary){
 610                                 referenceBase.setTitleCache(refCache, true);
 611                         }
 612                         if (! biblioRefToSave.containsKey(refId)){
 613                                 biblioRefToSave.put(refId, referenceBase);
 614                         }else{
 615                                 logger.warn("Duplicate refId in Berlin Model database. Second reference was not imported !!");
 616                         }
 617                         hasBiblioRef = true;
 618
 619                         //??
 620                         //biblioRefToSave.put(refId, referenceBase);
 621                         refCounter.referenceCount++;
 622                 }
 623                 //refId
 624                 if (hasNomRef){
 625                         ImportHelper.setOriginalSource(nomReference, sourceReference, refId, NOM_REFERENCE_NAMESPACE);
 626                 }
 627                 if (hasBiblioRef){
 628                         ImportHelper.setOriginalSource(referenceBase, sourceReference, refId, BIBLIO_REFERENCE_NAMESPACE);
 629                 }
 630
 631                 return true;
 632
 633         }
 634
 635         /**
 636          * Copies the created and updated information from the nomReference to the cloned bibliographic reference
 637          * @param referenceBase
 638          * @param nomReference
 639          */
 640         private void copyCreatedUpdated(Reference<?> biblioReference, Reference nomReference) {
 641                 biblioReference.setCreatedBy(nomReference.getCreatedBy());
 642                 biblioReference.setCreated(nomReference.getCreated());
 643                 biblioReference.setUpdatedBy(nomReference.getUpdatedBy());
 644                 biblioReference.setUpdated(nomReference.getUpdated());
 645
 646         }
 647
 648         private Reference<?> makeArticle (Map<String, Object> valueMap, Map<Integer, Reference> biblioRefToSave, Map<Integer, Reference> nomRefToSave, Map<String, Reference> relatedBiblioReferences, Map<String, Reference> relatedNomReferences){
 649
 650                 IArticle article = ReferenceFactory.newArticle();
 651                 Object inRefFk = valueMap.get("inRefFk".toLowerCase());
 652                 Integer inRefCategoryFk = (Integer)valueMap.get("inRefCategoryFk".toLowerCase());
 653                 Integer refId = (Integer)valueMap.get("refId".toLowerCase());
 654
 655                 if (inRefFk != null){
 656                         if (inRefCategoryFk != REF_JOURNAL){
 657                                 logger.warn("Wrong inrefCategory for Article (refID = " + refId +"). Type must be 'Journal' but was not (RefCategoryFk=" + inRefCategoryFk + "))." +
 658                                         " InReference was added anyway! ");
 659                         }
 660                 }else{
 661                         logger.warn ("Article has no inreference: " + refId);
 662                 }
 663                 makeStandardMapper(valueMap, (Reference)article); //url, pages, series, volume
 664                 return (Reference)article;
 665         }
 666
 667         private Reference<?> makePartOfOtherTitle (Map<String, Object> valueMap, Map<Integer, Reference> biblioRefToSave, Map<Integer, Reference> nomRefToSave, Map<String, Reference> relatedBiblioReferences, Map<String, Reference> relatedNomReferences){
 668                 Reference<?> result;
 669                 Object inRefFk = valueMap.get("inRefFk".toLowerCase());
 670                 Integer inRefCategoryFk = (Integer)valueMap.get("inRefCategoryFk".toLowerCase());
 671                 Integer refId = (Integer)valueMap.get("refId".toLowerCase());
 672
 673                 if (inRefCategoryFk == null){
 674                         //null -> error
 675                         logger.warn("Part-Of-Other-Title has no inRefCategoryFk! RefId = " + refId + ". ReferenceType set to Generic.");
 676                         result = makeUnknown(valueMap);
 677                 }else if (inRefFk == null){
 678                         logger.warn("Part-Of-Other-Title has in in reference: " + refId);
 679                         result = makeUnknown(valueMap);
 680                 }else if (inRefCategoryFk == REF_BOOK){
 681                         //BookSection
 682                         IBookSection bookSection = ReferenceFactory.newBookSection();
 683                         result = (Reference<?>)bookSection;
 684                 }else if (inRefCategoryFk == REF_ARTICLE){
 685                         //Article
 686                         //TODO
 687                         logger.info("Reference (refId = " + refId + ") of type 'part_of_other_title' is part of 'article'." +
 688                                         " There is no specific reference type for such in references yet. Generic reference created instead") ;
 689                         result = ReferenceFactory.newGeneric();
 690                 }else if (inRefCategoryFk == REF_JOURNAL){
 691                         //TODO
 692                         logger.warn("Reference (refId = " + refId + ") of type 'part_of_other_title' has inReference of type 'journal'." +
 693                                         " This is not allowed! Generic reference created instead") ;
 694                         result = ReferenceFactory.newGeneric();
 695                         result.addMarker(Marker.NewInstance(MarkerType.TO_BE_CHECKED(), true));
 696                 }else{
 697                         logger.warn("InReference type (catFk = " + inRefCategoryFk + ") of part-of-reference not recognized for refId " + refId + "." +
 698                                 " Create 'Generic' reference instead");
 699                         result = ReferenceFactory.newGeneric();
 700                 }
 701                 makeStandardMapper(valueMap, result); //url, pages
 702                 return result;
 703         }
 704
 705
 706         /**
 707          * @param inRefFkInt
 708          * @param biblioRefToSave
 709          * @param nomRefToSave
 710          * @param relatedBiblioReferences
 711          * @param relatedNomReferences
 712          * @return
 713          */
 714         private boolean existsInMapOrToSave(Integer inRefFkInt, Map<Integer, Reference> biblioRefToSave, Map<Integer, Reference> nomRefToSave, Map<String, Reference> relatedBiblioReferences, Map<String, Reference> relatedNomReferences) {
 715                 boolean result = false;
 716                 if (inRefFkInt == null){
 717                         return false;
 718                 }
 719                 result |= nomRefToSave.containsKey(inRefFkInt);
 720                 result |= biblioRefToSave.containsKey(inRefFkInt);
 721                 result |= relatedBiblioReferences.containsKey(String.valueOf(inRefFkInt));
 722                 result |= relatedNomReferences.containsKey(String.valueOf(inRefFkInt));
 723                 return result;
 724         }
 725
 726         private Reference<?> makeWebSite(Map<String, Object> valueMap){
 727                 if (logger.isDebugEnabled()){logger.debug("RefType 'Website'");}
 728                 Reference<?> webPage = ReferenceFactory.newWebPage();
 729                 makeStandardMapper(valueMap, webPage); //placePublished, publisher
 730                 return webPage;
 731         }
 732
 733         private Reference<?> makeUnknown(Map<String, Object> valueMap){
 734                 if (logger.isDebugEnabled()){logger.debug("RefType 'Unknown'");}
 735                 Reference<?> generic = ReferenceFactory.newGeneric();
 736 //              generic.setSeries(series);
 737                 makeStandardMapper(valueMap, generic); //pages, placePublished, publisher, series, volume
 738                 return generic;
 739         }
 740
 741         private Reference<?> makeInformal(Map<String, Object> valueMap){
 742                 if (logger.isDebugEnabled()){logger.debug("RefType 'Informal'");}
 743                 Reference<?> generic = ReferenceFactory.newGeneric();
 744 //              informal.setSeries(series);
 745                 makeStandardMapper(valueMap, generic);//editor, pages, placePublished, publisher, series, volume
 746                 String informal = (String)valueMap.get("InformalRefCategory".toLowerCase());
 747                 if (CdmUtils.isNotEmpty(informal) ){
 748                         generic.addExtension(informal, ExtensionType.INFORMAL_CATEGORY());
 749                 }
 750                 return generic;
 751         }
 752
 753         private Reference<?> makeDatabase(Map<String, Object> valueMap){
 754                 if (logger.isDebugEnabled()){logger.debug("RefType 'Database'");}
 755                 Reference database =  ReferenceFactory.newDatabase();
 756                 makeStandardMapper(valueMap, database); //?
 757                 return database;
 758         }
 759
 760         private Reference<?> makeJournal(Map<String, Object> valueMap){
 761                 if (logger.isDebugEnabled()){logger.debug("RefType 'Journal'");}
 762                 Reference journal = ReferenceFactory.newJournal();
 763
 764                 Set<String> omitAttributes = new HashSet<String>();
 765                 String series = "series";
 766 //              omitAttributes.add(series);
 767
 768                 makeStandardMapper(valueMap, journal, omitAttributes); //issn,placePublished,publisher
 769 //              if (valueMap.get(series) != null){
 770 //                      logger.warn("Series not yet implemented for journal!");
 771 //              }
 772                 return journal;
 773         }
 774
 775         private Reference<?> makeBook(
 776                                 Map<String, Object> valueMap,
 777                                 Map<Integer, Reference> biblioRefToSave,
 778                                 Map<Integer, Reference> nomRefToSave,
 779                                 Map<String, Reference> relatedBiblioReferences,
 780                                 Map<String, Reference> relatedNomReferences){
 781                 if (logger.isDebugEnabled()){logger.debug("RefType 'Book'");}
 782                 Reference<?> book = ReferenceFactory.newBook();
 783                 Integer refId = (Integer)valueMap.get("refId".toLowerCase());
 784
 785                 //Set bookAttributes = new String[]{"edition", "isbn", "pages","publicationTown","publisher","volume"};
 786
 787                 Set<String> omitAttributes = new HashSet<String>();
 788                 String attrSeries = "series";
 789 //              omitAttributes.add(attrSeries);
 790
 791                 makeStandardMapper(valueMap, book, omitAttributes);
 792
 793                 //Series (as String)
 794                 IPrintSeries printSeries = null;
 795                 if (valueMap.get(attrSeries) != null){
 796                         String series = (String)valueMap.get("title".toLowerCase());
 797                         if (series == null){
 798                                 String nomTitle = (String)valueMap.get("nomTitleAbbrev".toLowerCase());
 799                                 series = nomTitle;
 800                         }
 801                         printSeries = ReferenceFactory.newPrintSeries(series);
 802                         logger.info("Implementation of printSeries is preliminary");
 803                 }
 804                 Object inRefFk = valueMap.get("inRefFk".toLowerCase());
 805                 //Series (as Reference)
 806                 if (inRefFk != null && false){  //&&false added for first/second path implementation, following code may be removed if this is successful
 807                         int inRefFkInt = (Integer)inRefFk;
 808                         if (existsInMapOrToSave(inRefFkInt, biblioRefToSave, nomRefToSave, relatedBiblioReferences, relatedNomReferences)){
 809                                 Reference<?> inSeries = getReferenceFromMaps(inRefFkInt, nomRefToSave, relatedNomReferences);
 810                                 if (inSeries == null){
 811                                         inSeries = getReferenceFromMaps(inRefFkInt, biblioRefToSave, relatedBiblioReferences);
 812                                         logger.info("inSeries (" + inRefFkInt + ") found in referenceStore instead of nomRefStore.");
 813                                         nomRefToSave.put(inRefFkInt, inSeries);
 814                                 }
 815                                 if (inSeries == null){
 816                                         logger.warn("inSeries for " + inRefFkInt + " is null. "+
 817                                         " InReference relation could not be set");;
 818                                 //}else if (PrintSeries.class.isAssignableFrom(inSeries.getClass())){
 819                                 }else if (inSeries.getType().equals(ReferenceType.PrintSeries)){
 820                                         book.setInSeries((IPrintSeries)inSeries);
 821                                         //TODO
 822                                 }else{
 823                                         logger.warn("inSeries is not of type PrintSeries but of type " + inSeries.getType().getMessage() +
 824                                                         ". In-reference relation could not be set for refId " + refId + " and inRefFk " + inRefFk);
 825                                 }
 826                         }else{
 827                                 logger.error("PrintSeries (refId = " + inRefFkInt + ") for book (refID = " + refId +") could not be found in nomRefStore. Inconsistency error. ");
 828                                 //success = false;
 829                         }
 830                 }
 831                 if (book.getInSeries() != null && printSeries != null){
 832                         logger.warn("Book has series string and inSeries reference. Can not take both. Series string neglected");
 833                 }else{
 834                         book.setInSeries(printSeries);
 835                 }
 836                 book.setEditor(null);
 837                 return book;
 838
 839         }
 840
 841         /**
 842          * Returns the requested object if it exists in one of both maps. Prefers the refToSaveMap in ambigious cases.
 843          * @param inRefFkInt
 844          * @param nomRefToSave
 845          * @param relatedNomReferences
 846          * @return
 847          */
 848         private Reference<?> getReferenceFromMaps(
 849                         int inRefFkInt,
 850                         Map<Integer, Reference> refToSaveMap,
 851                         Map<String, Reference> relatedRefMap) {
 852                 Reference<?> result = null;
 853                 result = refToSaveMap.get(inRefFkInt);
 854                 if (result == null){
 855                         result = relatedRefMap.get(String.valueOf(inRefFkInt));
 856                 }
 857                 return result;
 858         }
 859
 860         private Reference<?> makePrintSeries(Map<String, Object> valueMap){
 861                 if (logger.isDebugEnabled()){logger.debug("RefType 'PrintSeries'");}
 862                 Reference<?> printSeries = ReferenceFactory.newPrintSeries();
 863                 makeStandardMapper(valueMap, printSeries, null);
 864                 return printSeries;
 865         }
 866
 867         private Reference<?> makeProceedings(Map<String, Object> valueMap){
 868                 if (logger.isDebugEnabled()){logger.debug("RefType 'Proceedings'");}
 869                 Reference<?> proceedings = ReferenceFactory.newProceedings();
 870                 makeStandardMapper(valueMap, proceedings, null);
 871                 return proceedings;
 872         }
 873
 874         private Reference<?> makeThesis(Map<String, Object> valueMap){
 875                 if (logger.isDebugEnabled()){logger.debug("RefType 'Thesis'");}
 876                 Reference<?> thesis = ReferenceFactory.newThesis();
 877                 makeStandardMapper(valueMap, thesis, null);
 878                 return thesis;
 879         }
 880
 881
 882         private Reference<?> makeJournalVolume(Map<String, Object> valueMap){
 883                 if (logger.isDebugEnabled()){logger.debug("RefType 'JournalVolume'");}
 884                 //Proceedings proceedings = Proceedings.NewInstance();
 885                 Reference<?> journalVolume = ReferenceFactory.newGeneric();
 886                 makeStandardMapper(valueMap, journalVolume, null);
 887                 logger.warn("Journal volumes not yet implemented. Generic created instead but with errors");
 888                 return journalVolume;
 889         }
 890
 891         private boolean makeStandardMapper(Map<String, Object> valueMap, Reference<?> ref){
 892                 return makeStandardMapper(valueMap, ref, null);
 893         }
 894
 895
 896         private boolean makeStandardMapper(Map<String, Object> valueMap, CdmBase cdmBase, Set<String> omitAttributes){
 897                 boolean result = true;
 898                 for (CdmAttributeMapperBase mapper : classMappers){
 899                         if (mapper instanceof CdmSingleAttributeMapperBase){
 900                                 result &= makeStandardSingleMapper(valueMap, cdmBase, (CdmSingleAttributeMapperBase)mapper, omitAttributes);
 901                         }else if (mapper instanceof CdmOneToManyMapper){
 902                                 result &= makeMultipleValueAddMapper(valueMap, cdmBase, (CdmOneToManyMapper)mapper, omitAttributes);
 903                         }else{
 904                                 logger.error("Unknown mapper type");
 905                                 result = false;
 906                         }
 907                 }
 908                 return result;
 909         }
 910
 911         private boolean makeStandardSingleMapper(Map<String, Object> valueMap, CdmBase cdmBase, CdmSingleAttributeMapperBase mapper, Set<String> omitAttributes){
 912                 boolean result = true;
 913                 if (omitAttributes == null){
 914                         omitAttributes = new HashSet<String>();
 915                 }
 916                 if (mapper instanceof DbImportExtensionMapper){
 917                         result &= ((DbImportExtensionMapper)mapper).invoke(valueMap, cdmBase);
 918                 }else if (mapper instanceof DbImportMarkerMapper){
 919                         result &= ((DbImportMarkerMapper)mapper).invoke(valueMap, cdmBase);
 920                 }else{
 921                         String sourceAttribute = mapper.getSourceAttributeList().get(0).toLowerCase();
 922                         Object value = valueMap.get(sourceAttribute);
 923                         if (mapper instanceof CdmUriMapper && value != null){
 924                                 try {
 925                                         value = new URI (value.toString());
 926                                 } catch (URISyntaxException e) {
 927                                         logger.error("URI syntax exception: " + value.toString());
 928                                         value = null;
 929                                 }
 930                         }
 931                         if (value != null){
 932                                 String destinationAttribute = mapper.getDestinationAttribute();
 933                                 if (! omitAttributes.contains(destinationAttribute)){
 934                                         result &= ImportHelper.addValue(value, cdmBase, destinationAttribute, mapper.getTypeClass(), OVERWRITE, OBLIGATORY);
 935                                 }
 936                         }
 937                 }
 938                 return result;
 939         }
 940
 941
 942         private boolean makeMultipleValueAddMapper(Map<String, Object> valueMap, CdmBase cdmBase, CdmOneToManyMapper<CdmBase, CdmBase, CdmSingleAttributeMapperBase> mapper, Set<String> omitAttributes){
 943                 if (omitAttributes == null){
 944                         omitAttributes = new HashSet<String>();
 945                 }
 946                 boolean result = true;
 947                 String destinationAttribute = mapper.getSingleAttributeName();
 948                 List<Object> sourceValues = new ArrayList<Object>();
 949                 List<Class> classes = new ArrayList<Class>();
 950                 for (CdmSingleAttributeMapperBase singleMapper : mapper.getSingleMappers()){
 951                         String sourceAttribute = singleMapper.getSourceAttribute();
 952                         Object value = valueMap.get(sourceAttribute);
 953                         sourceValues.add(value);
 954                         Class<?> clazz = singleMapper.getTypeClass();
 955                         classes.add(clazz);
 956                 }
 957
 958                 result &= ImportHelper.addMultipleValues(sourceValues, cdmBase, destinationAttribute, classes, NO_OVERWRITE, OBLIGATORY);
 959 //              //only for testing
 960 //              if (cdmBase instanceof PublicationBase){
 961 //                      PublicationBase pub = ((PublicationBase)cdmBase);
 962 //                      pub.addPublisher("A new publisher for " + pub.getTitleCache(), "A nice place");
 963 //              }
 964                 return result;
 965         }
 966
 967
 968         private static TeamOrPersonBase<?> getAuthorTeam(String authorString, TeamOrPersonBase<?> nomAuthor, boolean preferNomeclaturalAuthor){
 969                 TeamOrPersonBase<?> result;
 970                 if (preferNomeclaturalAuthor){
 971                         if (nomAuthor != null){
 972                                 result = nomAuthor;
 973                         }else{
 974                                 if (CdmUtils.isEmpty(authorString)){
 975                                         result = null;
 976                                 }else{
 977                                         TeamOrPersonBase<?> team = Team.NewInstance();
 978                                         //TODO which one to use??
 979                                         team.setNomenclaturalTitle(authorString);
 980                                         team.setTitleCache(authorString, true);
 981                                         result = team;
 982                                 }
 983                         }
 984                 }else{ //prefer bibliographic
 985                         if (CdmUtils.isNotEmpty(authorString)){
 986                                 TeamOrPersonBase<?> team = Team.NewInstance();
 987                                 //TODO which one to use??
 988                                 team.setNomenclaturalTitle(authorString);
 989                                 team.setTitleCache(authorString, true);
 990                                 result = team;
 991                         }else{
 992                                 result = nomAuthor;
 993                         }
 994                 }
 995                 return result;
 996         }
 997
 998
 999         /**
1000          * @param lowerCase
1001          * @param config
1002          * @return
1003          */
1004         public Set<String> getObligatoryAttributes(boolean lowerCase, BerlinModelImportConfigurator config){
1005                 Set<String> result = new HashSet<String>();
1006                 Class<ICdmIO>[] ioClassList = config.getIoClassList();
1007                 logger.warn("getObligatoryAttributes has been commented because it still needs to be adapted to the new package structure");
1008                 result.addAll(Arrays.asList(unclearMappers));
1009                 result.addAll(Arrays.asList(createdAndNotesAttributes));
1010                 result.addAll(Arrays.asList(operationalAttributes));
1011                 CdmIoMapping mapping = new CdmIoMapping();
1012                 for (CdmAttributeMapperBase mapper : classMappers){
1013                         mapping.addMapper(mapper);
1014                 }
1015                 result.addAll(mapping.getSourceAttributes());
1016                 if (lowerCase){
1017                         Set<String> lowerCaseResult = new HashSet<String>();
1018                         for (String str : result){
1019                                 if (str != null){lowerCaseResult.add(str.toLowerCase());}
1020                         }
1021                         result = lowerCaseResult;
1022                 }
1023                 return result;
1024         }
1025
1026         /* (non-Javadoc)
1027          * @see eu.etaxonomy.cdm.io.common.CdmIoBase#doCheck(eu.etaxonomy.cdm.io.common.IoStateBase)
1028          */
1029         @Override
1030         protected boolean doCheck(BerlinModelImportState state){
1031                 BerlinModelReferenceImportValidator validator = new BerlinModelReferenceImportValidator();
1032                 return validator.validate(state, this);
1033         }
1034
1035
1036         /* (non-Javadoc)
1037          * @see eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelImportBase#getTableName()
1038          */
1039         @Override
1040         protected String getTableName() {
1041                 return dbTableName;
1042         }
1043
1044         /* (non-Javadoc)
1045          * @see eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelImportBase#getPluralString()
1046          */
1047         @Override
1048         public String getPluralString() {
1049                 return pluralString;
1050         }
1051
1052         /* (non-Javadoc)
1053          * @see eu.etaxonomy.cdm.io.common.CdmIoBase#isIgnore(eu.etaxonomy.cdm.io.common.IImportConfigurator)
1054          */
1055         protected boolean isIgnore(BerlinModelImportState state){
1056                 return (state.getConfig().getDoReferences() == IImportConfigurator.DO_REFERENCES.NONE);
1057         }
1058
1059
1060
1061
1062 }