cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/berlinModel/in/BerlinModelReferenceImport.java

   1 /**
   2 * Copyright (C) 2007 EDIT
   3 * European Distributed Institute of Taxonomy
   4 * http://www.e-taxonomy.eu
   5 *
   6 * The contents of this file are subject to the Mozilla Public License Version 1.1
   7 * See LICENSE.TXT at the top of this package for the full license terms.
   8 */
   9
  10 package eu.etaxonomy.cdm.io.berlinModel.in;
  11
  12 import static eu.etaxonomy.cdm.io.berlinModel.BerlinModelTransformer.REF_ARTICLE;
  13 import static eu.etaxonomy.cdm.io.berlinModel.BerlinModelTransformer.REF_BOOK;
  14 import static eu.etaxonomy.cdm.io.berlinModel.BerlinModelTransformer.REF_CONFERENCE_PROCEEDINGS;
  15 import static eu.etaxonomy.cdm.io.berlinModel.BerlinModelTransformer.REF_DATABASE;
  16 import static eu.etaxonomy.cdm.io.berlinModel.BerlinModelTransformer.REF_INFORMAL;
  17 import static eu.etaxonomy.cdm.io.berlinModel.BerlinModelTransformer.REF_JOURNAL;
  18 import static eu.etaxonomy.cdm.io.berlinModel.BerlinModelTransformer.REF_JOURNAL_VOLUME;
  19 import static eu.etaxonomy.cdm.io.berlinModel.BerlinModelTransformer.REF_PART_OF_OTHER_TITLE;
  20 import static eu.etaxonomy.cdm.io.berlinModel.BerlinModelTransformer.REF_PRINT_SERIES;
  21 import static eu.etaxonomy.cdm.io.berlinModel.BerlinModelTransformer.REF_UNKNOWN;
  22 import static eu.etaxonomy.cdm.io.berlinModel.BerlinModelTransformer.REF_WEBSITE;
  23 import static eu.etaxonomy.cdm.io.common.ImportHelper.NO_OVERWRITE;
  24 import static eu.etaxonomy.cdm.io.common.ImportHelper.OBLIGATORY;
  25 import static eu.etaxonomy.cdm.io.common.ImportHelper.OVERWRITE;
  26
  27 import java.net.URI;
  28 import java.net.URISyntaxException;
  29 import java.sql.ResultSet;
  30 import java.sql.SQLException;
  31 import java.util.ArrayList;
  32 import java.util.Arrays;
  33 import java.util.HashMap;
  34 import java.util.HashSet;
  35 import java.util.List;
  36 import java.util.Map;
  37 import java.util.Set;
  38 import java.util.UUID;
  39
  40 import org.apache.commons.lang.StringUtils;
  41 import org.apache.log4j.Logger;
  42 import org.springframework.stereotype.Component;
  43
  44 import eu.etaxonomy.cdm.common.CdmUtils;
  45 import eu.etaxonomy.cdm.io.berlinModel.CdmOneToManyMapper;
  46 import eu.etaxonomy.cdm.io.berlinModel.CdmStringMapper;
  47 import eu.etaxonomy.cdm.io.berlinModel.CdmUriMapper;
  48 import eu.etaxonomy.cdm.io.berlinModel.in.validation.BerlinModelReferenceImportValidator;
  49 import eu.etaxonomy.cdm.io.common.ICdmIO;
  50 import eu.etaxonomy.cdm.io.common.IImportConfigurator;
  51 import eu.etaxonomy.cdm.io.common.ImportHelper;
  52 import eu.etaxonomy.cdm.io.common.ResultSetPartitioner;
  53 import eu.etaxonomy.cdm.io.common.Source;
  54 import eu.etaxonomy.cdm.io.common.mapping.CdmAttributeMapperBase;
  55 import eu.etaxonomy.cdm.io.common.mapping.CdmIoMapping;
  56 import eu.etaxonomy.cdm.io.common.mapping.CdmSingleAttributeMapperBase;
  57 import eu.etaxonomy.cdm.io.common.mapping.DbImportExtensionMapper;
  58 import eu.etaxonomy.cdm.io.common.mapping.DbImportMarkerMapper;
  59 import eu.etaxonomy.cdm.io.common.mapping.DbSingleAttributeImportMapperBase;
  60 import eu.etaxonomy.cdm.model.agent.Team;
  61 import eu.etaxonomy.cdm.model.agent.TeamOrPersonBase;
  62 import eu.etaxonomy.cdm.model.common.CdmBase;
  63 import eu.etaxonomy.cdm.model.common.ExtensionType;
  64 import eu.etaxonomy.cdm.model.common.IdentifiableSource;
  65 import eu.etaxonomy.cdm.model.common.Marker;
  66 import eu.etaxonomy.cdm.model.common.MarkerType;
  67 import eu.etaxonomy.cdm.model.reference.IArticle;
  68 import eu.etaxonomy.cdm.model.reference.IBookSection;
  69 import eu.etaxonomy.cdm.model.reference.IPrintSeries;
  70 import eu.etaxonomy.cdm.model.reference.Reference;
  71 import eu.etaxonomy.cdm.model.reference.ReferenceFactory;
  72 import eu.etaxonomy.cdm.model.reference.ReferenceType;
  73
  74 /**
  75  * @author a.mueller
  76  * @created 20.03.2008
  77  */
  78 @Component
  79 public class BerlinModelReferenceImport extends BerlinModelImportBase {
  80         private static final Logger logger = Logger.getLogger(BerlinModelReferenceImport.class);
  81
  82         public static final String REFERENCE_NAMESPACE = "Reference";
  83
  84         public static final UUID REF_DEPOSITED_AT_UUID = UUID.fromString("23ca88c7-ce73-41b2-8ca3-2cb22f013beb");
  85         public static final UUID REF_SOURCE_UUID = UUID.fromString("d6432582-2216-4b08-b0db-76f6c1013141");
  86         public static final UUID DATE_STRING_UUID = UUID.fromString("e4130eae-606e-4b0c-be4f-e93dc161be7d");
  87         public static final UUID IS_PAPER_UUID = UUID.fromString("8a326129-d0d0-4f9d-bbdf-8d86b037c65e");
  88
  89
  90         private int modCount = 1000;
  91         private static final String pluralString = "references";
  92         private static final String dbTableName = "reference";
  93
  94
  95         public BerlinModelReferenceImport(){
  96                 super(dbTableName, pluralString);
  97         }
  98
  99         protected void initializeMappers(BerlinModelImportState state){
 100                 for (CdmAttributeMapperBase mapper: classMappers){
 101                         if (mapper instanceof DbSingleAttributeImportMapperBase){
 102                                 DbSingleAttributeImportMapperBase singleMapper = (DbSingleAttributeImportMapperBase)mapper;
 103                                 singleMapper.initialize(state, Reference.class);
 104                         }
 105                 }
 106                 return;
 107         }
 108
 109         protected static CdmAttributeMapperBase[] classMappers = new CdmAttributeMapperBase[]{
 110                 new CdmStringMapper("edition", "edition"),
 111                 new CdmStringMapper("volume", "volume"),
 112                 new CdmStringMapper("publisher", "publisher"),
 113                 new CdmStringMapper("publicationTown", "placePublished"),
 114                 new CdmStringMapper("isbn", "isbn"),
 115                 new CdmStringMapper("isbn", "isbn"),
 116                 new CdmStringMapper("pageString", "pages"),
 117                 new CdmStringMapper("series", "series"),
 118                 new CdmStringMapper("issn", "issn"),
 119                 new CdmUriMapper("url", "uri"),
 120                 DbImportExtensionMapper.NewInstance("NomStandard", ExtensionType.NOMENCLATURAL_STANDARD()),
 121                 DbImportExtensionMapper.NewInstance("DateString", DATE_STRING_UUID, "Date String", "Date String", "dates"),
 122                 DbImportExtensionMapper.NewInstance("RefDepositedAt", REF_DEPOSITED_AT_UUID, "RefDepositedAt", "reference is deposited at", "at"),
 123                 DbImportExtensionMapper.NewInstance("RefSource", REF_SOURCE_UUID, "RefSource", "reference source", "source"),
 124                 DbImportMarkerMapper.NewInstance("isPaper", IS_PAPER_UUID, "is paper", "is paper", "paper", false)
 125         };
 126
 127
 128         protected static String[] operationalAttributes = new String[]{
 129                 "refId", "refCache", "nomRefCache", "preliminaryFlag", "inRefFk", "title", "nomTitleAbbrev",
 130                 "refAuthorString", "nomAuthorTeamFk",
 131                 "refCategoryFk", "thesisFlag", "informalRefCategory", "idInSource"
 132         };
 133
 134         protected static String[] createdAndNotesAttributes = new String[]{
 135                         "created_When", "updated_When", "created_Who", "updated_Who", "notes"
 136         };
 137
 138         protected static String[] unclearMappers = new String[]{
 139                         /*"isPaper",*/ "exportDate",
 140         };
 141
 142         //TODO isPaper
 143         //
 144
 145
 146
 147         //type to count the references nomReferences that have been created and saved
 148         private class RefCounter{
 149                 RefCounter() {refCount = 0;};
 150                 int refCount;
 151
 152                 public String toString(){return String.valueOf(refCount) ;};
 153         }
 154
 155
 156         /* (non-Javadoc)
 157          * @see eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelImportBase#getRecordQuery(eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelImportConfigurator)
 158          */
 159         @Override
 160         protected String getRecordQuery(BerlinModelImportConfigurator config) {
 161                 return null;  //not needed
 162         }
 163
 164         @Override
 165         protected void doInvoke(BerlinModelImportState state){
 166                 logger.info("start make " + getPluralString() + " ...");
 167
 168                 boolean success = true;
 169                 initializeMappers(state);
 170                 BerlinModelImportConfigurator config = state.getConfig();
 171                 Source source = config.getSource();
 172
 173                 String strSelectId = " SELECT Reference.RefId as refId ";
 174                 String strSelectFull =
 175                         " SELECT Reference.* ,InReference.RefCategoryFk as InRefCategoryFk, RefSource.RefSource " ;
 176                 String strFrom =  " FROM %s  " +
 177                         " LEFT OUTER JOIN Reference as InReference ON InReference.refId = Reference.inRefFk " +
 178                                 " LEFT OUTER JOIN RefSource ON Reference.RefSourceFk = RefSource.RefSourceId " +
 179                         " WHERE (1=1) ";
 180                 String strWherePartitioned = " AND (Reference.refId IN ("+ ID_LIST_TOKEN + ") ) ";
 181
 182                 String referenceTable = CdmUtils.Nz(state.getConfig().getReferenceIdTable());
 183                 referenceTable = referenceTable.isEmpty() ? " Reference"  : referenceTable + " as Reference ";
 184                 String strIdFrom = String.format(strFrom, referenceTable );
 185
 186                 String strSelectIdBase = strSelectId + strIdFrom;
 187
 188                 String referenceFilter = CdmUtils.Nz(state.getConfig().getReferenceIdTable());
 189                 if (! referenceFilter.isEmpty()){
 190                         referenceFilter = " AND " + referenceFilter + " ";
 191                 }
 192                 referenceFilter = "";  //don't use it for now
 193
 194                 String strIdQueryFirstPath = strSelectId + strIdFrom ;
 195                 String strIdQuerySecondPath = strSelectId + strIdFrom + " AND (Reference.InRefFk is NOT NULL) ";
 196
 197 //              if (config.getDoReferences() == CONCEPT_REFERENCES){
 198 //                      strIdQueryNoInRef += " AND ( Reference.refId IN ( SELECT ptRefFk FROM PTaxon) ) " + referenceFilter;
 199 //              }
 200
 201                 String strRecordQuery = strSelectFull + String.format(strFrom, " Reference ") + strWherePartitioned;
 202
 203                 int recordsPerTransaction = config.getRecordsPerTransaction();
 204                 try{
 205                         //firstPath
 206                         ResultSetPartitioner partitioner = ResultSetPartitioner.NewInstance(source, strIdQueryFirstPath, strRecordQuery, recordsPerTransaction);
 207                         while (partitioner.nextPartition()){
 208                                 partitioner.doPartition(this, state);
 209                         }
 210                         logger.info("end make references without in-references ... " + getSuccessString(success));
 211                         state.setReferenceSecondPath(true);
 212
 213 //                      if (config.getDoReferences() == ALL || config.getDoReferences() == NOMENCLATURAL){
 214
 215                         //secondPath
 216                         partitioner = ResultSetPartitioner.NewInstance(source, strIdQuerySecondPath, strRecordQuery, recordsPerTransaction);
 217                         while (partitioner.nextPartition()){
 218                                 partitioner.doPartition(this, state);
 219                         }
 220                         logger.info("end make references with no 1 in-reference ... " + getSuccessString(success));
 221                         state.setReferenceSecondPath(false);
 222
 223 //                      }
 224
 225                 } catch (SQLException e) {
 226                         logger.error("SQLException:" +  e);
 227                         state.setUnsuccessfull();
 228                         return;
 229                 }
 230                 logger.info("end make " + getPluralString() + " ... " + getSuccessString(success));
 231                 if (! success){
 232                         state.setUnsuccessfull();
 233                 }
 234                 return;
 235         }
 236
 237
 238
 239
 240         /* (non-Javadoc)
 241          * @see eu.etaxonomy.cdm.io.berlinModel.in.IPartitionedIO#doPartition(eu.etaxonomy.cdm.io.berlinModel.in.ResultSetPartitioner, eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelImportState)
 242          */
 243         public boolean doPartition(ResultSetPartitioner partitioner, BerlinModelImportState state) {
 244                 if (state.isReferenceSecondPath()){
 245                         return doPartitionSecondPath(partitioner, state);
 246                 }
 247                 boolean success = true;
 248
 249                 Map<Integer, Reference> refToSave = new HashMap<Integer, Reference>();
 250
 251                 Map<String, Reference> relatedReferences = partitioner.getObjectMap(REFERENCE_NAMESPACE);
 252
 253                 BerlinModelImportConfigurator config = state.getConfig();
 254
 255                 try {
 256
 257                                 int i = 0;
 258                                 RefCounter refCounter  = new RefCounter();
 259
 260                                 ResultSet rs = partitioner.getResultSet();
 261
 262                                 //for each resultset
 263                                 while (rs.next()){
 264                                         if ((i++ % modCount) == 0 && i!= 1 ){ logger.info("References handled: " + (i-1) + " in round -" );}
 265
 266                                         success &= makeSingleReferenceRecord(rs, state, partitioner, refToSave, relatedReferences, refCounter);
 267                                 } // end resultSet
 268
 269                                 //for the concept reference a fixed uuid may be needed -> change uuid
 270                                 Integer sourceSecId = (Integer)config.getSourceSecId();
 271                                 Reference<?> sec = refToSave.get(sourceSecId);
 272
 273                                 if (sec != null){
 274                                         sec.setUuid(config.getSecUuid());
 275                                         logger.info("SecUuid changed to: " + config.getSecUuid());
 276                                 }
 277
 278                                 //save and store in map
 279                                 logger.info("Save references (" + refCounter.refCount + ")");
 280                                 getReferenceService().saveOrUpdate(refToSave.values());
 281
 282 //                      }//end resultSetList
 283
 284 //                      logger.info("end makeReferences ..." + getSuccessString(success));;
 285                         return success;
 286                 } catch (SQLException e) {
 287                         logger.error("SQLException:" +  e);
 288                         return false;
 289                 }
 290         }
 291
 292
 293
 294         /**
 295          * Adds the inReference to the according references.
 296          * @param partitioner
 297          * @param state
 298          * @return
 299          */
 300         private boolean doPartitionSecondPath(ResultSetPartitioner partitioner, BerlinModelImportState state) {
 301                 boolean success = true;
 302
 303                 Map<Integer, Reference> refToSave = new HashMap<Integer, Reference>();
 304
 305                 Map<String, Reference> relatedReferencesMap = partitioner.getObjectMap(REFERENCE_NAMESPACE);
 306
 307                 try {
 308                                 int i = 0;
 309                                 RefCounter refCounter  = new RefCounter();
 310
 311                                 ResultSet rs = partitioner.getResultSet();
 312                                 //for each resultset
 313                                 while (rs.next()){
 314                                         if ((i++ % modCount) == 0 && i!= 1 ){ logger.info("References handled: " + (i-1) + " in round -" );}
 315
 316                                         Integer refId = rs.getInt("refId");
 317                                         Integer inRefFk = rs.getInt("inRefFk");
 318
 319                                         if (inRefFk != null){
 320
 321                                                 Reference<?> thisRef = relatedReferencesMap.get(String.valueOf(refId));
 322
 323                                                 Reference<?> inRef = relatedReferencesMap.get(String.valueOf(inRefFk));
 324
 325                                                 if (thisRef != null){
 326                                                         if (inRef == null){
 327                                                                 logger.warn("No InRef found for nomRef: " + thisRef.getTitleCache() + "; RefId: " +  refId + "; inRefFK: " +  inRefFk);
 328                                                         }
 329                                                         thisRef.setInReference(inRef);
 330                                                         refToSave.put(refId, thisRef);
 331                                                         thisRef.setTitleCache(null);
 332                                                         thisRef.getTitleCache();
 333                                                 }
 334                                         }
 335
 336                                 } // end resultSet
 337
 338                                 //save and store in map
 339                                 logger.info("Save references (" + refCounter.refCount + ")");
 340                                 getReferenceService().saveOrUpdate(refToSave.values());
 341
 342 //                      }//end resultSetList
 343
 344 //                      logger.info("end makeReferences ..." + getSuccessString(success));;
 345                         return success;
 346                 } catch (SQLException e) {
 347                         logger.error("SQLException:" +  e);
 348                         return false;
 349                 }
 350         }
 351
 352
 353
 354         /* (non-Javadoc)
 355          * @see eu.etaxonomy.cdm.io.berlinModel.in.IPartitionedIO#getRelatedObjectsForPartition(java.sql.ResultSet)
 356          */
 357         public Map<Object, Map<String, ? extends CdmBase>> getRelatedObjectsForPartition(ResultSet rs) {
 358                 String nameSpace;
 359                 Class cdmClass;
 360                 Set<String> idSet;
 361
 362                 Map<Object, Map<String, ? extends CdmBase>> result = new HashMap<Object, Map<String, ? extends CdmBase>>();
 363
 364                 try{
 365                         Set<String> teamIdSet = new HashSet<String>();
 366                         Set<String> referenceIdSet = new HashSet<String>();
 367
 368                         while (rs.next()){
 369                                 handleForeignKey(rs, teamIdSet, "NomAuthorTeamFk");
 370                                 handleForeignKey(rs, referenceIdSet, "InRefFk");
 371                                 //TODO only needed in second path but state not available here to check if state is second path
 372                                 handleForeignKey(rs, referenceIdSet, "refId");
 373                         }
 374
 375                         //team map
 376                         nameSpace = BerlinModelAuthorTeamImport.NAMESPACE;
 377                         cdmClass = Team.class;
 378                         idSet = teamIdSet;
 379                         Map<String, Team> teamMap = (Map<String, Team>)getCommonService().getSourcedObjectsByIdInSource(cdmClass, idSet, nameSpace);
 380                         result.put(nameSpace, teamMap);
 381
 382                         //reference map
 383                         nameSpace = BerlinModelReferenceImport.REFERENCE_NAMESPACE;
 384                         cdmClass = Reference.class;
 385                         idSet = referenceIdSet;
 386                         Map<String, Reference> referenceMap = (Map<String, Reference>)getCommonService().getSourcedObjectsByIdInSource(cdmClass, idSet, nameSpace);
 387                         result.put(nameSpace, referenceMap);
 388
 389                 } catch (SQLException e) {
 390                         throw new RuntimeException(e);
 391                 }
 392                 return result;
 393         }
 394
 395
 396         /**
 397          * Handles a single reference record
 398          * @param rs
 399          * @param state
 400          * @param biblioRefToSave
 401          * @param nomRefToSave
 402          * @param relatedBiblioReferences
 403          * @param relatedNomReferences
 404          * @param refCounter
 405          * @return
 406          */
 407         private boolean makeSingleReferenceRecord(
 408                                 ResultSet rs,
 409                                 BerlinModelImportState state,
 410                                 ResultSetPartitioner<BerlinModelImportState> partitioner,
 411                                 Map<Integer, Reference> refToSave,
 412                                 Map<String, Reference> relatedReferences,
 413                                 RefCounter refCounter){
 414                 boolean success = true;
 415
 416                 Integer refId = null;
 417                 try {
 418                         Map<String, Object> valueMap = getValueMap(rs);
 419
 420                         Integer categoryFk = (Integer)valueMap.get("refCategoryFk".toLowerCase());
 421                         refId = (Integer)valueMap.get("refId".toLowerCase());
 422                         Boolean thesisFlag = (Boolean)valueMap.get("thesisFlag".toLowerCase());
 423
 424
 425                         Reference<?> referenceBase;
 426                         logger.debug("RefCategoryFk: " + categoryFk);
 427
 428                         if (thesisFlag){
 429                                 referenceBase = makeThesis(valueMap);
 430                         }else if (categoryFk == REF_JOURNAL){
 431                                 referenceBase = makeJournal(valueMap);
 432                         }else if(categoryFk == REF_BOOK){
 433                                 referenceBase = makeBook(valueMap, refToSave, relatedReferences);
 434                         }else if(categoryFk == REF_DATABASE){
 435                                 referenceBase = makeDatabase(valueMap);
 436                         }else if(categoryFk == REF_INFORMAL){
 437                                 referenceBase = makeInformal(valueMap);
 438                         }else if(categoryFk == REF_WEBSITE){
 439                                 referenceBase = makeWebSite(valueMap);
 440                         }else if(categoryFk == REF_UNKNOWN){
 441                                 referenceBase = makeUnknown(valueMap);
 442                         }else if(categoryFk == REF_PRINT_SERIES){
 443                                 referenceBase = makePrintSeries(valueMap);
 444                         }else if(categoryFk == REF_CONFERENCE_PROCEEDINGS){
 445                                 referenceBase = makeProceedings(valueMap);
 446                         }else if(categoryFk == REF_ARTICLE){
 447                                 referenceBase = makeArticle(valueMap, refToSave, relatedReferences);
 448                         }else if(categoryFk == REF_JOURNAL_VOLUME){
 449                                 referenceBase = makeJournalVolume(valueMap);
 450                         }else if(categoryFk == REF_PART_OF_OTHER_TITLE){
 451                                 referenceBase = makePartOfOtherTitle(valueMap, refToSave, relatedReferences);
 452                         }else{
 453                                 logger.warn("Unknown categoryFk (" + categoryFk + "). Create 'Generic instead'");
 454                                 referenceBase = ReferenceFactory.newGeneric();
 455                                 success = false;
 456                         }
 457
 458                         //refYear
 459                         String refYear = (String)valueMap.get("refYear".toLowerCase());
 460                         referenceBase.setDatePublished(ImportHelper.getDatePublished(refYear));
 461
 462                         //created, updated, notes
 463                         doCreatedUpdatedNotes(state, referenceBase, rs);
 464
 465                         //idInSource
 466                         String idInSource = (String)valueMap.get("IdInSource".toLowerCase());
 467                         if (isNotBlank(idInSource)){
 468                                 IdentifiableSource source = IdentifiableSource.NewDataImportInstance(idInSource);
 469                                 source.setIdNamespace("import to Berlin Model");
 470                                 referenceBase.addSource(source);
 471                         }
 472
 473                         //nom&BiblioReference  - must be last because a clone is created
 474                         success &= makeNomAndBiblioReference(rs, state, partitioner, refId, referenceBase, refCounter, refToSave);
 475
 476
 477                 } catch (Exception e) {
 478                         logger.warn("Reference with BM refId '" + CdmUtils.Nz(refId) +  "' threw Exception and could not be saved");
 479                         e.printStackTrace();
 480                         success = false;
 481                 }
 482                 return success;
 483         }
 484
 485
 486         /**
 487          * Creates and saves a nom. reference and a biblio. reference after checking necessity
 488          * @param rs
 489          * @param refId
 490          * @param ref
 491          * @param refCounter
 492          * @param biblioRefToSave
 493          * @param nomRefToSave
 494          * @param teamMap
 495          * @param stores
 496          * @return
 497          * @throws SQLException
 498          */
 499         private boolean makeNomAndBiblioReference(
 500                                 ResultSet rs,
 501                                 BerlinModelImportState state,
 502                                 ResultSetPartitioner partitioner,
 503                                 int refId,
 504                                 Reference<?> ref,
 505                                 RefCounter refCounter,
 506                                 Map<Integer, Reference> refToSave
 507                                 ) throws SQLException{
 508
 509                 Map<String, Team> teamMap = partitioner.getObjectMap(BerlinModelAuthorTeamImport.NAMESPACE);
 510
 511                 String refCache = rs.getString("refCache");
 512                 String nomRefCache = rs.getString("nomRefCache");
 513                 String title = rs.getString("title");
 514                 String nomTitleAbbrev = rs.getString("nomTitleAbbrev");
 515                 boolean isPreliminary = rs.getBoolean("PreliminaryFlag");
 516                 String refAuthorString = rs.getString("refAuthorString");
 517                 Integer nomAuthorTeamFk = rs.getInt("NomAuthorTeamFk");
 518                 String strNomAuthorTeamFk = String.valueOf(nomAuthorTeamFk);
 519                 TeamOrPersonBase<?> nomAuthor = teamMap.get(strNomAuthorTeamFk);
 520
 521                 Reference<?> sourceReference = state.getTransactionalSourceReference();
 522
 523                 //preliminary
 524                 if (isPreliminary){
 525                         ref.setAbbrevTitleCache(nomRefCache, true);
 526                         ref.setTitleCache(refCache, true);
 527                 }
 528
 529                 //title/abbrevTitle
 530                 if (StringUtils.isNotBlank(nomTitleAbbrev)){
 531                         ref.setAbbrevTitle(nomTitleAbbrev);
 532                 }
 533                 if (StringUtils.isNotBlank(title)){
 534                         ref.setTitle(title);
 535                 }
 536
 537                 //author
 538                 TeamOrPersonBase<?> author = getAuthorTeam(refAuthorString , nomAuthor);
 539                 ref.setAuthorTeam(author);
 540
 541                 //save
 542                 if (! refToSave.containsKey(refId)){
 543                         refToSave.put(refId, ref);
 544                 }else{
 545                         logger.warn("Duplicate refId in Berlin Model database. Second reference was not imported !!");
 546                 }
 547                 refCounter.refCount++;
 548
 549                 //refId
 550                 ImportHelper.setOriginalSource(ref, sourceReference, refId, REFERENCE_NAMESPACE);
 551
 552                 return true;
 553         }
 554
 555         /**
 556          * Copies the created and updated information from the nomReference to the cloned bibliographic reference
 557          * @param referenceBase
 558          * @param nomReference
 559          */
 560         private void copyCreatedUpdated(Reference<?> biblioReference, Reference nomReference) {
 561                 biblioReference.setCreatedBy(nomReference.getCreatedBy());
 562                 biblioReference.setCreated(nomReference.getCreated());
 563                 biblioReference.setUpdatedBy(nomReference.getUpdatedBy());
 564                 biblioReference.setUpdated(nomReference.getUpdated());
 565
 566         }
 567
 568         private Reference<?> makeArticle (Map<String, Object> valueMap, Map<Integer, Reference> refToSave, Map<String, Reference> relatedReferences){
 569
 570                 IArticle article = ReferenceFactory.newArticle();
 571                 Object inRefFk = valueMap.get("inRefFk".toLowerCase());
 572                 Integer inRefCategoryFk = (Integer)valueMap.get("inRefCategoryFk".toLowerCase());
 573                 Integer refId = (Integer)valueMap.get("refId".toLowerCase());
 574
 575                 if (inRefFk != null){
 576                         if (inRefCategoryFk != REF_JOURNAL){
 577                                 logger.warn("Wrong inrefCategory for Article (refID = " + refId +"). Type must be 'Journal' but was not (RefCategoryFk=" + inRefCategoryFk + "))." +
 578                                         " InReference was added anyway! ");
 579                         }
 580                 }else{
 581                         logger.warn ("Article has no inreference: " + refId);
 582                 }
 583                 makeStandardMapper(valueMap, (Reference)article); //url, pages, series, volume
 584                 return (Reference)article;
 585         }
 586
 587         private Reference<?> makePartOfOtherTitle (Map<String, Object> valueMap,
 588                         Map<Integer, Reference> refToSave,
 589                         Map<String, Reference> relatedReferences){
 590
 591                 Reference<?> result;
 592                 Object inRefFk = valueMap.get("inRefFk".toLowerCase());
 593                 Integer inRefCategoryFk = (Integer)valueMap.get("inRefCategoryFk".toLowerCase());
 594                 Integer refId = (Integer)valueMap.get("refId".toLowerCase());
 595
 596                 if (inRefCategoryFk == null){
 597                         //null -> error
 598                         logger.warn("Part-Of-Other-Title has no inRefCategoryFk! RefId = " + refId + ". ReferenceType set to Generic.");
 599                         result = makeUnknown(valueMap);
 600                 }else if (inRefFk == null){
 601                         logger.warn("Part-Of-Other-Title has in in reference: " + refId);
 602                         result = makeUnknown(valueMap);
 603                 }else if (inRefCategoryFk == REF_BOOK){
 604                         //BookSection
 605                         IBookSection bookSection = ReferenceFactory.newBookSection();
 606                         result = (Reference<?>)bookSection;
 607                 }else if (inRefCategoryFk == REF_ARTICLE){
 608                         //Article
 609                         //TODO
 610                         logger.info("Reference (refId = " + refId + ") of type 'part_of_other_title' is part of 'article'." +
 611                                         " There is no specific reference type for such in references yet. Generic reference created instead") ;
 612                         result = ReferenceFactory.newGeneric();
 613                 }else if (inRefCategoryFk == REF_JOURNAL){
 614                         //TODO
 615                         logger.warn("Reference (refId = " + refId + ") of type 'part_of_other_title' has inReference of type 'journal'." +
 616                                         " This is not allowed! Generic reference created instead") ;
 617                         result = ReferenceFactory.newGeneric();
 618                         result.addMarker(Marker.NewInstance(MarkerType.TO_BE_CHECKED(), true));
 619                 }else{
 620                         logger.warn("InReference type (catFk = " + inRefCategoryFk + ") of part-of-reference not recognized for refId " + refId + "." +
 621                                 " Create 'Generic' reference instead");
 622                         result = ReferenceFactory.newGeneric();
 623                 }
 624                 makeStandardMapper(valueMap, result); //url, pages
 625                 return result;
 626         }
 627
 628
 629         /**
 630          * @param inRefFkInt
 631          * @param biblioRefToSave
 632          * @param nomRefToSave
 633          * @param relatedBiblioReferences
 634          * @param relatedNomReferences
 635          * @return
 636          */
 637         private boolean existsInMapOrToSave(Integer inRefFkInt, Map<Integer, Reference> refToSave, Map<String, Reference> relatedReferences) {
 638                 boolean result = false;
 639                 if (inRefFkInt == null){
 640                         return false;
 641                 }
 642                 result |= refToSave.containsKey(inRefFkInt);
 643                 result |= relatedReferences.containsKey(String.valueOf(inRefFkInt));
 644                 return result;
 645         }
 646
 647         private Reference<?> makeWebSite(Map<String, Object> valueMap){
 648                 if (logger.isDebugEnabled()){logger.debug("RefType 'Website'");}
 649                 Reference<?> webPage = ReferenceFactory.newWebPage();
 650                 makeStandardMapper(valueMap, webPage); //placePublished, publisher
 651                 return webPage;
 652         }
 653
 654         private Reference<?> makeUnknown(Map<String, Object> valueMap){
 655                 if (logger.isDebugEnabled()){logger.debug("RefType 'Unknown'");}
 656                 Reference<?> generic = ReferenceFactory.newGeneric();
 657 //              generic.setSeries(series);
 658                 makeStandardMapper(valueMap, generic); //pages, placePublished, publisher, series, volume
 659                 return generic;
 660         }
 661
 662         private Reference<?> makeInformal(Map<String, Object> valueMap){
 663                 if (logger.isDebugEnabled()){logger.debug("RefType 'Informal'");}
 664                 Reference<?> generic = ReferenceFactory.newGeneric();
 665 //              informal.setSeries(series);
 666                 makeStandardMapper(valueMap, generic);//editor, pages, placePublished, publisher, series, volume
 667                 String informal = (String)valueMap.get("InformalRefCategory".toLowerCase());
 668                 if (CdmUtils.isNotEmpty(informal) ){
 669                         generic.addExtension(informal, ExtensionType.INFORMAL_CATEGORY());
 670                 }
 671                 return generic;
 672         }
 673
 674         private Reference<?> makeDatabase(Map<String, Object> valueMap){
 675                 if (logger.isDebugEnabled()){logger.debug("RefType 'Database'");}
 676                 Reference database =  ReferenceFactory.newDatabase();
 677                 makeStandardMapper(valueMap, database); //?
 678                 return database;
 679         }
 680
 681         private Reference<?> makeJournal(Map<String, Object> valueMap){
 682                 if (logger.isDebugEnabled()){logger.debug("RefType 'Journal'");}
 683                 Reference journal = ReferenceFactory.newJournal();
 684
 685                 Set<String> omitAttributes = new HashSet<String>();
 686                 String series = "series";
 687 //              omitAttributes.add(series);
 688
 689                 makeStandardMapper(valueMap, journal, omitAttributes); //issn,placePublished,publisher
 690 //              if (valueMap.get(series) != null){
 691 //                      logger.warn("Series not yet implemented for journal!");
 692 //              }
 693                 return journal;
 694         }
 695
 696         private Reference<?> makeBook(
 697                                 Map<String, Object> valueMap,
 698                                 Map<Integer, Reference> refToSave,
 699                                 Map<String, Reference> relatedReferences){
 700                 if (logger.isDebugEnabled()){logger.debug("RefType 'Book'");}
 701                 Reference<?> book = ReferenceFactory.newBook();
 702                 Integer refId = (Integer)valueMap.get("refId".toLowerCase());
 703
 704                 //Set bookAttributes = new String[]{"edition", "isbn", "pages","publicationTown","publisher","volume"};
 705
 706                 Set<String> omitAttributes = new HashSet<String>();
 707                 String attrSeries = "series";
 708 //              omitAttributes.add(attrSeries);
 709
 710                 makeStandardMapper(valueMap, book, omitAttributes);
 711
 712                 //Series (as String)
 713                 IPrintSeries printSeries = null;
 714                 if (valueMap.get(attrSeries) != null){
 715                         String series = (String)valueMap.get("title".toLowerCase());
 716                         if (series == null){
 717                                 String nomTitle = (String)valueMap.get("nomTitleAbbrev".toLowerCase());
 718                                 series = nomTitle;
 719                         }
 720                         printSeries = ReferenceFactory.newPrintSeries(series);
 721                         logger.info("Implementation of printSeries is preliminary");
 722                 }
 723                 //Series (as Reference)
 724                 if (book.getInSeries() != null && printSeries != null){
 725                         logger.warn("Book has series string and inSeries reference. Can not take both. Series string neglected");
 726                 }else{
 727                         book.setInSeries(printSeries);
 728                 }
 729                 book.setEditor(null);
 730                 return book;
 731
 732         }
 733
 734         /**
 735          * Returns the requested object if it exists in one of both maps. Prefers the refToSaveMap in ambigious cases.
 736          * @param inRefFkInt
 737          * @param nomRefToSave
 738          * @param relatedNomReferences
 739          * @return
 740          */
 741         private Reference<?> getReferenceFromMaps(
 742                         int inRefFkInt,
 743                         Map<Integer, Reference> refToSaveMap,
 744                         Map<String, Reference> relatedRefMap) {
 745                 Reference<?> result = null;
 746                 result = refToSaveMap.get(inRefFkInt);
 747                 if (result == null){
 748                         result = relatedRefMap.get(String.valueOf(inRefFkInt));
 749                 }
 750                 return result;
 751         }
 752
 753         private Reference<?> makePrintSeries(Map<String, Object> valueMap){
 754                 if (logger.isDebugEnabled()){logger.debug("RefType 'PrintSeries'");}
 755                 Reference<?> printSeries = ReferenceFactory.newPrintSeries();
 756                 makeStandardMapper(valueMap, printSeries, null);
 757                 return printSeries;
 758         }
 759
 760         private Reference<?> makeProceedings(Map<String, Object> valueMap){
 761                 if (logger.isDebugEnabled()){logger.debug("RefType 'Proceedings'");}
 762                 Reference<?> proceedings = ReferenceFactory.newProceedings();
 763                 makeStandardMapper(valueMap, proceedings, null);
 764                 return proceedings;
 765         }
 766
 767         private Reference<?> makeThesis(Map<String, Object> valueMap){
 768                 if (logger.isDebugEnabled()){logger.debug("RefType 'Thesis'");}
 769                 Reference<?> thesis = ReferenceFactory.newThesis();
 770                 makeStandardMapper(valueMap, thesis, null);
 771                 return thesis;
 772         }
 773
 774
 775         private Reference<?> makeJournalVolume(Map<String, Object> valueMap){
 776                 if (logger.isDebugEnabled()){logger.debug("RefType 'JournalVolume'");}
 777                 //Proceedings proceedings = Proceedings.NewInstance();
 778                 Reference<?> journalVolume = ReferenceFactory.newGeneric();
 779                 makeStandardMapper(valueMap, journalVolume, null);
 780                 logger.warn("Journal volumes not yet implemented. Generic created instead but with errors");
 781                 return journalVolume;
 782         }
 783
 784         private boolean makeStandardMapper(Map<String, Object> valueMap, Reference<?> ref){
 785                 return makeStandardMapper(valueMap, ref, null);
 786         }
 787
 788
 789         private boolean makeStandardMapper(Map<String, Object> valueMap, CdmBase cdmBase, Set<String> omitAttributes){
 790                 boolean result = true;
 791                 for (CdmAttributeMapperBase mapper : classMappers){
 792                         if (mapper instanceof CdmSingleAttributeMapperBase){
 793                                 result &= makeStandardSingleMapper(valueMap, cdmBase, (CdmSingleAttributeMapperBase)mapper, omitAttributes);
 794                         }else if (mapper instanceof CdmOneToManyMapper){
 795                                 result &= makeMultipleValueAddMapper(valueMap, cdmBase, (CdmOneToManyMapper)mapper, omitAttributes);
 796                         }else{
 797                                 logger.error("Unknown mapper type");
 798                                 result = false;
 799                         }
 800                 }
 801                 return result;
 802         }
 803
 804         private boolean makeStandardSingleMapper(Map<String, Object> valueMap, CdmBase cdmBase, CdmSingleAttributeMapperBase mapper, Set<String> omitAttributes){
 805                 boolean result = true;
 806                 if (omitAttributes == null){
 807                         omitAttributes = new HashSet<String>();
 808                 }
 809                 if (mapper instanceof DbImportExtensionMapper){
 810                         result &= ((DbImportExtensionMapper)mapper).invoke(valueMap, cdmBase);
 811                 }else if (mapper instanceof DbImportMarkerMapper){
 812                         result &= ((DbImportMarkerMapper)mapper).invoke(valueMap, cdmBase);
 813                 }else{
 814                         String sourceAttribute = mapper.getSourceAttributeList().get(0).toLowerCase();
 815                         Object value = valueMap.get(sourceAttribute);
 816                         if (mapper instanceof CdmUriMapper && value != null){
 817                                 try {
 818                                         value = new URI (value.toString());
 819                                 } catch (URISyntaxException e) {
 820                                         logger.error("URI syntax exception: " + value.toString());
 821                                         value = null;
 822                                 }
 823                         }
 824                         if (value != null){
 825                                 String destinationAttribute = mapper.getDestinationAttribute();
 826                                 if (! omitAttributes.contains(destinationAttribute)){
 827                                         result &= ImportHelper.addValue(value, cdmBase, destinationAttribute, mapper.getTypeClass(), OVERWRITE, OBLIGATORY);
 828                                 }
 829                         }
 830                 }
 831                 return result;
 832         }
 833
 834
 835         private boolean makeMultipleValueAddMapper(Map<String, Object> valueMap, CdmBase cdmBase, CdmOneToManyMapper<CdmBase, CdmBase, CdmSingleAttributeMapperBase> mapper, Set<String> omitAttributes){
 836                 if (omitAttributes == null){
 837                         omitAttributes = new HashSet<String>();
 838                 }
 839                 boolean result = true;
 840                 String destinationAttribute = mapper.getSingleAttributeName();
 841                 List<Object> sourceValues = new ArrayList<Object>();
 842                 List<Class> classes = new ArrayList<Class>();
 843                 for (CdmSingleAttributeMapperBase singleMapper : mapper.getSingleMappers()){
 844                         String sourceAttribute = singleMapper.getSourceAttribute();
 845                         Object value = valueMap.get(sourceAttribute);
 846                         sourceValues.add(value);
 847                         Class<?> clazz = singleMapper.getTypeClass();
 848                         classes.add(clazz);
 849                 }
 850
 851                 result &= ImportHelper.addMultipleValues(sourceValues, cdmBase, destinationAttribute, classes, NO_OVERWRITE, OBLIGATORY);
 852                 return result;
 853         }
 854
 855
 856         private static TeamOrPersonBase<?> getAuthorTeam(String authorString, TeamOrPersonBase<?> nomAuthor){
 857                 TeamOrPersonBase<?> result;
 858                 if (nomAuthor != null){
 859                         result = nomAuthor;
 860                 } else if (StringUtils.isNotBlank(authorString)){
 861 //                      xx;
 862                         //FIXME check for existing team / persons
 863                         TeamOrPersonBase<?> team = Team.NewInstance();
 864                         team.setNomenclaturalTitle(authorString);
 865                         team.setTitleCache(authorString, true);
 866                         team.setNomenclaturalTitle(authorString);
 867                         result = team;
 868                 }else{
 869                         result = null;
 870                 }
 871
 872                 return result;
 873         }
 874
 875
 876         /**
 877          * @param lowerCase
 878          * @param config
 879          * @return
 880          */
 881         public Set<String> getObligatoryAttributes(boolean lowerCase, BerlinModelImportConfigurator config){
 882                 Set<String> result = new HashSet<String>();
 883                 Class<ICdmIO>[] ioClassList = config.getIoClassList();
 884                 logger.warn("getObligatoryAttributes has been commented because it still needs to be adapted to the new package structure");
 885                 result.addAll(Arrays.asList(unclearMappers));
 886                 result.addAll(Arrays.asList(createdAndNotesAttributes));
 887                 result.addAll(Arrays.asList(operationalAttributes));
 888                 CdmIoMapping mapping = new CdmIoMapping();
 889                 for (CdmAttributeMapperBase mapper : classMappers){
 890                         mapping.addMapper(mapper);
 891                 }
 892                 result.addAll(mapping.getSourceAttributes());
 893                 if (lowerCase){
 894                         Set<String> lowerCaseResult = new HashSet<String>();
 895                         for (String str : result){
 896                                 if (str != null){lowerCaseResult.add(str.toLowerCase());}
 897                         }
 898                         result = lowerCaseResult;
 899                 }
 900                 return result;
 901         }
 902
 903         /* (non-Javadoc)
 904          * @see eu.etaxonomy.cdm.io.common.CdmIoBase#doCheck(eu.etaxonomy.cdm.io.common.IoStateBase)
 905          */
 906         @Override
 907         protected boolean doCheck(BerlinModelImportState state){
 908                 BerlinModelReferenceImportValidator validator = new BerlinModelReferenceImportValidator();
 909                 return validator.validate(state, this);
 910         }
 911
 912
 913         /* (non-Javadoc)
 914          * @see eu.etaxonomy.cdm.io.common.CdmIoBase#isIgnore(eu.etaxonomy.cdm.io.common.IImportConfigurator)
 915          */
 916         protected boolean isIgnore(BerlinModelImportState state){
 917                 return (state.getConfig().getDoReferences() == IImportConfigurator.DO_REFERENCES.NONE);
 918         }
 919
 920
 921
 922
 923 }