cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/berlinModel/in/BerlinModelTaxonRelationImport.java

   1 /**
   2 * Copyright (C) 2007 EDIT
   3 * European Distributed Institute of Taxonomy
   4 * http://www.e-taxonomy.eu
   5 *
   6 * The contents of this file are subject to the Mozilla Public License Version 1.1
   7 * See LICENSE.TXT at the top of this package for the full license terms.
   8 */
   9
  10 package eu.etaxonomy.cdm.io.berlinModel.in;
  11
  12 import static eu.etaxonomy.cdm.io.berlinModel.BerlinModelTransformer.TAX_REL_IS_HETEROTYPIC_SYNONYM_OF;
  13 import static eu.etaxonomy.cdm.io.berlinModel.BerlinModelTransformer.TAX_REL_IS_HOMOTYPIC_SYNONYM_OF;
  14 import static eu.etaxonomy.cdm.io.berlinModel.BerlinModelTransformer.TAX_REL_IS_INCLUDED_IN;
  15 import static eu.etaxonomy.cdm.io.berlinModel.BerlinModelTransformer.TAX_REL_IS_MISAPPLIED_NAME_OF;
  16 import static eu.etaxonomy.cdm.io.berlinModel.BerlinModelTransformer.TAX_REL_IS_PARTIAL_HETEROTYPIC_SYNONYM_OF;
  17 import static eu.etaxonomy.cdm.io.berlinModel.BerlinModelTransformer.TAX_REL_IS_PARTIAL_HOMOTYPIC_SYNONYM_OF;
  18 import static eu.etaxonomy.cdm.io.berlinModel.BerlinModelTransformer.TAX_REL_IS_PARTIAL_SYN_OF;
  19 import static eu.etaxonomy.cdm.io.berlinModel.BerlinModelTransformer.TAX_REL_IS_PROPARTE_HETEROTYPIC_SYNONYM_OF;
  20 import static eu.etaxonomy.cdm.io.berlinModel.BerlinModelTransformer.TAX_REL_IS_PROPARTE_HOMOTYPIC_SYNONYM_OF;
  21 import static eu.etaxonomy.cdm.io.berlinModel.BerlinModelTransformer.TAX_REL_IS_PROPARTE_SYN_OF;
  22 import static eu.etaxonomy.cdm.io.berlinModel.BerlinModelTransformer.TAX_REL_IS_SYNONYM_OF;
  23
  24 import java.sql.ResultSet;
  25 import java.sql.SQLException;
  26 import java.util.HashMap;
  27 import java.util.HashSet;
  28 import java.util.Map;
  29 import java.util.Set;
  30 import java.util.UUID;
  31
  32 import org.apache.commons.lang.StringUtils;
  33 import org.apache.log4j.Logger;
  34 import org.springframework.stereotype.Component;
  35 import org.springframework.transaction.TransactionStatus;
  36
  37 import eu.etaxonomy.cdm.common.ResultWrapper;
  38 import eu.etaxonomy.cdm.io.berlinModel.BerlinModelTransformer;
  39 import eu.etaxonomy.cdm.io.berlinModel.in.validation.BerlinModelTaxonRelationImportValidator;
  40 import eu.etaxonomy.cdm.io.common.IOValidator;
  41 import eu.etaxonomy.cdm.io.common.ResultSetPartitioner;
  42 import eu.etaxonomy.cdm.io.common.Source;
  43 import eu.etaxonomy.cdm.model.common.AnnotatableEntity;
  44 import eu.etaxonomy.cdm.model.common.CdmBase;
  45 import eu.etaxonomy.cdm.model.common.IdentifiableSource;
  46 import eu.etaxonomy.cdm.model.reference.Reference;
  47 import eu.etaxonomy.cdm.model.taxon.Synonym;
  48 import eu.etaxonomy.cdm.model.taxon.SynonymRelationship;
  49 import eu.etaxonomy.cdm.model.taxon.SynonymRelationshipType;
  50 import eu.etaxonomy.cdm.model.taxon.Taxon;
  51 import eu.etaxonomy.cdm.model.taxon.TaxonBase;
  52 import eu.etaxonomy.cdm.model.taxon.TaxonNode;
  53 import eu.etaxonomy.cdm.model.taxon.TaxonRelationshipType;
  54 import eu.etaxonomy.cdm.model.taxon.Classification;
  55 import eu.etaxonomy.cdm.strategy.exceptions.UnknownCdmTypeException;
  56
  57 /**
  58  * @author a.mueller
  59  * @created 20.03.2008
  60  * @version 1.0
  61  */
  62 @Component
  63 public class BerlinModelTaxonRelationImport  extends BerlinModelImportBase  {
  64         private static final Logger logger = Logger.getLogger(BerlinModelTaxonRelationImport.class);
  65
  66         public static final String TREE_NAMESPACE = "PTRefFk";
  67
  68         private static int modCount = 30000;
  69         private static final String pluralString = "taxon relations";
  70         private static final String dbTableName = "RelPTaxon";
  71
  72
  73         public BerlinModelTaxonRelationImport(){
  74                 super(dbTableName, pluralString);
  75         }
  76
  77         /**
  78          * Creates a classification for each PTaxon reference which belongs to a taxon that is included at least in one
  79          * <i>taxonomically included</i> relationship
  80          * @param state
  81          * @return
  82          * @throws SQLException
  83          */
  84         private void makeClassifications(BerlinModelImportState state) throws SQLException{
  85                 logger.info("start make classification ...");
  86
  87                 Set<String> idSet = getTreeReferenceIdSet(state);
  88
  89                 //nom reference map
  90                 String nameSpace = BerlinModelReferenceImport.NOM_REFERENCE_NAMESPACE;
  91                 Class cdmClass = Reference.class;
  92 //                      idSet = new HashSet<String>();
  93                 Map<String, Reference> nomRefMap = (Map<String, Reference>)getCommonService().getSourcedObjectsByIdInSource(cdmClass, idSet, nameSpace);
  94
  95                 //biblio reference map
  96                 nameSpace = BerlinModelReferenceImport.BIBLIO_REFERENCE_NAMESPACE;
  97                 cdmClass = Reference.class;
  98 //              idSet = new HashSet<String>();
  99                 Map<String, Reference> biblioRefMap = (Map<String, Reference>)getCommonService().getSourcedObjectsByIdInSource(cdmClass, idSet, nameSpace);
 100
 101                 String treeName = "Classification - No Name";
 102
 103                 ResultSet rs = state.getConfig().getSource().getResultSet(getClassificationQuery(state)) ;
 104                 int i = 0;
 105                 //for each reference
 106                 try {
 107                         //TODO handle case useSingleClassification = true && sourceSecId = null, which returns no record
 108                         while (rs.next()){
 109
 110                                 try {
 111                                         if ((i++ % modCount) == 0 && i!= 1 ){ logger.info("RelPTaxa handled: " + (i-1));}
 112
 113                                         Integer ptRefFkInt = nullSafeInt(rs,"PTRefFk");
 114                                         String ptRefFk= String.valueOf(ptRefFkInt);
 115                                         Reference<?> ref = getReferenceOnlyFromMaps(biblioRefMap, nomRefMap, ptRefFk);
 116
 117                                         String refCache = rs.getString("RefCache");
 118                                         if (StringUtils.isNotBlank(refCache)){
 119                                                 treeName = refCache;
 120                                         }
 121                                         if (ref != null && StringUtils.isNotBlank(ref.getTitleCache())){
 122                                                 treeName = ref.getTitleCache();
 123                                         }
 124                                         Classification tree = Classification.NewInstance(treeName);
 125                                         tree.setReference(ref);
 126                                         if (i == 1 && state.getConfig().getClassificationUuid() != null){
 127                                                 tree.setUuid(state.getConfig().getClassificationUuid());
 128                                         }
 129                                         IdentifiableSource identifiableSource = IdentifiableSource.NewInstance(ptRefFk, TREE_NAMESPACE);
 130                                         tree.addSource(identifiableSource);
 131
 132                                         getClassificationService().save(tree);
 133                                         state.putClassificationUuidInt(ptRefFkInt, tree);
 134                                 } catch (Exception e) {
 135                                         logger.error("Error in BerlinModleTaxonRelationImport.makeClassifications: " + e.getMessage());
 136                                         e.printStackTrace();
 137                                 }
 138                         }
 139                 } catch (SQLException e) {
 140                         logger.error("Error in BerlinModleTaxonRelationImport.makeClassifications: " + e.getMessage());
 141                         throw e;
 142                 }
 143                 logger.info("end make classification ...");
 144
 145                 return;
 146         }
 147
 148         /**
 149          * @return
 150          * @throws SQLException
 151          */
 152         private Set<String> getTreeReferenceIdSet(BerlinModelImportState state) throws SQLException {
 153                 Source source = state.getConfig().getSource();
 154                 Set<String> result = new HashSet<String>();
 155                 ResultSet rs = source.getResultSet(getClassificationQuery(state)) ;
 156                 while (rs.next()){
 157                         Object id = rs.getObject("PTRefFk");
 158                         result.add(String.valueOf(id));
 159                 }
 160                 return result;
 161         }
 162
 163         /**
 164          * @return
 165          */
 166         private String getClassificationQuery(BerlinModelImportState state) {
 167                 boolean includeAllClassifications = state.getConfig().isIncludeAllNonMisappliedRelatedClassifications();
 168                 String strQuerySelect = "SELECT PTaxon.PTRefFk, r.RefCache ";
 169                 String strQueryFrom = " FROM RelPTaxon " +
 170                                                         " INNER JOIN PTaxon AS PTaxon ON RelPTaxon.PTNameFk2 = PTaxon.PTNameFk AND RelPTaxon.PTRefFk2 = PTaxon.PTRefFk " +
 171                                                         " INNER JOIN Reference r ON PTaxon.PTRefFk = r.RefId ";
 172                 String strQueryWhere = " WHERE (RelPTaxon.RelQualifierFk = 1) ";
 173                 if (includeAllClassifications){
 174                         strQueryWhere = " WHERE (RelPTaxon.RelQualifierFk <> 3) ";
 175                 }else{
 176                         if (state.getConfig().isUseSingleClassification()){
 177                                 if (state.getConfig().getSourceSecId()!= null){
 178                                         strQueryWhere += " AND PTaxon.PTRefFk = " + state.getConfig().getSourceSecId() +  " ";
 179                                 }else{
 180                                         strQueryWhere += " AND (1=0) ";
 181                                 }
 182                         }
 183                 }
 184
 185                 String strQueryGroupBy = " GROUP BY PTaxon.PTRefFk, r.RefCache ";
 186                 String strQuery = strQuerySelect + " " + strQueryFrom + " " + strQueryWhere + " " + strQueryGroupBy;
 187
 188
 189                 if (includeAllClassifications){
 190                         //add otherdirection
 191                         strQuerySelect = "SELECT PTaxon.PTRefFk, r.RefCache ";
 192                         strQueryFrom = " FROM RelPTaxon rel " +
 193                                                                 " INNER JOIN PTaxon AS PTaxon ON rel.PTNameFk1 = PTaxon.PTNameFk AND rel.PTRefFk1 = PTaxon.PTRefFk " +
 194                                                                 " INNER JOIN Reference r ON PTaxon.PTRefFk = r.RefId ";
 195                         strQueryWhere =" WHERE (rel.RelQualifierFk <> 3) ";
 196                         String strAllQuery =  strQuerySelect + " " + strQueryFrom + " " + strQueryWhere + " " + strQueryGroupBy;
 197                         strQuery = strQuery + " UNION " + strAllQuery;
 198                 }
 199
 200
 201
 202                 boolean includeFlatClassifications = state.getConfig().isIncludeFlatClassifications();
 203                 //concepts with
 204                 if (includeFlatClassifications){
 205                         String strFlatQuery =
 206                                         " SELECT pt.PTRefFk AS secRefFk, r.RefCache AS secRef " +
 207                                         " FROM PTaxon AS pt LEFT OUTER JOIN " +
 208                                                   " Reference r ON pt.PTRefFk = r.RefId LEFT OUTER JOIN " +
 209                                                   " RelPTaxon rel1 ON pt.PTNameFk = rel1.PTNameFk2 AND pt.PTRefFk = rel1.PTRefFk2 LEFT OUTER JOIN " +
 210                                                   " RelPTaxon AS rel2 ON pt.PTNameFk = rel2.PTNameFk1 AND pt.PTRefFk = rel2.PTRefFk1 " +
 211                                         " WHERE (rel2.RelQualifierFk IS NULL) AND (rel1.RelQualifierFk IS NULL) " +
 212                                         " GROUP BY pt.PTRefFk, r.RefCache "
 213                                         ;
 214
 215                         strQuery = strQuery + " UNION " + strFlatQuery;
 216                 }
 217
 218
 219
 220                 if (state.getConfig().getClassificationQuery() != null){
 221                         strQuery = state.getConfig().getClassificationQuery();
 222                 }
 223                 return strQuery;
 224         }
 225
 226
 227         /* (non-Javadoc)
 228          * @see eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelImportBase#getRecordQuery(eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelImportConfigurator)
 229          */
 230         @Override
 231         protected String getRecordQuery(BerlinModelImportConfigurator config) {
 232                 String strQuery =
 233                         " SELECT RelPTaxon.*, FromTaxon.RIdentifier as taxon1Id, ToTaxon.RIdentifier as taxon2Id, ToTaxon.PTRefFk as treeRefFk, FromTaxon.PTRefFk as fromRefFk, q.is_concept_relation " +
 234                         " FROM PTaxon as FromTaxon " +
 235                 " INNER JOIN RelPTaxon ON FromTaxon.PTNameFk = RelPTaxon.PTNameFk1 AND FromTaxon.PTRefFk = RelPTaxon.PTRefFk1 " +
 236                 " INNER JOIN PTaxon AS ToTaxon ON RelPTaxon.PTNameFk2 = ToTaxon.PTNameFk AND RelPTaxon.PTRefFk2 = ToTaxon.PTRefFk " +
 237                 " INNER JOIN RelPTQualifier q ON q.RelPTQualifierId = RelPTaxon.RelQualifierFk " +
 238             " WHERE RelPTaxon.RelPTaxonId IN ("+ID_LIST_TOKEN+") ";
 239                 return strQuery;
 240         }
 241
 242         /* (non-Javadoc)
 243          * @see eu.etaxonomy.cdm.io.berlinModel.in.IPartitionedIO#doPartition(eu.etaxonomy.cdm.io.berlinModel.in.ResultSetPartitioner, eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelImportState)
 244          */
 245         public boolean doPartition(ResultSetPartitioner partitioner, BerlinModelImportState state) {
 246                 boolean success = true ;
 247                 BerlinModelImportConfigurator config = state.getConfig();
 248                 Set<TaxonBase> taxaToSave = new HashSet<TaxonBase>();
 249                 Map<String, TaxonBase> taxonMap = (Map<String, TaxonBase>) partitioner.getObjectMap(BerlinModelTaxonImport.NAMESPACE);
 250                 Map<Integer, Classification> classificationMap = new HashMap<Integer, Classification>();
 251                 Map<String, Reference> biblioRefMap = (Map<String, Reference>) partitioner.getObjectMap(BerlinModelReferenceImport.BIBLIO_REFERENCE_NAMESPACE);
 252                 Map<String, Reference> nomRefMap = (Map<String, Reference>) partitioner.getObjectMap(BerlinModelReferenceImport.NOM_REFERENCE_NAMESPACE);
 253
 254                 ResultSet rs = partitioner.getResultSet();
 255
 256                 try{
 257                         int i = 0;
 258                         //for each reference
 259                         while (rs.next()){
 260
 261                                 if ((i++ % modCount) == 0 && i!= 1 ){ logger.info("RelPTaxa handled: " + (i-1));}
 262
 263                                 int relPTaxonId = rs.getInt("RelPTaxonId");
 264                                 Integer taxon1Id = nullSafeInt(rs, "taxon1Id");
 265                                 Integer taxon2Id = nullSafeInt(rs, "taxon2Id");
 266                                 try {
 267                                         Integer relRefFk = nullSafeInt(rs,"relRefFk");
 268                                         int treeRefFk = rs.getInt("treeRefFk");
 269                                         int fromRefFk = rs.getInt("fromRefFk");
 270
 271                                         int relQualifierFk = rs.getInt("relQualifierFk");
 272                                         String notes = rs.getString("notes");
 273                                         boolean isConceptRelationship = rs.getBoolean("is_concept_relation");
 274
 275                                         TaxonBase taxon1 = taxonMap.get(String.valueOf(taxon1Id));
 276                                         TaxonBase taxon2 = taxonMap.get(String.valueOf(taxon2Id));
 277
 278                                         String refFk = String.valueOf(relRefFk);
 279                                         Reference citation = getReferenceOnlyFromMaps(biblioRefMap,     nomRefMap, refFk);
 280
 281                                         String microcitation = null; //does not exist in RelPTaxon
 282
 283                                         if (taxon2 != null && taxon1 != null){
 284                                                 if (!(taxon2 instanceof Taxon)){
 285                                                         logger.error("ToTaxon (ID = " + taxon2.getId()+ ", RIdentifier = " + taxon2Id + ") can't be casted to Taxon. RelPTaxon: " + relPTaxonId );
 286                                                         success = false;
 287                                                         continue;
 288                                                 }
 289                                                 AnnotatableEntity taxonRelationship = null;
 290                                                 Taxon toTaxon = (Taxon)taxon2;
 291                                                 if (isTaxonRelationship(relQualifierFk)){
 292                                                         if (!(taxon1 instanceof Taxon)){
 293                                                                 logger.error("TaxonBase (ID = " + taxon1.getId()+ ", RIdentifier = " + taxon1Id + ") for TaxonRelation ("+relPTaxonId+") can't be casted to Taxon");
 294                                                                 success = false;
 295                                                                 continue;
 296                                                         }
 297                                                         Taxon fromTaxon = (Taxon)taxon1;
 298                                                         if (relQualifierFk == TAX_REL_IS_INCLUDED_IN){
 299                                                                 taxonRelationship = makeTaxonomicallyIncluded(state, classificationMap, treeRefFk, fromTaxon, toTaxon, citation, microcitation);
 300                                                         }else if (relQualifierFk == TAX_REL_IS_MISAPPLIED_NAME_OF){
 301                                                                  taxonRelationship = toTaxon.addMisappliedName(fromTaxon, citation, microcitation);
 302                                                         }else{
 303                                                                 handleAllRelatedTaxa(state, fromTaxon, classificationMap, fromRefFk);
 304                                                                 handleAllRelatedTaxa(state, toTaxon, classificationMap, treeRefFk);
 305                                                                 logger.warn("Unhandled taxon relationship: RelId:" + relPTaxonId + "; QualifierId: " + relQualifierFk);
 306                                                         }
 307                                                 }else if (isSynonymRelationship(relQualifierFk)){
 308                                                         if (!(taxon1 instanceof Synonym)){
 309                                                                 logger.warn("Validated: Taxon (ID = " + taxon1.getId()+ ", RIdentifier = " + taxon1Id + ") can't be casted to Synonym");
 310                                                                 success = false;
 311                                                                 continue;
 312                                                         }
 313                                                         handleAllRelatedTaxa(state, toTaxon, classificationMap, treeRefFk);
 314                                                         Synonym synonym = (Synonym)taxon1;
 315                                                         SynonymRelationship synRel = getSynRel(relQualifierFk, toTaxon, synonym, citation, microcitation);
 316                                                         taxonRelationship = synRel;
 317
 318                                                         if (relQualifierFk == TAX_REL_IS_SYNONYM_OF ||
 319                                                                         relQualifierFk == TAX_REL_IS_HOMOTYPIC_SYNONYM_OF ||
 320                                                                         relQualifierFk == TAX_REL_IS_HETEROTYPIC_SYNONYM_OF){
 321                                                                 addProParteAndPartial(synRel, synonym, config);
 322                                                         }else if (relQualifierFk == TAX_REL_IS_PROPARTE_SYN_OF ||
 323                                                                         relQualifierFk == TAX_REL_IS_PROPARTE_HOMOTYPIC_SYNONYM_OF ||
 324                                                                         relQualifierFk == TAX_REL_IS_PROPARTE_HETEROTYPIC_SYNONYM_OF ){
 325                                                                         synRel.setProParte(true);
 326                                                         }else if(relQualifierFk == TAX_REL_IS_PARTIAL_SYN_OF ||
 327                                                                         relQualifierFk == TAX_REL_IS_PARTIAL_HOMOTYPIC_SYNONYM_OF ||
 328                                                                         relQualifierFk == TAX_REL_IS_PARTIAL_HETEROTYPIC_SYNONYM_OF ){
 329                                                                         synRel.setPartial(true);
 330                                                         }else{
 331                                                                 success = false;
 332                                                                 logger.warn("Proparte/Partial not yet implemented for TaxonRelationShipType " + relQualifierFk);
 333                                                         }
 334                                                 }else if (isConceptRelationship){
 335                                                         ResultWrapper<Boolean> isInverse = new ResultWrapper<Boolean>();
 336                                                         try {
 337                                                                 TaxonRelationshipType relType = BerlinModelTransformer.taxonRelId2TaxonRelType(relQualifierFk, isInverse);
 338                                                                 if (! (taxon1 instanceof Taxon)){
 339                                                                         success = false;
 340                                                                         logger.error("TaxonBase (ID = " + taxon1.getId()+ ", RIdentifier = " + taxon1Id + ") can't be casted to Taxon");
 341                                                                 }else{
 342                                                                         Taxon fromTaxon = (Taxon)taxon1;
 343                                                                         taxonRelationship = fromTaxon.addTaxonRelation(toTaxon, relType, citation, microcitation);
 344                                                                         handleAllRelatedTaxa(state, toTaxon, classificationMap, treeRefFk);
 345                                                                         handleAllRelatedTaxa(state, fromTaxon, classificationMap, fromRefFk);
 346                                                                 }
 347                                                         } catch (UnknownCdmTypeException e) {
 348                                                                 //TODO other relationships
 349                                                                 logger.warn("TaxonRelationShipType " + relQualifierFk + " (conceptRelationship) not yet implemented");
 350                                                                 success = false;
 351                                                         }
 352                                                 }else {
 353                                                         //TODO
 354                                                         logger.warn("TaxonRelationShipType " + relQualifierFk + " not yet implemented: RelPTaxonId = " + relPTaxonId );
 355                                                         success = false;
 356                                                 }
 357
 358                                                 doNotes(taxonRelationship, notes);
 359                                                 taxaToSave.add(taxon2);
 360
 361                                                 //TODO
 362                                                 //etc.
 363                                         }else{
 364                                                 if (taxon2 != null && taxon1 == null){
 365                                                         logger.warn("First taxon ("+taxon1Id+") for RelPTaxon " + relPTaxonId + " does not exist in store. RelType: " + relQualifierFk);
 366                                                 }else if (taxon2 == null && taxon1 != null){
 367                                                         logger.warn("Second taxon ("+taxon2Id +") for RelPTaxon " + relPTaxonId + " does not exist in store. RelType: " + relQualifierFk);
 368                                                 }else{
 369                                                         logger.warn("Both taxa ("+taxon1Id+","+taxon2Id +") for RelPTaxon " + relPTaxonId + " do not exist in store. RelType: " + relQualifierFk);
 370                                                 }
 371
 372                                                 success = false;
 373                                         }
 374                                 } catch (Exception e) {
 375                                         logger.error("Exception occurred when trying to handle taxon relationship " + relPTaxonId + " (" + taxon1Id + ","+ taxon2Id + "): " + e.getMessage());
 376 //                                      e.printStackTrace();
 377                                 }
 378                         }
 379                 }catch(SQLException e){
 380                         throw new RuntimeException(e);
 381                 }
 382                 logger.info("Taxa to save: " + taxaToSave.size());
 383                 partitioner.startDoSave();
 384                 getTaxonService().save(taxaToSave);
 385                 classificationMap = null;
 386                 taxaToSave = null;
 387
 388                         return success;
 389         }
 390
 391
 392         private void handleAllRelatedTaxa(BerlinModelImportState state, Taxon taxon, Map<Integer, Classification> classificationMap, Integer secRefFk) {
 393                 if (taxon.getTaxonNodes().size() > 0){
 394                         return;
 395                 }else{
 396                         Classification classification = getClassificationTree(state, classificationMap, secRefFk);
 397                         classification.addChildTaxon(taxon, null, null, null);
 398                 }
 399
 400
 401
 402         }
 403
 404         /* (non-Javadoc)
 405          * @see eu.etaxonomy.cdm.io.common.CdmIoBase#doInvoke(eu.etaxonomy.cdm.io.common.IImportConfigurator, eu.etaxonomy.cdm.api.application.CdmApplicationController, java.util.Map)
 406          */
 407         @Override
 408         protected void doInvoke(BerlinModelImportState state){
 409                 try {
 410                         makeClassifications(state);
 411                         super.doInvoke(state);
 412                         makeFlatClassificationTaxa(state);
 413                         return;
 414                 } catch (SQLException e) {
 415                         throw new RuntimeException(e);
 416                 }
 417
 418         }
 419
 420
 421         private void makeFlatClassificationTaxa(BerlinModelImportState state) {
 422                 //Note: this part still does not use partitions
 423                 logger.info("Flat classifications start");
 424                 TransactionStatus txStatus = startTransaction();
 425                 if (! state.getConfig().isIncludeFlatClassifications()){
 426                         return;
 427                 }
 428                 String sql = " SELECT pt.PTRefFk AS secRefFk, pt.RIdentifier " +
 429                                                 " FROM PTaxon AS pt " +
 430                                                         " LEFT OUTER JOIN RelPTaxon ON pt.PTNameFk = RelPTaxon.PTNameFk2 AND pt.PTRefFk = RelPTaxon.PTRefFk2 " +
 431                                                         "  LEFT OUTER JOIN RelPTaxon AS RelPTaxon_1 ON pt.PTNameFk = RelPTaxon_1.PTNameFk1 AND pt.PTRefFk = RelPTaxon_1.PTRefFk1 " +
 432                                                 " WHERE (RelPTaxon_1.RelQualifierFk IS NULL) AND (dbo.RelPTaxon.RelQualifierFk IS NULL) " +
 433                                                 " ORDER BY pt.PTRefFk " ;
 434                 ResultSet rs = state.getConfig().getSource().getResultSet(sql);
 435                 Map<Object, Map<String, ? extends CdmBase>> maps = getRelatedObjectsForFlatPartition(rs);
 436
 437                 Map<String, TaxonBase> taxonMap = (Map<String, TaxonBase>) maps.get(BerlinModelTaxonImport.NAMESPACE);
 438                 Map<Integer, Classification> classificationMap = new HashMap<Integer, Classification>();
 439
 440                 rs = state.getConfig().getSource().getResultSet(sql);
 441                 try {
 442                         while (rs.next()){
 443                                 Integer treeRefFk = rs.getInt("secRefFk");
 444                                 String taxonId = rs.getString("RIdentifier");
 445                                 Classification classification = getClassificationTree(state, classificationMap, treeRefFk);
 446                                 TaxonBase<?> taxon = taxonMap.get(taxonId);
 447                                 if (taxon.isInstanceOf(Taxon.class)){
 448                                         classification.addChildTaxon(CdmBase.deproxy(taxon, Taxon.class), null, null, null);
 449                                 }else{
 450                                         String message = "TaxonBase for taxon is not of class Taxon but %s (RIdentifier %s)";
 451                                         logger.warn(String.format(message, taxon.getClass(), taxonId));
 452                                 }
 453                         }
 454                 } catch (SQLException e) {
 455                         // TODO Auto-generated catch block
 456                         e.printStackTrace();
 457                 }
 458                 commitTransaction(txStatus);
 459                 logger.info("Flat classifications end");
 460
 461         }
 462
 463         /* (non-Javadoc)
 464          * @see eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelImportBase#getIdQuery(eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelImportState)
 465          */
 466         @Override
 467         protected String getIdQuery(BerlinModelImportState state) {
 468                 if (state.getConfig().getRelTaxaIdQuery() != null){
 469                         return state.getConfig().getRelTaxaIdQuery();
 470                 }else{
 471                         return super.getIdQuery(state);
 472                 }
 473         }
 474
 475
 476         /* (non-Javadoc)
 477          * @see eu.etaxonomy.cdm.io.berlinModel.in.IPartitionedIO#getRelatedObjectsForPartition(java.sql.ResultSet)
 478          */
 479         public Map<Object, Map<String, ? extends CdmBase>> getRelatedObjectsForPartition( ResultSet rs) {
 480                 String nameSpace;
 481                 Class cdmClass;
 482                 Set<String> idSet;
 483                 Map<Object, Map<String, ? extends CdmBase>> result = new HashMap<Object, Map<String, ? extends CdmBase>>();
 484
 485                 try{
 486                         Set<String> taxonIdSet = new HashSet<String>();
 487                         Set<String> referenceIdSet = new HashSet<String>();
 488 //                      Set<String> classificationIdSet = new HashSet<String>();
 489                         while (rs.next()){
 490                                 handleForeignKey(rs, taxonIdSet, "taxon1Id");
 491                                 handleForeignKey(rs, taxonIdSet, "taxon2Id");
 492 //                              handleForeignKey(rs, classificationIdSet, "treeRefFk");
 493                                 handleForeignKey(rs, referenceIdSet, "RelRefFk");
 494         }
 495
 496                         //taxon map
 497                         nameSpace = BerlinModelTaxonImport.NAMESPACE;
 498                         cdmClass = TaxonBase.class;
 499                         idSet = taxonIdSet;
 500                         Map<String, TaxonBase> taxonMap = (Map<String, TaxonBase>)getCommonService().getSourcedObjectsByIdInSource(cdmClass, idSet, nameSpace);
 501                         result.put(nameSpace, taxonMap);
 502
 503 //                      //tree map
 504 //                      nameSpace = "Classification";
 505 //                      cdmClass = Classification.class;
 506 //                      idSet = classificationIdSet;
 507 //                      Map<String, Classification> treeMap = (Map<String, Classification>)getCommonService().getSourcedObjectsByIdInSource(cdmClass, idSet, nameSpace);
 508 //                      result.put(cdmClass, treeMap);
 509 //                      Set<UUID> treeUuidSet = state
 510 //                      getClassificationService().find(uuidSet);
 511 //
 512
 513                         //nom reference map
 514                         nameSpace = BerlinModelReferenceImport.NOM_REFERENCE_NAMESPACE;
 515                         cdmClass = Reference.class;
 516                         idSet = referenceIdSet;
 517                         Map<String, Reference> nomReferenceMap = (Map<String, Reference>)getCommonService().getSourcedObjectsByIdInSource(cdmClass, idSet, nameSpace);
 518                         result.put(nameSpace, nomReferenceMap);
 519
 520                         //biblio reference map
 521                         nameSpace = BerlinModelReferenceImport.BIBLIO_REFERENCE_NAMESPACE;
 522                         cdmClass = Reference.class;
 523                         idSet = referenceIdSet;
 524                         Map<String, Reference> biblioReferenceMap = (Map<String, Reference>)getCommonService().getSourcedObjectsByIdInSource(cdmClass, idSet, nameSpace);
 525                         result.put(nameSpace, biblioReferenceMap);
 526
 527                 } catch (SQLException e) {
 528                         throw new RuntimeException(e);
 529                 }
 530                 return result;
 531         }
 532
 533         /* (non-Javadoc)
 534          * @see eu.etaxonomy.cdm.io.berlinModel.in.IPartitionedIO#getRelatedObjectsForPartition(java.sql.ResultSet)
 535          */
 536         public Map<Object, Map<String, ? extends CdmBase>> getRelatedObjectsForFlatPartition( ResultSet rs) {
 537                 String nameSpace;
 538                 Class cdmClass;
 539                 Set<String> idSet;
 540                 Map<Object, Map<String, ? extends CdmBase>> result = new HashMap<Object, Map<String, ? extends CdmBase>>();
 541
 542                 try{
 543                         Set<String> taxonIdSet = new HashSet<String>();
 544                         Set<String> referenceIdSet = new HashSet<String>();
 545 //                      Set<String> classificationIdSet = new HashSet<String>();
 546                         while (rs.next()){
 547                                 handleForeignKey(rs, taxonIdSet, "RIdentifier");
 548 //                              handleForeignKey(rs, classificationIdSet, "treeRefFk");
 549         }
 550
 551                         //taxon map
 552                         nameSpace = BerlinModelTaxonImport.NAMESPACE;
 553                         cdmClass = TaxonBase.class;
 554                         idSet = taxonIdSet;
 555                         Map<String, TaxonBase> taxonMap = (Map<String, TaxonBase>)getCommonService().getSourcedObjectsByIdInSource(cdmClass, idSet, nameSpace);
 556                         result.put(nameSpace, taxonMap);
 557
 558 //                      //tree map
 559 //                      nameSpace = "Classification";
 560 //                      cdmClass = Classification.class;
 561 //                      idSet = classificationIdSet;
 562 //                      Map<String, Classification> treeMap = (Map<String, Classification>)getCommonService().getSourcedObjectsByIdInSource(cdmClass, idSet, nameSpace);
 563 //                      result.put(cdmClass, treeMap);
 564 //                      Set<UUID> treeUuidSet = state
 565 //                      getClassificationService().find(uuidSet);
 566 //
 567
 568                 } catch (SQLException e) {
 569                         throw new RuntimeException(e);
 570                 }
 571                 return result;
 572         }
 573
 574
 575         private SynonymRelationship getSynRel (int relQualifierFk, Taxon toTaxon, Synonym synonym, Reference citation, String microcitation){
 576                 SynonymRelationship result;
 577                 if (relQualifierFk == TAX_REL_IS_HOMOTYPIC_SYNONYM_OF ||
 578                                 relQualifierFk == TAX_REL_IS_PROPARTE_HOMOTYPIC_SYNONYM_OF ||
 579                                 relQualifierFk == TAX_REL_IS_PARTIAL_HOMOTYPIC_SYNONYM_OF){
 580                         result = toTaxon.addHomotypicSynonym(synonym, citation, microcitation);
 581                 }else if (relQualifierFk == TAX_REL_IS_HETEROTYPIC_SYNONYM_OF ||
 582                                 relQualifierFk == TAX_REL_IS_PROPARTE_HETEROTYPIC_SYNONYM_OF ||
 583                                 relQualifierFk == TAX_REL_IS_PARTIAL_HETEROTYPIC_SYNONYM_OF){
 584                         result = toTaxon.addSynonym(synonym, SynonymRelationshipType.HETEROTYPIC_SYNONYM_OF(), citation, microcitation);
 585                 }else if (relQualifierFk == TAX_REL_IS_SYNONYM_OF ||
 586                                 relQualifierFk == TAX_REL_IS_PROPARTE_SYN_OF ||
 587                                 relQualifierFk == TAX_REL_IS_PARTIAL_SYN_OF){
 588                         result = toTaxon.addSynonym(synonym, SynonymRelationshipType.SYNONYM_OF(), citation, microcitation);
 589                 }else{
 590                         logger.warn("SynonymyRelationShipType could not be defined for relQualifierFk " + relQualifierFk + ". 'Unknown'-Type taken instead.");
 591                         result = toTaxon.addSynonym(synonym, SynonymRelationshipType.SYNONYM_OF(), citation, microcitation);
 592                 }
 593                 return result;
 594
 595         }
 596
 597         private  boolean isSynonymRelationship(int relQualifierFk){
 598                 if (relQualifierFk == TAX_REL_IS_SYNONYM_OF ||
 599                         relQualifierFk == TAX_REL_IS_HOMOTYPIC_SYNONYM_OF ||
 600                         relQualifierFk == TAX_REL_IS_HETEROTYPIC_SYNONYM_OF ||
 601                         relQualifierFk == TAX_REL_IS_PROPARTE_SYN_OF ||
 602                         relQualifierFk == TAX_REL_IS_PARTIAL_SYN_OF ||
 603                         relQualifierFk == TAX_REL_IS_PROPARTE_HOMOTYPIC_SYNONYM_OF ||
 604                         relQualifierFk == TAX_REL_IS_PROPARTE_HETEROTYPIC_SYNONYM_OF ||
 605                         relQualifierFk == TAX_REL_IS_PARTIAL_HOMOTYPIC_SYNONYM_OF ||
 606                         relQualifierFk == TAX_REL_IS_PARTIAL_HETEROTYPIC_SYNONYM_OF
 607                 ){
 608                         return true;
 609                 }else{
 610                         return false;
 611                 }
 612         }
 613
 614         private  boolean isTaxonRelationship(int relQualifierFk){
 615                 if (relQualifierFk == TAX_REL_IS_INCLUDED_IN ||
 616                 relQualifierFk == TAX_REL_IS_MISAPPLIED_NAME_OF){
 617                         return true;
 618                 }else{
 619                         return false;
 620                 }
 621         }
 622
 623         private void addProParteAndPartial(SynonymRelationship synRel, Synonym synonym, BerlinModelImportConfigurator bmiConfig){
 624                 if (bmiConfig.isPartialSynonym(synonym)){
 625                         synRel.setPartial(true);
 626                 }
 627                 if (bmiConfig.isProParteSynonym(synonym)){
 628                         synRel.setProParte(true);
 629                 }
 630         }
 631
 632         private TaxonNode makeTaxonomicallyIncluded(BerlinModelImportState state, Map<Integer, Classification> classificationMap, int treeRefFk, Taxon child, Taxon parent, Reference citation, String microCitation){
 633                 Classification tree = getClassificationTree(state, classificationMap, treeRefFk);
 634                 return tree.addParentChild(parent, child, citation, microCitation);
 635         }
 636
 637         private Classification getClassificationTree(BerlinModelImportState state, Map<Integer, Classification> classificationMap, int treeRefFk) {
 638                 if (state.getConfig().isUseSingleClassification()){
 639                         if (state.getConfig().getSourceSecId() != null){
 640                                 treeRefFk = (Integer)state.getConfig().getSourceSecId();
 641                         }else{
 642                                 treeRefFk = 1;
 643                         }
 644
 645                 }
 646                 Classification tree = classificationMap.get(treeRefFk);
 647                 if (tree == null){
 648                         UUID treeUuid = state.getTreeUuidByIntTreeKey(treeRefFk);
 649                         tree = getClassificationService().find(treeUuid);
 650                         classificationMap.put(treeRefFk, tree);
 651                 }
 652                 if (tree == null){
 653                         throw new IllegalStateException("Tree for ToTaxon reference " + treeRefFk + " does not exist.");
 654                 }
 655                 return tree;
 656         }
 657
 658
 659         /* (non-Javadoc)
 660          * @see eu.etaxonomy.cdm.io.common.CdmIoBase#doCheck(eu.etaxonomy.cdm.io.common.IoStateBase)
 661          */
 662         @Override
 663         protected boolean doCheck(BerlinModelImportState state){
 664                 IOValidator<BerlinModelImportState> validator = new BerlinModelTaxonRelationImportValidator();
 665                 return validator.validate(state);
 666         }
 667
 668         /* (non-Javadoc)
 669          * @see eu.etaxonomy.cdm.io.common.CdmIoBase#isIgnore(eu.etaxonomy.cdm.io.common.IImportConfigurator)
 670          */
 671         protected boolean isIgnore(BerlinModelImportState state){
 672                 return ! state.getConfig().isDoRelTaxa();
 673         }
 674
 675
 676 }