app-import/src/main/java/eu/etaxonomy/cdm/io/redlist/gefaesspflanzen/RedListGefaesspflanzenImportNames.java

   1 /**
   2  * Copyright (C) 2007 EDIT
   3  * European Distributed Institute of Taxonomy
   4  * http://www.e-taxonomy.eu
   5  *
   6  * The contents of this file are subject to the Mozilla Public License Version 1.1
   7  * See LICENSE.TXT at the top of this package for the full license terms.
   8  */
   9
  10 package eu.etaxonomy.cdm.io.redlist.gefaesspflanzen;
  11
  12 import java.sql.ResultSet;
  13 import java.sql.SQLException;
  14 import java.util.HashMap;
  15 import java.util.HashSet;
  16 import java.util.Map;
  17 import java.util.Set;
  18
  19 import org.apache.log4j.Logger;
  20 import org.springframework.stereotype.Component;
  21
  22 import eu.etaxonomy.cdm.common.CdmUtils;
  23 import eu.etaxonomy.cdm.io.common.DbImportBase;
  24 import eu.etaxonomy.cdm.io.common.IPartitionedIO;
  25 import eu.etaxonomy.cdm.io.common.ImportHelper;
  26 import eu.etaxonomy.cdm.io.common.ResultSetPartitioner;
  27 import eu.etaxonomy.cdm.io.common.mapping.UndefinedTransformerMethodException;
  28 import eu.etaxonomy.cdm.model.agent.AgentBase;
  29 import eu.etaxonomy.cdm.model.agent.TeamOrPersonBase;
  30 import eu.etaxonomy.cdm.model.common.CdmBase;
  31 import eu.etaxonomy.cdm.model.name.BotanicalName;
  32 import eu.etaxonomy.cdm.model.name.NomenclaturalStatus;
  33 import eu.etaxonomy.cdm.model.name.NomenclaturalStatusType;
  34 import eu.etaxonomy.cdm.model.name.Rank;
  35 import eu.etaxonomy.cdm.model.name.TaxonNameBase;
  36 import eu.etaxonomy.cdm.model.taxon.Synonym;
  37 import eu.etaxonomy.cdm.model.taxon.Taxon;
  38 import eu.etaxonomy.cdm.model.taxon.TaxonBase;
  39 import eu.etaxonomy.cdm.model.taxon.TaxonRelationship;
  40 import eu.etaxonomy.cdm.model.taxon.TaxonRelationshipType;
  41
  42 /**
  43  *
  44  * @author pplitzner
  45  * @date Mar 1, 2016
  46  *
  47  */
  48
  49 @Component
  50 @SuppressWarnings("serial")
  51 public class RedListGefaesspflanzenImportNames extends DbImportBase<RedListGefaesspflanzenImportState, RedListGefaesspflanzenImportConfigurator> {
  52
  53     private static final Logger logger = Logger.getLogger(RedListGefaesspflanzenImportNames.class);
  54
  55     private static final String tableName = "Rote Liste Gefäßpflanzen";
  56
  57     private static final String pluralString = "names";
  58
  59     public RedListGefaesspflanzenImportNames() {
  60         super(tableName, pluralString);
  61     }
  62
  63     @Override
  64     protected String getIdQuery(RedListGefaesspflanzenImportState state) {
  65         return "SELECT NAMNR "
  66                 + "FROM V_TAXATLAS_D20_EXPORT t "
  67                 + " ORDER BY NAMNR";
  68     }
  69
  70     @Override
  71     protected String getRecordQuery(RedListGefaesspflanzenImportConfigurator config) {
  72         String result = " SELECT * "
  73                 + " FROM V_TAXATLAS_D20_EXPORT t "
  74                 + " WHERE t.NAMNR IN (@IDSET)";
  75         result = result.replace("@IDSET", IPartitionedIO.ID_LIST_TOKEN);
  76         return result;
  77     }
  78
  79     @Override
  80     protected void doInvoke(RedListGefaesspflanzenImportState state) {
  81         super.doInvoke(state);
  82     }
  83
  84
  85     @Override
  86     public boolean doPartition(ResultSetPartitioner partitioner, RedListGefaesspflanzenImportState state) {
  87         ResultSet rs = partitioner.getResultSet();
  88         Set<TaxonNameBase> namesToSave = new HashSet<TaxonNameBase>();
  89         Set<TaxonBase> taxaToSave = new HashSet<TaxonBase>();
  90         try {
  91             while (rs.next()){
  92                 makeSingleNameAndTaxon(state, rs, namesToSave, taxaToSave);
  93
  94             }
  95         } catch (SQLException e) {
  96             e.printStackTrace();
  97         }
  98
  99         getNameService().saveOrUpdate(namesToSave);
 100         getTaxonService().saveOrUpdate(taxaToSave);
 101         return true;
 102     }
 103
 104     private void makeSingleNameAndTaxon(RedListGefaesspflanzenImportState state, ResultSet rs, Set<TaxonNameBase> namesToSave, Set<TaxonBase> taxaToSave)
 105             throws SQLException {
 106         //cell values
 107         long id = rs.getLong(RedListUtil.NAMNR);
 108         String taxNameString = rs.getString(RedListUtil.TAXNAME);
 109         String gueltString = rs.getString(RedListUtil.GUELT);
 110         String rangString = rs.getString(RedListUtil.RANG);
 111         String ep1String = rs.getString(RedListUtil.EPI1);
 112         String ep2String = rs.getString(RedListUtil.EPI2);
 113         String ep3String = rs.getString(RedListUtil.EPI3);
 114         String nomZusatzString = rs.getString(RedListUtil.NOM_ZUSATZ);
 115         String taxZusatzString = rs.getString(RedListUtil.TAX_ZUSATZ);
 116         String zusatzString = rs.getString(RedListUtil.ZUSATZ);
 117         String nonString = rs.getString(RedListUtil.NON);
 118         String sensuString = rs.getString(RedListUtil.SENSU);
 119         String authorKombString = rs.getString(RedListUtil.AUTOR_KOMB);
 120         String authorBasiString = rs.getString(RedListUtil.AUTOR_BASI);
 121         String hybString = rs.getString(RedListUtil.HYB);
 122         String clTaxonString = rs.getString(RedListUtil.CL_TAXON);
 123         String relationE = rs.getString(RedListUtil.E);
 124         String relationW = rs.getString(RedListUtil.W);
 125         String relationK = rs.getString(RedListUtil.K);
 126         String relationAW = rs.getString(RedListUtil.AW);
 127         String relationAO = rs.getString(RedListUtil.AO);
 128         String relationR = rs.getString(RedListUtil.R);
 129         String relationO = rs.getString(RedListUtil.O);
 130         String relationS = rs.getString(RedListUtil.S);
 131
 132         //---NAME---
 133         BotanicalName name = importName(state, id, taxNameString, rangString, ep1String, ep2String, ep3String,
 134                 nomZusatzString, hybString, namesToSave);
 135
 136
 137         //--- AUTHORS ---
 138         importAuthors(state, rs, id, nomZusatzString, taxZusatzString, zusatzString, authorKombString,
 139                 authorBasiString, name);
 140
 141         //---TAXON---
 142         TaxonBase taxonBase = importTaxon(id, taxNameString, gueltString, authorBasiString, hybString, name);
 143         if(taxonBase==null){
 144             RedListUtil.logMessage(id, "Taxon for name "+name+" could not be created.", logger);
 145             return;
 146         }
 147
 148         //---CONCEPT RELATIONSHIPS---
 149         /*check if taxon/synonym also exists in other classification
 150          * 1. create new taxon with the same name (in that classification)
 151          * 2. create concept relationship between both
 152          */
 153         //checklist
 154         if(CdmUtils.isNotBlank(clTaxonString) && !clTaxonString.trim().equals("-")){
 155             cloneTaxon(taxonBase, name, TaxonRelationshipType.CONGRUENT_TO(), taxaToSave, id, RedListUtil.TAXON_CHECKLISTE_NAMESPACE, false, true, state);
 156         }
 157         //E, W, K, AW, AO, R, O, S
 158         addConceptRelation(relationE, RedListUtil.CLASSIFICATION_NAMESPACE_E, taxonBase, name, taxaToSave, id, state);
 159         addConceptRelation(relationW, RedListUtil.CLASSIFICATION_NAMESPACE_W, taxonBase, name, taxaToSave, id, state);
 160         addConceptRelation(relationK, RedListUtil.CLASSIFICATION_NAMESPACE_K, taxonBase, name, taxaToSave, id, state);
 161         addConceptRelation(relationAW, RedListUtil.CLASSIFICATION_NAMESPACE_AW, taxonBase, name, taxaToSave, id, state);
 162         addConceptRelation(relationAO, RedListUtil.CLASSIFICATION_NAMESPACE_AO, taxonBase, name, taxaToSave, id, state);
 163         addConceptRelation(relationR, RedListUtil.CLASSIFICATION_NAMESPACE_R, taxonBase, name, taxaToSave, id, state);
 164         addConceptRelation(relationO, RedListUtil.CLASSIFICATION_NAMESPACE_O, taxonBase, name, taxaToSave, id, state);
 165         addConceptRelation(relationS, RedListUtil.CLASSIFICATION_NAMESPACE_S, taxonBase, name, taxaToSave, id, state);
 166
 167         //NOTE: the source has to be added after cloning or otherwise the clone would also get the source
 168         ImportHelper.setOriginalSource(taxonBase, state.getTransactionalSourceReference(), id, RedListUtil.TAXON_GESAMTLISTE_NAMESPACE);
 169         taxaToSave.add(taxonBase);
 170     }
 171
 172     private void addConceptRelation(String relationString, String classificationNamespace, TaxonBase taxonBase, TaxonNameBase name, Set<TaxonBase> taxaToSave, long id, RedListGefaesspflanzenImportState state){
 173         if(CdmUtils.isNotBlank(relationString) && !relationString.equals(".")){
 174             String substring = relationString.substring(relationString.length()-1, relationString.length());
 175             TaxonRelationshipType taxonRelationshipTypeByKey = new RedListGefaesspflanzenTransformer().getTaxonRelationshipTypeByKey(substring);
 176             if(taxonRelationshipTypeByKey==null){
 177                 RedListUtil.logMessage(id, "Could not interpret relationship "+relationString+" for taxon "+taxonBase.generateTitle(), logger);
 178             }
 179             //there is no type "included in" so we have to reverse the direction
 180             if(substring.equals("<")){
 181                 cloneTaxon(taxonBase, name, taxonRelationshipTypeByKey, taxaToSave, id, classificationNamespace, true, false, state);
 182             }
 183             else{
 184                 cloneTaxon(taxonBase, name, taxonRelationshipTypeByKey, taxaToSave, id, classificationNamespace, false, false, state);
 185             }
 186         }
 187     }
 188
 189     /**
 190      * <b>NOTE:</b> the {@link TaxonRelationshipType} passed as parameter is
 191      * directed <b>from the clone</b> to the taxon.<br>
 192      * This can be changed with parameter <i>reverseRelation</i>
 193      */
 194     private void cloneTaxon(TaxonBase taxonBase, TaxonNameBase name, TaxonRelationshipType relationFromCloneToTaxon, Set<TaxonBase> taxaToSave, long id, String sourceNameSpace, boolean reverseRelation, boolean doubtful, RedListGefaesspflanzenImportState state){
 195         TaxonBase clone = (TaxonBase) taxonBase.clone();
 196         clone.setName(name);
 197         if(taxonBase.isInstanceOf(Taxon.class)){
 198             TaxonRelationship taxonRelation;
 199             if(reverseRelation){
 200                 taxonRelation = ((Taxon) taxonBase).addTaxonRelation((Taxon) clone, relationFromCloneToTaxon, null, null);
 201             }
 202             else {
 203                 taxonRelation = ((Taxon) clone).addTaxonRelation((Taxon) taxonBase, relationFromCloneToTaxon, null, null);
 204             }
 205             taxonRelation.setDoubtful(doubtful);
 206         }
 207         ImportHelper.setOriginalSource(clone, state.getTransactionalSourceReference(), id, sourceNameSpace);
 208         taxaToSave.add(clone);
 209     }
 210
 211     private TaxonBase importTaxon(long id, String taxNameString, String gueltString, String authorBasiString,
 212             String hybString, BotanicalName name) {
 213         TaxonBase taxonBase = null;
 214         if(authorBasiString.trim().contains(RedListUtil.AUCT)){
 215             taxonBase = Taxon.NewInstance(name, null);
 216             taxonBase.setAppendedPhrase(RedListUtil.AUCT);
 217         }
 218         else if(gueltString.equals(RedListUtil.GUELT_ACCEPTED_TAXON)){
 219             taxonBase = Taxon.NewInstance(name, null);
 220         }
 221         else if(gueltString.equals(RedListUtil.GUELT_SYNONYM) || gueltString.equals(RedListUtil.GUELT_BASIONYM)){
 222             taxonBase = Synonym.NewInstance(name, null);
 223         }
 224         else{
 225             return null;
 226         }
 227
 228         //check taxon name consistency
 229         checkTaxonNameConsistency(id, taxNameString, hybString, taxonBase);
 230         return taxonBase;
 231     }
 232
 233     private void importAuthors(RedListGefaesspflanzenImportState state, ResultSet rs, long id, String nomZusatzString,
 234             String taxZusatzString, String zusatzString, String authorKombString, String authorBasiString,
 235             BotanicalName name) throws SQLException {
 236         //combination author
 237         if(authorKombString.contains(RedListUtil.EX)){
 238             //TODO: what happens with multiple ex authors??
 239             String[] kombSplit = authorKombString.split(RedListUtil.EX);
 240             if(kombSplit.length!=2){
 241                 RedListUtil.logMessage(id, "Multiple ex combination authors found", logger);
 242             }
 243             for (int i = 0; i < kombSplit.length; i++) {
 244                 if(i==0){
 245                     //first author is ex author
 246                     TeamOrPersonBase authorKomb = (TeamOrPersonBase) state.getRelatedObject(RedListUtil.AUTHOR_NAMESPACE, kombSplit[i]);
 247                     name.setExCombinationAuthorship(authorKomb);
 248                 }
 249                 else{
 250                     TeamOrPersonBase authorKomb = (TeamOrPersonBase) state.getRelatedObject(RedListUtil.AUTHOR_NAMESPACE, kombSplit[i]);
 251                     name.setCombinationAuthorship(authorKomb);
 252                 }
 253             }
 254         }
 255         else if(authorKombString.trim().contains(RedListUtil.AUCT)){
 256             RedListUtil.logMessage(id, "AUCT information in "+RedListUtil.AUTOR_KOMB+" column", logger);
 257         }
 258         else if(CdmUtils.isNotBlank(authorKombString)){
 259             TeamOrPersonBase authorKomb = (TeamOrPersonBase) state.getRelatedObject(RedListUtil.AUTHOR_NAMESPACE, authorKombString);
 260             name.setCombinationAuthorship(authorKomb);
 261         }
 262         //basionym author
 263         if(authorBasiString.contains(RedListUtil.EX)){
 264             String[] basiSplit = authorBasiString.split(RedListUtil.EX);
 265             for (int i = 0; i < basiSplit.length; i++) {
 266                 if(basiSplit.length!=2){
 267                     RedListUtil.logMessage(id, "Multiple ex basionymn authors found", logger);
 268                 }
 269                 if(i==0){
 270                     TeamOrPersonBase authorBasi= (TeamOrPersonBase) state.getRelatedObject(RedListUtil.AUTHOR_NAMESPACE, basiSplit[i]);
 271                     if(CdmUtils.isBlank(authorKombString)){
 272                         name.setExCombinationAuthorship(authorBasi);
 273                     }
 274                     else{
 275                         name.setExBasionymAuthorship(authorBasi);
 276                     }
 277                 }
 278                 else{
 279                     TeamOrPersonBase authorBasi= (TeamOrPersonBase) state.getRelatedObject(RedListUtil.AUTHOR_NAMESPACE, basiSplit[i]);
 280                     if(CdmUtils.isBlank(authorKombString)){
 281                         name.setCombinationAuthorship(authorBasi);
 282                     }
 283                     else{
 284                         name.setBasionymAuthorship(authorBasi);
 285                     }
 286                 }
 287             }
 288         }
 289         else if(CdmUtils.isNotBlank(authorBasiString)){
 290             //this seems to be a convention in the source database: When there is only a single author then only the "AUTOR_BASI" column is used
 291             TeamOrPersonBase authorBasi= (TeamOrPersonBase) state.getRelatedObject(RedListUtil.AUTHOR_NAMESPACE, authorBasiString);
 292             if(CdmUtils.isBlank(authorKombString)){
 293                 name.setCombinationAuthorship(authorBasi);
 294             }
 295             else{
 296                 name.setBasionymAuthorship(authorBasi);
 297             }
 298         }
 299
 300         //check authorship consistency
 301         String authorString = rs.getString(RedListUtil.AUTOR);
 302         String authorshipCache = name.getAuthorshipCache();
 303         checkAuthorShipConsistency(id, nomZusatzString, taxZusatzString, zusatzString, authorString, authorshipCache);
 304     }
 305
 306     private BotanicalName importName(RedListGefaesspflanzenImportState state, long id, String taxNameString,
 307             String rangString, String ep1String, String ep2String, String ep3String, String nomZusatzString,
 308             String hybString, Set<TaxonNameBase> namesToSave) {
 309         if(CdmUtils.isBlank(taxNameString) && CdmUtils.isBlank(ep1String)){
 310             RedListUtil.logMessage(id, "No name found!", logger);
 311         }
 312
 313         Rank rank = makeRank(id, state, rangString);
 314         BotanicalName name = BotanicalName.NewInstance(rank);
 315
 316         //ep1 should always be present
 317         if(CdmUtils.isBlank(ep1String)){
 318             RedListUtil.logMessage(id, RedListUtil.EPI1+" is empty!", logger);
 319         }
 320         name.setGenusOrUninomial(ep1String);
 321         if(CdmUtils.isNotBlank(ep2String)){
 322             name.setSpecificEpithet(ep2String);
 323         }
 324         if(CdmUtils.isNotBlank(ep3String)){
 325             if(rank==Rank.SUBSPECIES() ||
 326                     rank==Rank.VARIETY()){
 327                 name.setInfraSpecificEpithet(ep3String);
 328             }
 329         }
 330         //nomenclatural status
 331         if(CdmUtils.isNotBlank(nomZusatzString)){
 332             NomenclaturalStatusType status = makeNomenclaturalStatus(id, state, nomZusatzString);
 333             if(status!=null){
 334                 name.addStatus(NomenclaturalStatus.NewInstance(status));
 335             }
 336         }
 337         //hybrid
 338         if(hybString.equals(RedListUtil.HYB_X)){
 339             name.setBinomHybrid(true);
 340         }
 341         else if(hybString.equals(RedListUtil.HYB_XF)){
 342             name.setTrinomHybrid(true);
 343         }
 344         //add source
 345         ImportHelper.setOriginalSource(name, state.getTransactionalSourceReference(), id, RedListUtil.NAME_NAMESPACE);
 346
 347         namesToSave.add(name);
 348         return name;
 349     }
 350
 351     private void checkAuthorShipConsistency(long id, String nomZusatzString, String taxZusatzString,
 352             String zusatzString, String authorString, String authorshipCache) {
 353         if(CdmUtils.isNotBlank(zusatzString)){
 354             authorString = authorString.replace(", "+zusatzString, "");
 355         }
 356         if(CdmUtils.isNotBlank(nomZusatzString)){
 357             authorString = authorString.replace(", "+nomZusatzString, "");
 358         }
 359         if(CdmUtils.isNotBlank(taxZusatzString)){
 360             authorString = authorString.replace(", "+taxZusatzString, "");
 361         }
 362         if(authorString.equals(RedListUtil.AUCT)){
 363             authorString = "";
 364         }
 365         if(!authorString.equals(authorshipCache)){
 366             RedListUtil.logMessage(id, "Authorship inconsistent! name.authorhshipCache <-> Column "+RedListUtil.AUTOR+": "+authorshipCache+" <-> "+authorString, logger);
 367         }
 368     }
 369
 370     private void checkTaxonNameConsistency(long id, String taxNameString, String hybString, TaxonBase taxonBase) {
 371         String nameCache = ((BotanicalName)taxonBase.getName()).getNameCache().trim();
 372
 373         if(taxNameString.endsWith("agg.")){
 374             taxNameString = taxNameString.replace("agg.", "aggr.");
 375         }
 376         if(hybString.equalsIgnoreCase(RedListUtil.HYB_X)){
 377             taxNameString = taxNameString.replace("× ", "×");//hybrid sign has no space after it in titleCache for binomial hybrids
 378         }
 379         if(taxNameString.endsWith(Rank.SPECIESGROUP().toString())){
 380             taxNameString.replaceAll(Rank.SPECIESGROUP().toString(), "- Gruppe");
 381             if(!taxNameString.trim().equals(nameCache)){
 382                 taxNameString.replaceAll(Rank.SPECIESGROUP().toString(), "- group");
 383             }
 384         }
 385         if(!taxNameString.trim().equals(nameCache)){
 386             RedListUtil.logMessage(id, "Taxon name inconsistent! taxon.titleCache <-> Column "+RedListUtil.TAXNAME+": "+nameCache+" <-> "+taxNameString, logger);
 387         }
 388     }
 389
 390     private Rank makeRank(long id, RedListGefaesspflanzenImportState state, String rankStr) {
 391         Rank rank = null;
 392         try {
 393             rank = state.getTransformer().getRankByKey(rankStr);
 394         } catch (UndefinedTransformerMethodException e) {
 395             e.printStackTrace();
 396         }
 397         if(rank==null){
 398             RedListUtil.logMessage(id, rankStr+" could not be associated to a known rank.", logger);
 399         }
 400         return rank;
 401     }
 402
 403     private NomenclaturalStatusType makeNomenclaturalStatus(long id, RedListGefaesspflanzenImportState state, String nomZusatzString) {
 404         NomenclaturalStatusType status = null;
 405         try {
 406             status = state.getTransformer().getNomenclaturalStatusByKey(nomZusatzString);
 407         } catch (UndefinedTransformerMethodException e) {
 408             e.printStackTrace();
 409         }
 410         if(status==null){
 411             RedListUtil.logMessage(id, nomZusatzString+" could not be associated to a known nomenclatural status.", logger);
 412         }
 413         return status;
 414     }
 415
 416
 417
 418     @Override
 419     public Map<Object, Map<String, ? extends CdmBase>> getRelatedObjectsForPartition(ResultSet rs,
 420             RedListGefaesspflanzenImportState state) {
 421         Map<Object, Map<String, ? extends CdmBase>> result = new HashMap<>();
 422         Map<String, AgentBase<?>> authorMap = new HashMap<String, AgentBase<?>>();
 423
 424         try {
 425             while (rs.next()){
 426                 String authorKombString = rs.getString(RedListUtil.AUTOR_KOMB);
 427
 428                 if(authorKombString.contains(RedListUtil.EX)){
 429                     String[] kombSplit = authorKombString.split(RedListUtil.EX);
 430                     for (int i = 0; i < kombSplit.length; i++) {
 431                         if(!authorMap.containsKey(kombSplit[i])){
 432                             authorMap.put(kombSplit[i], getAgentService().load(state.getAuthorMap().get(kombSplit[i])));
 433                         }
 434                     }
 435                 }
 436                 else if(CdmUtils.isNotBlank(authorKombString) && !authorMap.containsKey(authorKombString)){
 437                     authorMap.put(authorKombString, getAgentService().load(state.getAuthorMap().get(authorKombString)));
 438                 }
 439
 440                 String authorBasiString = rs.getString(RedListUtil.AUTOR_BASI);
 441                 //basionym author
 442                 if(authorBasiString.contains(RedListUtil.EX)){
 443                     String[] basiSplit = authorBasiString.split(RedListUtil.EX);
 444                     for (int i = 0; i < basiSplit.length; i++) {
 445                         if(!authorMap.containsKey(basiSplit[i])){
 446                             authorMap.put(basiSplit[i], getAgentService().load(state.getAuthorMap().get(basiSplit[i])));
 447                         }
 448                     }
 449                 }
 450                 else if(CdmUtils.isNotBlank(authorBasiString) && !authorMap.containsKey(authorBasiString)){
 451                     authorMap.put(authorBasiString, getAgentService().load(state.getAuthorMap().get(authorBasiString)));
 452                 }
 453             }
 454         } catch (SQLException e) {
 455             e.printStackTrace();
 456         }
 457         result.put(RedListUtil.AUTHOR_NAMESPACE, authorMap);
 458
 459         return result;
 460     }
 461
 462     @Override
 463     protected boolean doCheck(RedListGefaesspflanzenImportState state) {
 464         return false;
 465     }
 466
 467     @Override
 468     protected boolean isIgnore(RedListGefaesspflanzenImportState state) {
 469         return false;
 470     }
 471
 472 }