app-import/src/main/java/eu/etaxonomy/cdm/app/cyprus/CyprusAltitudeActivator.java

   1 /**
   2 * Copyright (C) 2007 EDIT
   3 * European Distributed Institute of Taxonomy
   4 * http://www.e-taxonomy.eu
   5 *
   6 * The contents of this file are subject to the Mozilla Public License Version 1.1
   7 * See LICENSE.TXT at the top of this package for the full license terms.
   8 */
   9
  10 package eu.etaxonomy.cdm.app.cyprus;
  11
  12 import java.io.FileNotFoundException;
  13 import java.net.URI;
  14 import java.net.URISyntaxException;
  15 import java.util.ArrayList;
  16 import java.util.HashMap;
  17 import java.util.HashSet;
  18 import java.util.Set;
  19 import java.util.UUID;
  20
  21 import org.apache.commons.lang.StringUtils;
  22 import org.apache.log4j.Logger;
  23 import org.springframework.transaction.TransactionStatus;
  24
  25 import eu.etaxonomy.cdm.api.application.CdmApplicationController;
  26 import eu.etaxonomy.cdm.app.common.CdmDestinations;
  27 import eu.etaxonomy.cdm.common.ExcelUtils;
  28 import eu.etaxonomy.cdm.database.DbSchemaValidation;
  29 import eu.etaxonomy.cdm.database.ICdmDataSource;
  30 import eu.etaxonomy.cdm.io.api.application.CdmIoApplicationController;
  31 import eu.etaxonomy.cdm.io.common.IImportConfigurator.CHECK;
  32 import eu.etaxonomy.cdm.model.common.CdmBase;
  33 import eu.etaxonomy.cdm.model.common.OriginalSourceType;
  34 import eu.etaxonomy.cdm.model.description.Feature;
  35 import eu.etaxonomy.cdm.model.description.MeasurementUnit;
  36 import eu.etaxonomy.cdm.model.description.QuantitativeData;
  37 import eu.etaxonomy.cdm.model.description.StatisticalMeasure;
  38 import eu.etaxonomy.cdm.model.description.StatisticalMeasurementValue;
  39 import eu.etaxonomy.cdm.model.description.TaxonDescription;
  40 import eu.etaxonomy.cdm.model.name.TaxonNameBase;
  41 import eu.etaxonomy.cdm.model.reference.Reference;
  42 import eu.etaxonomy.cdm.model.reference.ReferenceFactory;
  43 import eu.etaxonomy.cdm.model.taxon.Synonym;
  44 import eu.etaxonomy.cdm.model.taxon.SynonymRelationship;
  45 import eu.etaxonomy.cdm.model.taxon.Taxon;
  46 import eu.etaxonomy.cdm.model.taxon.TaxonBase;
  47
  48 /**
  49  * @author a.mueller
  50  * @created 16.12.2010
  51  */
  52 public class CyprusAltitudeActivator {
  53         private static final Logger logger = Logger.getLogger(CyprusAltitudeActivator.class);
  54
  55         //database validation status (create, update, validate ...)
  56         static DbSchemaValidation hbm2dll = DbSchemaValidation.VALIDATE;
  57 //      static final URI source = cyprus_distribution();
  58         static final URI source = cyprus_altitude();
  59
  60
  61         static final ICdmDataSource cdmDestination = CdmDestinations.localH2();
  62 //      static final ICdmDataSource cdmDestination = CdmDestinations.cdm_test_local_mysql_test();
  63 //      static final ICdmDataSource cdmDestination = CdmDestinations.cdm_cyprus_dev();
  64 //      static final ICdmDataSource cdmDestination = CdmDestinations.cdm_cyprus_production();
  65
  66
  67         //feature tree uuid
  68         public static final UUID featureTreeUuid = UUID.fromString("14d1e912-5ec2-4d10-878b-828788b70a87");
  69
  70         //classification
  71         static final UUID classificationUuid = UUID.fromString("0c2b5d25-7b15-4401-8b51-dd4be0ee5cab");
  72
  73         private static final String sourceReferenceTitle = "Cyprus Excel Altitude Import";
  74
  75
  76         //TODO move to Feature vocabulary
  77         private static final UUID uuidAltitudeFeature = UUID.fromString("1a28ed59-e15f-4001-b5c2-ea89f0012671");
  78
  79         //check - import
  80         static final CHECK check = CHECK.IMPORT_WITHOUT_CHECK;
  81
  82         private void doImport(ICdmDataSource cdmDestination){
  83
  84
  85                 ArrayList<HashMap<String, String>> excel;
  86                 try {
  87                         excel = ExcelUtils.parseXLS(source, "coreTax");
  88                 } catch (FileNotFoundException e) {
  89                         e.printStackTrace();
  90                         return;
  91                 }
  92
  93                 CdmApplicationController app = CdmIoApplicationController.NewInstance(cdmDestination, hbm2dll);
  94
  95                 Set<TaxonBase> taxaToSave = new HashSet<TaxonBase>();
  96
  97                 TransactionStatus tx = app.startTransaction();
  98
  99                 UUID uuidMikle77 = UUID.fromString("9f5fa7ee-538b-4ae5-bd82-2a9503fea1d6");
 100                 UUID uuidMikle85 = UUID.fromString("994403c4-c400-413d-9a1a-8531a40bfd8c");
 101
 102                 Reference<?> mikle77 = app.getReferenceService().find(uuidMikle77);
 103                 Reference<?> mikle85 = app.getReferenceService().find(uuidMikle85);
 104
 105
 106                 Feature altitudeFeature = (Feature) app.getTermService().find(uuidAltitudeFeature);
 107                 if (altitudeFeature == null){
 108 //                      altitudeFeature = Feature.NewInstance("Altitude", "Altitude", "alt.");
 109 //                      altitudeFeature.setUuid(uuidAltitudeFeature);
 110 //                      featureVoc = app.getVocabularyService().find(UUID.fromString("b187d555-f06f-4d65-9e53-da7c93f8eaa8"));
 111 //                      featureVoc.addTerm(altitudeFeature);
 112                         throw new RuntimeException("Could not find altitudinal range feature");
 113                 }
 114
 115                 MeasurementUnit meter = (MeasurementUnit)app.getTermService().find(UUID.fromString("8bef5055-789c-41e5-bea2-8dc2ea8ecdf6"));
 116 //              NamedArea cyprus = (NamedArea)app.getTermService().find(UUID.fromString("da4cce9a-439b-4cc4-8073-85dc75bae169"));
 117
 118                 int count =1;
 119                 for (HashMap<String, String> row : excel){
 120                         count++;
 121                         UUID baseUuid = makeUuid(row, "uuid");
 122                         UUID acceptedUuid = makeUuid(row, "acceptedNameUuid");
 123                         UUID parentUuid = makeUuid(row, "parentUuid");
 124
 125 //                      String altitude = row.get("Altitude-kumuliert");
 126
 127                         String altitudeMin = row.get("Min");
 128                         String altitudeMax = row.get("Max");
 129                         String acceptedName = row.get("AcceptedName");
 130
 131
 132
 133                         String source = row.get("Source");
 134
 135                         if (StringUtils.isBlank(altitudeMin)){
 136                                 continue;
 137                         }
 138
 139                         boolean hasAltitude = false;
 140                         Reference<?> sourceRef = getSource(source, mikle77, mikle85);
 141                         Taxon taxon = getTaxon(app, baseUuid, acceptedUuid, parentUuid, acceptedName, count);
 142                         if (taxon != null){
 143                                 TaxonDescription desc = getDescription(taxon, sourceRef);
 144
 145                                 hasAltitude = makeAltitude(altitudeMin, altitudeMax, altitudeFeature, sourceRef, desc, meter, count);
 146 //                              hasAltitude = makeAltitudeOld(altitude, altitudeFeature, sourceRef, desc, meter, count);
 147                                 if (hasAltitude){
 148                                         if(desc.getTaxon() == null){
 149                                                 taxon.addDescription(desc);
 150                                         }
 151                                         taxaToSave.add(taxon);
 152                                 }else{
 153                                         logger.warn("HasALtitude is false in " + count);
 154                                 }
 155                         }else{
 156                                 logger.warn("Taxon not recognized in line " + count);
 157                         }
 158                 }
 159
 160                 app.getTaxonService().saveOrUpdate(taxaToSave);
 161
 162 //              tx.setRollbackOnly();
 163                 app.commitTransaction(tx);
 164         }
 165
 166
 167         private Taxon getTaxon(CdmApplicationController app, UUID baseUuid, UUID acceptedUuid, UUID parentUuid, String acceptedName, int row) {
 168                 TaxonBase<?> base = app.getTaxonService().find(baseUuid);
 169 //              TaxonBase<?> parent = app.getTaxonService().find(parentUuid);
 170
 171                 //TODO
 172                 Taxon result = null;
 173                 if (base.isInstanceOf(Taxon.class)){
 174                         Taxon t = CdmBase.deproxy(base, Taxon.class);
 175                         if (t.getTaxonNodes().size() == 1 && t.getTaxonNodes().iterator().next().getClassification().getUuid().equals(classificationUuid)){
 176                                 result = t;
 177                         }else{
 178                                 logger.warn("Base taxon (uuid) not in classification. Row: " + row +  ", Taxon: " + base.getTitleCache());
 179                         }
 180                 }
 181                 if (result == null){
 182                         TaxonBase<?> accepted = app.getTaxonService().find(acceptedUuid);
 183                         Taxon t = CdmBase.deproxy(accepted, Taxon.class);;
 184                         if (t.getTaxonNodes().size() == 1 && t.getTaxonNodes().iterator().next().getClassification().getUuid().equals(classificationUuid)){
 185                                 if (hasSynonym(t, base)){
 186                                         result = t;
 187                                 }else{
 188                                         logger.warn("Synonym relation has changed somehow. Row: " + row +  ", Taxon: " + base.getTitleCache());
 189                                 }
 190
 191                         }else{
 192                                 logger.warn("Accepted taxon not in classification. Row: " + row +  ", Taxon: " + base.getTitleCache());
 193                         }
 194                 }
 195
 196                 if (result != null){
 197                         if (! result.getName().getTitleCache().equals(acceptedName)){
 198                                 logger.warn("AcceptedName and taxon name is not equal in " + row + ".\n" +
 199                                                 " Accepted Name: " + acceptedName + ";\n" +
 200                                                 " Taxon    Name: " + result.getName().getTitleCache());
 201                         }
 202                 }
 203
 204                 return result;
 205         }
 206
 207         private boolean hasSynonym(Taxon t, TaxonBase<?> base) {
 208                 if (base.isInstanceOf(Synonym.class)){
 209                         for (SynonymRelationship rel : t.getSynonymRelations()){
 210                                 if (rel.getSynonym().equals(base)){
 211                                         return true;
 212                                 }
 213                         }
 214                 }
 215                 return false;
 216         }
 217
 218 //      private static final Pattern altitudePattern = Pattern.compile("\\d{1,4}(-\\d{1,4})?");
 219
 220
 221         private boolean makeAltitude(String altitudeMin, String altitudeMax, Feature altitudeFeature,
 222                         Reference<?> sourceRef, TaxonDescription desc, MeasurementUnit meter, int row) {
 223
 224                 QuantitativeData data = QuantitativeData.NewInstance(altitudeFeature);
 225
 226                 //Meikle
 227                 if (source != null){
 228                         TaxonNameBase<?,?> nameUsedInSource = null;  //TODO
 229                         data.addSource(OriginalSourceType.PrimaryTaxonomicSource, null, null, sourceRef, null, nameUsedInSource, null);
 230                 }
 231 //              //Excel   //excel source not wanted by Ralf
 232 //              TaxonNameBase<?,?> nameUsedInSource = null;  //TODO probably we don't want this
 233 //              data.addSource(OriginalSourceType.Import, String.valueOf(row), "row", getSourceReference(), null, nameUsedInSource, null);
 234
 235                 data.setUnit(meter);
 236
 237                 Integer min = Integer.valueOf(altitudeMin);
 238                 StatisticalMeasurementValue minValue = StatisticalMeasurementValue.NewInstance(StatisticalMeasure.MIN(), min);
 239                 data.addStatisticalValue(minValue);
 240
 241                 Integer max = Integer.valueOf(altitudeMax);
 242                 StatisticalMeasurementValue maxValue = StatisticalMeasurementValue.NewInstance(StatisticalMeasure.MAX(), max);
 243                 data.addStatisticalValue(maxValue);
 244
 245                 desc.addElement(data);
 246                 return true;
 247         }
 248
 249 //      private boolean makeAltitudeOld(String altitudeOrig, Feature feature, Reference<?> source, TaxonDescription desc, MeasurementUnit meter, int row) {
 250 //              String altitude = altitudeOrig.trim().replace(" ", "");
 251 //              Matcher matcher = altitudePattern.matcher(altitude);
 252 //
 253 //              if (matcher.matches()){
 254 //                      QuantitativeData data = QuantitativeData.NewInstance(feature);
 255 //
 256 //                      //Meikle
 257 //                      if (source != null){
 258 //                              TaxonNameBase<?,?> nameUsedInSource = null;  //TODO
 259 //                              data.addSource(OriginalSourceType.PrimaryTaxonomicSource, null, null, source, null, nameUsedInSource, null);
 260 //                      }
 261 //                      //Excel
 262 //                      TaxonNameBase<?,?> nameUsedInSource = null;  //TODO probably we don't want this
 263 //                      data.addSource(OriginalSourceType.Import, String.valueOf(row), "row", getSourceReference(), null, nameUsedInSource, null);
 264 //                      data.setUnit(meter);
 265 //
 266 //                      String[] split = altitude.split("-");
 267 //
 268 //                      Integer min = Integer.valueOf(split[0]);
 269 //                      StatisticalMeasurementValue minValue = StatisticalMeasurementValue.NewInstance(StatisticalMeasure.MIN(), min);
 270 //                      data.addStatisticalValue(minValue);
 271 //
 272 //                      if (split.length > 1){
 273 //                              Integer max = Integer.valueOf(split[1]);
 274 //                              StatisticalMeasurementValue maxValue = StatisticalMeasurementValue.NewInstance(StatisticalMeasure.MAX(), max);
 275 //                              data.addStatisticalValue(maxValue);
 276 //                      }
 277 //                      desc.addElement(data);
 278 //                      return true;
 279 //              }else{
 280 //                      logger.warn("Altitude does not match in row " + row + ": "  + altitudeOrig);
 281 //                      return false;
 282 //              }
 283 //      }
 284
 285         private TaxonDescription getDescription(Taxon taxon, Reference<?> sourceRef) {
 286                 if (taxon != null){
 287                         //TODO Mikle existiert derzeit nicht also Source
 288
 289                         TaxonDescription desc = TaxonDescription.NewInstance();
 290                         desc.setTitleCache("Import from " + getSourceReference().getTitleCache(), true);
 291                         desc.addSource(OriginalSourceType.PrimaryTaxonomicSource, null, null, sourceRef,null);
 292                         desc.addSource(OriginalSourceType.Import, null, null, getSourceReference(), null);
 293
 294                         return desc;
 295                 }
 296                 return null;
 297         }
 298
 299         private Reference<?> getSource(String source, Reference<?> m77, Reference<?> m85) {
 300                 if(StringUtils.isNotBlank(source)){
 301                         if (source.equals("Meikle 1977")){
 302                                 return m77;
 303                         }else if (source.equals("Meikle 1985")){
 304                                 return m85;
 305                         }else{
 306                                 logger.warn("Source not recognized: " + source);
 307                         }
 308                 }
 309                 return null;
 310         }
 311
 312         /**
 313          * @param row
 314          * @return
 315          */
 316         private UUID makeUuid(HashMap<String, String> row, String colName) {
 317                 if (StringUtils.isBlank(row.get(colName))){
 318                         return null;
 319                 }else{
 320                         return UUID.fromString(row.get(colName));
 321                 }
 322         }
 323
 324 //      private void getRowValues(HashMap<String, String> row) {
 325 //              // TODO Auto-generated method stub
 326 //              HashMap<String, Object> = new HashM
 327 //              row
 328 //
 329 //
 330 //      }
 331
 332
 333         Reference<?> sourceReference;
 334         private Reference<?> getSourceReference() {
 335                 if (sourceReference == null){
 336                         sourceReference = ReferenceFactory.newGeneric();
 337                         sourceReference.setTitleCache(sourceReferenceTitle, true);
 338
 339                 }
 340                 return sourceReference;
 341
 342         }
 343
 344
 345         //Cyprus
 346         public static URI cyprus_altitude() {
 347                 URI sourceUrl;
 348                 try {
 349                         sourceUrl = new URI("file:/F:/data/cyprus/Cyprus-altitude-import-neu.xls");
 350 //                      sourceUrl = new URI("file:/F:/data/cyprus/Zypern-Altitude.xls");
 351                         return sourceUrl;
 352                 } catch (URISyntaxException e) {
 353                         e.printStackTrace();
 354                         return null;
 355                 }
 356         }
 357
 358
 359         /**
 360          * @param args
 361          */
 362         public static void main(String[] args) {
 363                 CyprusAltitudeActivator me = new CyprusAltitudeActivator();
 364                 me.doImport(cdmDestination);
 365                 me.testMatcher();
 366         }
 367
 368         private void testMatcher() {
 369 //              makeAltitude("0-4400", null, null);
 370
 371         }
 372
 373 }