app-import/src/main/java/eu/etaxonomy/cdm/app/synthesysImport/SynthesysCacheActivator.java

   1 /**
   2  * Copyright (C) 2007 EDIT
   3  * European Distributed Institute of Taxonomy
   4  * http://www.e-taxonomy.eu
   5  *
   6  * The contents of this file are subject to the Mozilla Public License Version 1.1
   7  * See LICENSE.TXT at the top of this package for the full license terms.
   8  */
   9
  10 package eu.etaxonomy.cdm.app.synthesysImport;
  11
  12 import java.io.FileNotFoundException;
  13 import java.net.URI;
  14 import java.net.URISyntaxException;
  15 import java.util.ArrayList;
  16 import java.util.HashMap;
  17 import java.util.List;
  18 import java.util.ListIterator;
  19
  20 import org.apache.log4j.Logger;
  21 import org.springframework.transaction.TransactionStatus;
  22
  23 import eu.etaxonomy.cdm.api.application.CdmApplicationController;
  24 import eu.etaxonomy.cdm.api.service.pager.Pager;
  25 import eu.etaxonomy.cdm.app.common.CdmDestinations;
  26 import eu.etaxonomy.cdm.app.common.CdmImportSources;
  27 import eu.etaxonomy.cdm.common.ExcelUtils;
  28 import eu.etaxonomy.cdm.database.DbSchemaValidation;
  29 import eu.etaxonomy.cdm.database.ICdmDataSource;
  30 import eu.etaxonomy.cdm.model.agent.AgentBase;
  31 import eu.etaxonomy.cdm.model.agent.Institution;
  32 import eu.etaxonomy.cdm.model.agent.Person;
  33 import eu.etaxonomy.cdm.model.common.Language;
  34 import eu.etaxonomy.cdm.model.common.LanguageString;
  35 import eu.etaxonomy.cdm.model.location.Country;
  36 import eu.etaxonomy.cdm.model.location.NamedArea;
  37 import eu.etaxonomy.cdm.model.location.Point;
  38 import eu.etaxonomy.cdm.model.name.TaxonNameBase;
  39 import eu.etaxonomy.cdm.model.occurrence.Collection;
  40 import eu.etaxonomy.cdm.model.occurrence.DerivationEvent;
  41 import eu.etaxonomy.cdm.model.occurrence.DerivedUnit;
  42 import eu.etaxonomy.cdm.model.occurrence.DeterminationEvent;
  43 import eu.etaxonomy.cdm.model.occurrence.FieldUnit;
  44 import eu.etaxonomy.cdm.model.occurrence.GatheringEvent;
  45 import eu.etaxonomy.cdm.model.occurrence.SpecimenOrObservationType;
  46 import eu.etaxonomy.cdm.model.reference.Reference;
  47 import eu.etaxonomy.cdm.model.reference.ReferenceFactory;
  48 import eu.etaxonomy.cdm.model.taxon.Taxon;
  49 import eu.etaxonomy.cdm.strategy.parser.NonViralNameParserImpl;
  50
  51 /**
  52  * @author PK
  53  * @created 19.09.2008
  54  * @version 1.0
  55  */
  56 public class SynthesysCacheActivator {
  57         private static final Logger logger = Logger.getLogger(SynthesysCacheActivator.class);
  58
  59         protected String fullScientificNameString = null;
  60         protected String institutionCode = null;
  61         protected String collectionCode = null;
  62         protected String unitID = null;
  63         protected String recordBasis = null;
  64         protected String accessionNumber = null;
  65         protected String collectorsNumber = null;
  66         protected String fieldNumber = null;
  67         protected Double longitude = null;
  68         protected Double latitude = null;
  69         protected String locality = null;
  70         protected String country = null;
  71         protected String isocountry = null;
  72         protected ArrayList<String> gatheringAgentList = new ArrayList<String>();
  73         protected ArrayList<String> identificationList = new ArrayList<String>();
  74
  75         static DbSchemaValidation hbm2dll = DbSchemaValidation.CREATE;
  76
  77         static ICdmDataSource desitination = CdmDestinations.localH2();
  78
  79
  80         public void saveUnit(HashMap<String,String> unit){
  81                 String author = unit.get("author");
  82                 author=author.replaceAll("None","");
  83                 String taxonName = unit.get("taxonName");
  84                 taxonName = taxonName.replaceAll("None", "");
  85
  86                 try {
  87                         this.institutionCode = unit.get("institution").replaceAll("None", null);
  88                 } catch (Exception e) {
  89                 }
  90
  91                 try {this.collectionCode = unit.get("collection").replaceAll("None", null);
  92                 } catch (Exception e) {
  93                 }
  94                 try {this.unitID = unit.get("unitID").replaceAll("None", null);
  95                 } catch (Exception e) {
  96                 }
  97                 try {this.recordBasis = unit.get("recordBasis").replaceAll("None", null);
  98                 } catch (Exception e) {
  99                 }
 100                 try {this.accessionNumber = null;
 101                 } catch (Exception e) {
 102                 }
 103                 try {this.locality = unit.get("locality").replaceAll("None", null);
 104                 } catch (Exception e) {
 105                 }
 106                 try {this.longitude = Double.valueOf(unit.get("longitude"));
 107                 } catch (Exception e) {
 108                 }
 109                 try {this.latitude = Double.valueOf(unit.get("latitude"));
 110                 } catch (Exception e) {
 111                 }
 112                 try {this.country = unit.get("country").replaceAll("None", null);
 113                 } catch (Exception e) {
 114                 }
 115                 try {this.isocountry = unit.get("isoCountry").replaceAll("None", null);
 116                 } catch (Exception e) {
 117                 }
 118                 try {this.fieldNumber = unit.get("field number").replaceAll("None", null);
 119                 } catch (Exception e) {
 120                 }
 121                 try {this.collectorsNumber = unit.get("collector number").replaceAll("None", null);
 122                 } catch (Exception e) {
 123                 }
 124                 try {String coll =unit.get("collector");
 125                         coll=coll.replaceAll("None", null);
 126                         this.gatheringAgentList.add(coll);
 127                 } catch (Exception e) {
 128                 }
 129                 try {this.identificationList.add(taxonName+" "+author);
 130                 } catch (Exception e) {System.out.println(e);
 131                 }
 132         }
 133
 134         public boolean invoke(){
 135                 boolean result = true;
 136                 boolean withCdm = true;
 137                 CdmApplicationController app = null;
 138                 TransactionStatus tx = null;
 139
 140                 app = CdmApplicationController.NewInstance(desitination, hbm2dll);
 141
 142                 tx = app.startTransaction();
 143                 try {
 144                         Reference sec = ReferenceFactory.newDatabase();
 145                         sec.setTitleCache("SYNTHESYS CACHE DATA", true);
 146
 147                         /**
 148                          * SPECIMEN OR OBSERVATION OR LIVING
 149                          */
 150                         DerivedUnit derivedThing = null;
 151                         //create specimen
 152                         if (this.recordBasis != null){
 153                                 if (this.recordBasis.toLowerCase().startsWith("s")) {//specimen
 154                                         derivedThing = DerivedUnit.NewPreservedSpecimenInstance();
 155                                 }
 156                                 else if (this.recordBasis.toLowerCase().startsWith("o")) {//observation
 157                                         derivedThing = DerivedUnit.NewInstance(SpecimenOrObservationType.Observation);
 158                                 }
 159                                 else if (this.recordBasis.toLowerCase().startsWith("l")) {//living -> fossil, herbarium sheet....???
 160                                         derivedThing = DerivedUnit.NewInstance(SpecimenOrObservationType.LivingSpecimen);
 161                                 }
 162                         }
 163                         if (derivedThing == null){
 164                                 derivedThing =  DerivedUnit.NewInstance(SpecimenOrObservationType.Observation);
 165                         }
 166
 167                         TaxonNameBase<?,?> taxonName = null;
 168                         Taxon taxon = null;
 169                         DeterminationEvent determinationEvent = null;
 170                         List<TaxonNameBase> names = null;
 171                         NonViralNameParserImpl nvnpi = NonViralNameParserImpl.NewInstance();
 172                         String scientificName="";
 173                         boolean preferredFlag=false;
 174                         System.out.println(this.identificationList);
 175                         for (int i = 0; i < this.identificationList.size(); i++) {
 176                                 this.fullScientificNameString = this.identificationList.get(i);
 177                                 this.fullScientificNameString = this.fullScientificNameString.replaceAll(" et ", " & ");
 178                                 if (this.fullScientificNameString.indexOf("_preferred_") != -1){
 179                                         scientificName = this.fullScientificNameString.split("_preferred_")[0];
 180                                         String pTmp = this.fullScientificNameString.split("_preferred_")[1];
 181                                         if (pTmp == "1" || pTmp.toLowerCase().indexOf("true") != -1) {
 182                         preferredFlag=true;
 183                     } else {
 184                         preferredFlag=false;
 185                     }
 186                                 } else {
 187                     scientificName = this.fullScientificNameString;
 188                 }
 189
 190 //                              taxonName = nvnpi.parseFullName(this.fullScientificNameString,NomenclaturalCode.ICZN(),null);
 191 //                              if (taxonName.hasProblem()){
 192 //                              System.out.println("pb ICZN");
 193 //                              taxonName  = nvnpi.parseFullName(this.fullScientificNameString,NomenclaturalCode.ICBN(),null);
 194 //                              if (taxonName.hasProblem()){
 195 //                              System.out.println("pb ICBN");
 196 //                              taxonName = nvnpi.parseFullName(this.fullScientificNameString,NomenclaturalCode.ICNB(), null);
 197 //                              if (taxonName.hasProblem()){
 198 //                              System.out.println("pb ICNB");
 199 //                              taxonName = nvnpi.parseFullName(this.fullScientificNameString,NomenclaturalCode.ICNCP(), null);
 200 //                              if (taxonName.hasProblem()){
 201 //                              System.out.println("pb ICNCP");
 202 //                              }
 203 //                              }
 204 //                              }
 205 //                              }
 206                                 taxonName = nvnpi.parseFullName(scientificName);
 207                                 if (withCdm){
 208                                         names = app.getNameService().getNamesByName(scientificName);
 209                                         if (names.size() == 0){
 210                                                 System.out.println("Name not found: " + scientificName);
 211                                         }else{
 212                                                 if (names.size() > 1){
 213                                                         System.out.println("More then 1 name found: " + scientificName);
 214                                                 }
 215                                                 System.out.println("Name found");
 216                                                 taxonName = names.get(0);
 217                                         }
 218                                 }
 219
 220
 221 //                              tx = app.startTransaction();
 222                                 app.getNameService().saveOrUpdate(taxonName);
 223                                 taxon = Taxon.NewInstance(taxonName, sec); //TODO use real reference for sec
 224 //                              app.commitTransaction(tx);
 225
 226
 227                                 determinationEvent = DeterminationEvent.NewInstance();
 228                                 determinationEvent.setTaxon(taxon);
 229                                 determinationEvent.setPreferredFlag(preferredFlag);
 230                                 derivedThing.addDetermination(determinationEvent);
 231                         }
 232
 233
 234                         //set catalogue number (unitID)
 235                         derivedThing.setCatalogNumber(this.unitID);
 236                         derivedThing.setAccessionNumber(this.accessionNumber);
 237                         derivedThing.setCollectorsNumber(this.collectorsNumber);
 238
 239
 240                         /**
 241                          * INSTITUTION & COLLECTION
 242                          */
 243                         //manage institution
 244                         Institution institution;
 245                         List<Institution> institutions;
 246                         try{
 247                                 System.out.println(this.institutionCode);
 248                                 institutions= app.getAgentService().searchInstitutionByCode(this.institutionCode);
 249                         }catch(Exception e){
 250                                 System.out.println("BLI "+e);
 251                                 institutions=new ArrayList<Institution>();
 252                         }
 253                         if (institutions.size() ==0){
 254                                 System.out.println("Institution (agent) unknown");
 255                                 //create institution
 256                                 institution = Institution.NewInstance();
 257                                 institution.setCode(this.institutionCode);
 258                         }
 259                         else{
 260                                 System.out.println("Institution (agent) already in the db");
 261                                 institution = institutions.get(0);
 262                         }
 263
 264                         //manage collection
 265                         Collection collection = Collection.NewInstance();
 266                         List<Collection> collections;
 267                         try{
 268                                 collections = app.getCollectionService().searchByCode(this.collectionCode);
 269                                 //.searchCollectionByCode(this.collectionCode);
 270                         }catch(Exception e){
 271                                 System.out.println("BLA"+e);
 272                                 collections=new ArrayList<Collection>();
 273                         }
 274                         if (collections.size() ==0){
 275                                 System.out.println("Collection not found "+this.collectionCode);
 276                                 //create new collection
 277                                 collection.setCode(this.collectionCode);
 278                                 collection.setCodeStandard("GBIF");
 279                                 collection.setInstitute(institution);
 280                         }
 281                         else{
 282                                 boolean collectionFound=false;
 283                                 for (int i=0; i<collections.size(); i++){
 284                                         collection = collections.get(i);
 285                                         try {
 286                                                 if (collection.getInstitute().getCode().equalsIgnoreCase(institution.getCode())){
 287                                                         //found a collection with the same code and the same institution
 288                                                         collectionFound=true;
 289                                                 }
 290                                         } catch (NullPointerException e) {}
 291                                 }
 292                                 System.out.println("a trouvé la collection avec la meme institution? "+collectionFound);
 293                                 if (!collectionFound){ //need to add a new collection with the pre-configured institution
 294                                         collection.setCode(this.collectionCode);
 295                                         collection.setCodeStandard("GBIF");
 296                                         collection.setInstitute(institution);
 297                                 }
 298
 299                         }
 300                         System.out.println("collection inserted");
 301                         //link specimen & collection
 302                         derivedThing.setCollection(collection);
 303
 304                         /**
 305                          * GATHERING EVENT
 306                          */
 307                         //create gathering event
 308                         GatheringEvent gatheringEvent = GatheringEvent.NewInstance();
 309                         //add locality
 310                         Language language = Language.DEFAULT();
 311                         LanguageString loc = LanguageString.NewInstance(this.locality,language);
 312                         gatheringEvent.setLocality(loc);
 313
 314                         //create coordinates point
 315                         Point coordinates = Point.NewInstance();
 316                         //add coordinates
 317                         coordinates.setLatitude(this.latitude);
 318                         coordinates.setLongitude(this.longitude);
 319                         gatheringEvent.setExactLocation(coordinates);
 320
 321                         NamedArea area = NamedArea.NewInstance();
 322
 323
 324                         Country country = null;
 325 //                      System.out.println("isocountry "+this.isocountry);
 326                         if (this.isocountry != null) {
 327                 country = app.getOccurrenceService().getCountryByIso(this.isocountry);
 328             }
 329
 330 //                      System.out.println(country.getLabel());
 331 //                      Set<Continent> cont = country.getContinents();
 332 //
 333 //                      System.out.println(cont.size());
 334 //                      Iterator<Continent> iter = cont.iterator();
 335 //                      while (iter.hasNext())
 336 //                              System.out.println(iter.next().toString());
 337
 338                         if (country != null){
 339                                 area.addCountry(country);
 340                                 System.out.println("country not null!");
 341                         }
 342 //                      else{
 343 //                              if (this.country != null){
 344 //                                      List<Country>countries = app.getOccurrenceService().getCountryByName(this.country);
 345 //                                      if (countries.size() >0)
 346 //                                              area.addCountry(countries.get(0));
 347 //                                      else
 348 //                                              System.out.println("NO COUNTRY");//TODO need to add a new country!
 349 //                              }
 350 //                      }
 351 //                      app.getTermService().saveTerm(area);
 352                         gatheringEvent.addCollectingArea(area);
 353
 354                         //create collector
 355                         AgentBase collector;
 356                         ListIterator<String> collectors = this.gatheringAgentList.listIterator();
 357                         //add the collectors
 358                         String collName;
 359                         while (collectors.hasNext()){
 360                                 collName = collectors.next();
 361                                 /*check if the collector does already exist*/
 362                                 try{
 363                                         Pager<AgentBase> col = app.getAgentService().findByTitle(null, collName, null, null, null, null, null, null);
 364                                         collector=col.getRecords().get(0);
 365                                         System.out.println("a trouve l'agent");
 366                                 }catch (Exception e) {
 367                                         collector = Person.NewInstance();
 368                                         collector.setTitleCache(collName, true);
 369                                 }
 370                                 gatheringEvent.setCollector(collector);
 371                         }
 372
 373                         //create field/observation
 374                         FieldUnit fieldUnit = FieldUnit.NewInstance();
 375                         //add fieldNumber
 376                         fieldUnit.setFieldNumber(this.fieldNumber);
 377
 378                         //join gatheringEvent to fieldObservation
 379                         fieldUnit.setGatheringEvent(gatheringEvent);
 380
 381
 382 //                      //link fieldObservation and specimen
 383                         DerivationEvent derivationEvent = DerivationEvent.NewInstance();
 384                         derivationEvent.addOriginal(fieldUnit);
 385                         derivedThing.addDerivationEvent(derivationEvent);
 386 //                      derivationEvent.addDerivative(derivedThing);
 387
 388                         /**
 389                          * SAVE AND STORE DATA
 390                          */
 391                         //save the specimen data
 392                         //      app.getOccurrenceService().saveSpecimenOrObservationBase(fieldObservation);
 393                         try {
 394 //                              tx = app.startTransaction();
 395                                 app.getTermService().saveOrUpdate(area);//save it sooner
 396                                 app.getOccurrenceService().saveOrUpdate(derivedThing);
 397 //                              app.commitTransaction(tx);
 398 //                              app.close();
 399                         } catch (Exception e) {
 400                                 // TODO Auto-generated catch block
 401                                 System.out.println("PATATE "+e);
 402                         }
 403
 404
 405                         logger.info("saved new specimen ...");
 406
 407
 408
 409                 } catch (Exception e) {
 410                         logger.warn("Error when reading record!!");
 411                         e.printStackTrace();
 412                         result = false;
 413                 }
 414 //
 415                 app.commitTransaction(tx);
 416                 System.out.println("commit done");
 417                 app.close();
 418
 419                 return result;
 420         }
 421
 422
 423
 424         private DeterminationEvent getDetermination(Taxon taxon, String actor){
 425                 logger.info("Create determination event");
 426                 DeterminationEvent determinationEvent = DeterminationEvent.NewInstance();
 427                 determinationEvent.setTaxon(taxon);
 428                 Person person = Person.NewTitledInstance(actor);
 429                 determinationEvent.setActor(person);
 430                 return determinationEvent;
 431         }
 432
 433
 434
 435         /**
 436          * @param args
 437          * @throws URISyntaxException
 438          */
 439         public static void main(String[] args) throws URISyntaxException {
 440                 URI uri = CdmImportSources.SYNTHESYS_SPECIMEN();
 441                 logger.info("main method");
 442                 SynthesysCacheActivator abcdAct = new SynthesysCacheActivator();
 443
 444                 ArrayList<HashMap<String, String>> units;
 445                 try {
 446                         units = ExcelUtils.parseXLS(uri);
 447                 } catch (FileNotFoundException e) {
 448                         logger.error("FileNotFound: " + uri);
 449                         return;
 450                 }
 451                 HashMap<String,String> unit=null;
 452                 for (int i=0; i<units.size();i++){
 453                         unit = units.get(i);
 454                         logger.info(unit);
 455                         abcdAct.saveUnit(unit);//and then invoke
 456                         abcdAct.invoke();
 457
 458                 }
 459         }
 460
 461
 462
 463 }