cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/berlinModel/in/BerlinModelCommonNamesImport.java

   1 /**
   2 * Copyright (C) 2007 EDIT
   3 * European Distributed Institute of Taxonomy
   4 * http://www.e-taxonomy.eu
   5 *
   6 * The contents of this file are subject to the Mozilla Public License Version 1.1
   7 * See LICENSE.TXT at the top of this package for the full license terms.
   8 */
   9
  10 package eu.etaxonomy.cdm.io.berlinModel.in;
  11
  12 import java.sql.ResultSet;
  13 import java.sql.SQLException;
  14 import java.util.ArrayList;
  15 import java.util.HashMap;
  16 import java.util.HashSet;
  17 import java.util.List;
  18 import java.util.Map;
  19 import java.util.Set;
  20 import java.util.SortedSet;
  21 import java.util.TreeSet;
  22 import java.util.UUID;
  23
  24 import org.apache.commons.lang.StringUtils;
  25 import org.apache.log4j.Logger;
  26 import org.springframework.stereotype.Component;
  27
  28 import eu.etaxonomy.cdm.common.CdmUtils;
  29 import eu.etaxonomy.cdm.io.berlinModel.BerlinModelTransformer;
  30 import eu.etaxonomy.cdm.io.berlinModel.in.validation.BerlinModelCommonNamesImportValidator;
  31 import eu.etaxonomy.cdm.io.common.IOValidator;
  32 import eu.etaxonomy.cdm.io.common.ResultSetPartitioner;
  33 import eu.etaxonomy.cdm.io.common.Source;
  34 import eu.etaxonomy.cdm.model.common.Annotation;
  35 import eu.etaxonomy.cdm.model.common.AnnotationType;
  36 import eu.etaxonomy.cdm.model.common.CdmBase;
  37 import eu.etaxonomy.cdm.model.common.DescriptionElementSource;
  38 import eu.etaxonomy.cdm.model.common.Extension;
  39 import eu.etaxonomy.cdm.model.common.ExtensionType;
  40 import eu.etaxonomy.cdm.model.common.Language;
  41 import eu.etaxonomy.cdm.model.common.Marker;
  42 import eu.etaxonomy.cdm.model.common.MarkerType;
  43 import eu.etaxonomy.cdm.model.common.Representation;
  44 import eu.etaxonomy.cdm.model.description.CommonTaxonName;
  45 import eu.etaxonomy.cdm.model.description.TaxonDescription;
  46 import eu.etaxonomy.cdm.model.location.NamedArea;
  47 import eu.etaxonomy.cdm.model.location.TdwgArea;
  48 import eu.etaxonomy.cdm.model.location.WaterbodyOrCountry;
  49 import eu.etaxonomy.cdm.model.name.TaxonNameBase;
  50 import eu.etaxonomy.cdm.model.reference.Reference;
  51 import eu.etaxonomy.cdm.model.taxon.Taxon;
  52 import eu.etaxonomy.cdm.model.taxon.TaxonBase;
  53
  54 /**
  55  *
  56  * @author a.mueller
  57  * @created 20.03.2008
  58  * @version 1.0
  59  */
  60 @Component
  61 public class BerlinModelCommonNamesImport  extends BerlinModelImportBase {
  62         private static final Logger logger = Logger.getLogger(BerlinModelCommonNamesImport.class);
  63
  64         public static final UUID REFERENCE_LANGUAGE_ISO639_2_UUID = UUID.fromString("40c4f8dd-3d9c-44a4-b77a-76e137a89a5f");
  65         public static final UUID REFERENCE_LANGUAGE_STRING_UUID = UUID.fromString("2a1b678f-c27d-48c1-b43e-98fd0d426305");
  66         public static final UUID STATUS_ANNOTATION_UUID = UUID.fromString("e3f7b80a-1286-458d-812c-5e818f731968");
  67
  68         public static final String NAMESPACE = "common name";
  69
  70
  71         private static final String pluralString = "common names";
  72         private static final String dbTableName = "emCommonName";
  73
  74
  75         //map that stores the regions (named areas) and makes them accessible via the regionFk
  76         private Map<String, NamedArea> regionMap = new HashMap<String, NamedArea>();
  77
  78
  79
  80         public BerlinModelCommonNamesImport(){
  81                 super();
  82         }
  83
  84
  85
  86         @Override
  87         protected String getIdQuery(BerlinModelImportState state) {
  88                 String result = " SELECT CommonNameId FROM emCommonName WHERE (1=1) ";
  89                 if (StringUtils.isNotBlank(state.getConfig().getCommonNameFilter())){
  90                         result += " AND " + state.getConfig().getCommonNameFilter();
  91                 }
  92
  93                 return result;
  94         }
  95
  96
  97
  98         /* (non-Javadoc)
  99          * @see eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelImportBase#getRecordQuery(eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelImportConfigurator)
 100          */
 101         @Override
 102         protected String getRecordQuery(BerlinModelImportConfigurator config) {
 103                 String recordQuery = "";
 104                 recordQuery =
 105                         " SELECT emCommonName.CommonNameId, emCommonName.CommonName, PTaxon.RIdentifier AS taxonId, emCommonName.PTNameFk, emCommonName.RefFk AS refId, emCommonName.Status, " +
 106                                 " emCommonName.RegionFks, emCommonName.MisNameRefFk, emCommonName.NameInSourceFk , emCommonName.Created_When, emCommonName.Updated_When, emCommonName.Created_Who, emCommonName.Updated_Who, emCommonName.Note as Notes," +
 107                         " regionLanguage.Language AS regionLanguage, languageCommonName.Language, languageCommonName.LanguageOriginal, languageCommonName.ISO639_1, languageCommonName.ISO639_2, " +
 108                         " emLanguageRegion.Region, emLanguageReference.RefFk as languageRefRefFk, emLanguageReference.ReferenceShort, " +
 109                         " emLanguageReference.ReferenceLong, emLanguageReference.LanguageFk, languageReferenceLanguage.Language AS refLanguage, " +
 110                         " languageReferenceLanguage.ISO639_2 AS refLanguageIso639_2, regionLanguage.ISO639_2 AS regionLanguageIso, " +
 111                         " misappliedTaxon.RIdentifier AS misappliedTaxonId " +
 112                 " FROM emLanguage as regionLanguage RIGHT OUTER JOIN " +
 113                         " emLanguageRegion ON regionLanguage.LanguageId = emLanguageRegion.LanguageFk RIGHT OUTER JOIN " +
 114                         " emLanguage AS languageReferenceLanguage RIGHT OUTER JOIN " +
 115                         " emLanguageReference ON languageReferenceLanguage.LanguageId = emLanguageReference.LanguageFk RIGHT OUTER JOIN " +
 116                         " emCommonName INNER JOIN " +
 117                         " PTaxon ON emCommonName.PTNameFk = PTaxon.PTNameFk AND emCommonName.PTRefFk = PTaxon.PTRefFk ON " +
 118                         " emLanguageReference.ReferenceId = emCommonName.LanguageRefFk LEFT OUTER JOIN " +
 119                         " emLanguage AS languageCommonName ON emCommonName.LanguageFk = languageCommonName.LanguageId ON " +
 120                         " emLanguageRegion.RegionId = emCommonName.RegionFks LEFT OUTER JOIN " +
 121                         " PTaxon as misappliedTaxon ON emCommonName.NameInSourceFk = misappliedTaxon.PTNameFk AND emCommonName.MisNameRefFk = misappliedTaxon.PTRefFk " +
 122                         " WHERE emCommonName.CommonNameId IN (" + ID_LIST_TOKEN + ")";
 123                 return recordQuery;
 124         }
 125
 126
 127
 128         @Override
 129         protected void doInvoke(BerlinModelImportState state) {
 130                 try {
 131                         makeRegions(state);
 132                 } catch (Exception e) {
 133                         logger.error("Error when creating common name regions:" + e.getMessage());
 134                         e.printStackTrace();
 135                         state.setUnsuccessfull();
 136                 }
 137                 super.doInvoke(state);
 138                 return;
 139         }
 140
 141         /**
 142          * @param state
 143          *
 144          */
 145         private void makeRegions(BerlinModelImportState state) {
 146                 try {
 147                         SortedSet<Integer> regionFks = new TreeSet<Integer>();
 148                         Source source = state.getConfig().getSource();
 149
 150                         getRegionFks(state, regionFks, source);
 151                         //concat filter string
 152                         String sqlWhere = getSqlWhere(regionFks);
 153
 154                         //get E+M - TDWG Mapping
 155                         Map<String, String> emTdwgMap = getEmTdwgMap(source);
 156                         //fill regionMap
 157                         fillRegionMap(state, sqlWhere, emTdwgMap);
 158
 159                         return;
 160                 } catch (NumberFormatException e) {
 161                         e.printStackTrace();
 162                         state.setUnsuccessfull();
 163                         return;
 164                 } catch (SQLException e) {
 165                         e.printStackTrace();
 166                         state.setUnsuccessfull();
 167                         return;
 168                 }
 169         }
 170
 171
 172         /* (non-Javadoc)
 173          * @see eu.etaxonomy.cdm.io.berlinModel.in.IPartitionedIO#doPartition(eu.etaxonomy.cdm.io.berlinModel.in.ResultSetPartitioner, eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelImportState)
 174          */
 175         public boolean doPartition(ResultSetPartitioner partitioner, BerlinModelImportState state)  {
 176                 boolean success = true ;
 177                 BerlinModelImportConfigurator config = state.getConfig();
 178                 Set<TaxonBase> taxaToSave = new HashSet<TaxonBase>();
 179                 Map<String, Taxon> taxonMap = (Map<String, Taxon>) partitioner.getObjectMap(BerlinModelTaxonImport.NAMESPACE);
 180                 Map<String, TaxonNameBase> taxonNameMap = (Map<String, TaxonNameBase>) partitioner.getObjectMap(BerlinModelTaxonNameImport.NAMESPACE);
 181
 182                 Map<String, Reference> biblioRefMap = (Map<String, Reference>) partitioner.getObjectMap(BerlinModelReferenceImport.BIBLIO_REFERENCE_NAMESPACE);
 183                 Map<String, Reference> nomRefMap = (Map<String, Reference>) partitioner.getObjectMap(BerlinModelReferenceImport.NOM_REFERENCE_NAMESPACE);
 184
 185                 Map<String, Language> iso6392Map = new HashMap<String, Language>();
 186
 187         //      logger.warn("MisappliedNameRefFk  not yet implemented for Common Names");
 188
 189                 ResultSet rs = partitioner.getResultSet();
 190                 try{
 191                         while (rs.next()){
 192
 193                                 //create TaxonName element
 194                                 Object commonNameId = rs.getObject("CommonNameId");
 195                                 int taxonId = rs.getInt("taxonId");
 196                                 Object refId = rs.getObject("refId");
 197                                 Object ptNameFk = rs.getObject("PTNameFk");
 198                                 String commonNameString = rs.getString("CommonName");
 199                                 String iso639_2 = rs.getString("ISO639_2");
 200                                 String iso639_1 = rs.getString("ISO639_1");
 201                                 String languageString = rs.getString("Language");
 202                                 String originalLanguageString = rs.getString("LanguageOriginal");
 203                                 Object misNameRefFk = rs.getObject("MisNameRefFk");
 204                                 Object languageRefRefFk = rs.getObject("languageRefRefFk");
 205                                 String refLanguage = rs.getString("refLanguage");
 206                                 String refLanguageIso639_2 = rs.getString("refLanguageIso639_2");
 207                                 String status = rs.getString("Status");
 208                                 Object nameInSourceFk = rs.getObject("NameInSourceFk");
 209                                 Object misappliedTaxonId = rs.getObject("misappliedTaxonId");
 210
 211                                 //regions
 212                                 String region = rs.getString("Region");
 213                                 String regionFks  = rs.getString("RegionFks");
 214                                 String[] regionFkSplit = regionFks.split(",");
 215
 216                                 //commonNameString
 217                                 if (CdmUtils.isBlank(commonNameString)){
 218                                         String message = "CommonName is empty or null. Do not import record for taxon " + taxonId;
 219                                         logger.warn(message);
 220                                         continue;
 221                                 }
 222
 223                                 //taxon
 224                                 Taxon taxon = null;
 225                                 TaxonBase taxonBase  = taxonMap.get(String.valueOf(taxonId));
 226                                 if (taxonBase == null){
 227                                         logger.warn("Taxon (" + taxonId + ") could not be found. Common name " + commonNameString + "(" + commonNameId + ") not imported");
 228                                         continue;
 229                                 }else if (! taxonBase.isInstanceOf(Taxon.class)){
 230                                         logger.warn("taxon (" + taxonId + ") is not accepted. Can't import common name " +  commonNameId);
 231                                         continue;
 232                                 }else{
 233                                         taxon = CdmBase.deproxy(taxonBase, Taxon.class);
 234                                 }
 235
 236                                 //Language
 237                                 Language language = getAndHandleLanguage(iso6392Map, iso639_2, iso639_1, languageString, originalLanguageString, state);
 238
 239                                 //CommonTaxonName
 240                                 List<CommonTaxonName> commonTaxonNames = new ArrayList<CommonTaxonName>();
 241                                 for (String regionFk : regionFkSplit){ //
 242                                         CommonTaxonName commonTaxonName;
 243                                         if (commonTaxonNames.size() == 0){
 244                                                 commonTaxonName = CommonTaxonName.NewInstance(commonNameString, language);
 245                                         }else{
 246                                                 commonTaxonName = (CommonTaxonName)commonTaxonNames.get(0).clone();
 247                                         }
 248                                         commonTaxonNames.add(commonTaxonName);
 249                                         regionFk = regionFk.trim();
 250                                         NamedArea area = regionMap.get(regionFk);
 251                                         if (area == null){
 252                                                 if (regionFkSplit.length > 1 && StringUtils.isNotBlank(regionFk)){
 253                                                         logger.warn("Area for " + regionFk + " not defined.");
 254                                                 }else{
 255                                                         //no region is defined
 256                                                 }
 257                                         }else{
 258                                                 commonTaxonName.setArea(area);
 259                                                 TaxonDescription description = getDescription(taxon);
 260                                                 description.addElement(commonTaxonName);
 261                                         }
 262                                 }
 263
 264                                 //Reference/Source
 265                                 String strRefId = String.valueOf(refId);
 266                                 String languageRefFk = String.valueOf(languageRefRefFk);
 267                                 if (! CdmUtils.nullSafeEqual(strRefId, languageRefFk)){
 268                                         //use strRefId if languageRefFk is null
 269                                         if (languageRefRefFk == null){
 270                                                 languageRefFk = strRefId;
 271                                         }else{
 272                                                 logger.warn("CommonName.RefFk (" + CdmUtils.Nz(strRefId) + ") and LanguageReference.RefFk " + CdmUtils.Nz(languageRefFk) + " are not equal. I will import only languageReference.RefFk");
 273                                         }
 274                                 }
 275
 276                                 Reference<?> reference = getReferenceOnlyFromMaps(biblioRefMap, nomRefMap, String.valueOf(languageRefRefFk));
 277                                 String microCitation = null;
 278                                 String originalNameString = null;
 279
 280                                 TaxonNameBase<?,?> nameUsedInSource = taxonNameMap.get(String.valueOf(nameInSourceFk));
 281                                 if (nameInSourceFk != null && nameUsedInSource == null){
 282                                         logger.warn("Name used in source (" + nameInSourceFk + ") was not found for common name " + commonNameId);
 283                                 }
 284                                 DescriptionElementSource source = DescriptionElementSource.NewInstance(reference, microCitation, nameUsedInSource, originalNameString);
 285                                 for (CommonTaxonName commonTaxonName : commonTaxonNames){
 286                                         commonTaxonName.addSource(source);
 287                                 }
 288
 289
 290                                 //MisNameRef
 291                                 if (misNameRefFk != null){
 292                                         //Taxon misappliedName = getMisappliedName(biblioRefMap, nomRefMap, misNameRefFk, taxon);
 293                                         Taxon misappliedNameTaxon = null;
 294                                         if (misappliedTaxonId != null){
 295                                                 TaxonBase misTaxonBase =  taxonMap.get(String.valueOf(misappliedTaxonId));
 296                                                 if (misTaxonBase.isInstanceOf(Taxon.class)){
 297                                                         misappliedNameTaxon = CdmBase.deproxy(misTaxonBase, Taxon.class);
 298                                                 }else{
 299                                                         logger.warn("Misapplied name taxon is not of type Taxon but " + misTaxonBase.getClass().getSimpleName());
 300                                                 }
 301                                         }else{
 302
 303                                                 Reference<?> sec = getReferenceOnlyFromMaps(biblioRefMap, nomRefMap, String.valueOf(misNameRefFk));
 304                                                 if (nameUsedInSource == null || sec == null){
 305                                                         logger.warn("Taxon name or misapplied name reference is null for common name " + commonNameId);
 306                                                 }else{
 307                                                         misappliedNameTaxon = Taxon.NewInstance(nameUsedInSource, sec);
 308                                                         MarkerType misCommonNameMarker = getMarkerType(state, BerlinModelTransformer.uuidMisappliedCommonName,"Misapplied Common Name in Berlin Model", "Misapplied taxon was automatically created by Berlin Model import for a common name with a misapplied name reference", "MCN");
 309                                                         Marker marker = Marker.NewInstance(misCommonNameMarker, true);
 310                                                         misappliedNameTaxon.addMarker(marker);
 311                                                         taxaToSave.add(misappliedNameTaxon);
 312                                                         logger.warn("Misapplied name taxon could not be found in database but misapplied name reference exists for common name. " +
 313                                                                         "New misapplied name for misapplied reference common name was added. CommonNameId: " + commonNameId);
 314                                                 }
 315                                         }
 316                                         if (misappliedNameTaxon != null){
 317
 318                                                 if (! taxon.getMisappliedNames().contains(misappliedNameTaxon)){
 319                                                         taxon.addMisappliedName(misappliedNameTaxon,state.getTransactionalSourceReference(), null);
 320                                                         logger.warn("Misapplied name for common name was not found related to the accepted taxon. Created new relationship. CommonNameId: " + commonNameId);
 321                                                 }
 322
 323                                                 TaxonDescription misappliedNameDescription = getDescription(misappliedNameTaxon);
 324                                                 for (CommonTaxonName commonTaxonName : commonTaxonNames){
 325                                                         CommonTaxonName commonNameClone = (CommonTaxonName)commonTaxonName.clone();
 326                                                         misappliedNameDescription.addElement(commonNameClone);
 327                                                 }
 328                                         }else{
 329                                                 logger.warn("Misapplied name is null for common name " + commonNameId);
 330                                         }
 331
 332                                 }
 333
 334
 335                                 //reference extensions
 336                                 if (reference != null){
 337                                         if (CdmUtils.isNotEmpty(refLanguage)){
 338                                                 ExtensionType refLanguageExtensionType = getExtensionType( state, REFERENCE_LANGUAGE_STRING_UUID, "reference language","The language of the reference","ref. lang.");
 339                                                 Extension.NewInstance(reference, refLanguage, refLanguageExtensionType);
 340                                         }
 341
 342                                         if (CdmUtils.isNotEmpty(refLanguageIso639_2)){
 343                                                 ExtensionType refLanguageIsoExtensionType = getExtensionType( state, REFERENCE_LANGUAGE_ISO639_2_UUID, "reference language iso 639-2","The iso 639-2 code of the references language","ref. lang. 639-2");
 344                                                 Extension.NewInstance(reference, refLanguageIso639_2, refLanguageIsoExtensionType);
 345                                         }
 346                                 }else if (CdmUtils.isNotEmpty(refLanguage) || CdmUtils.isNotEmpty(refLanguageIso639_2)){
 347                                         logger.warn("Reference is null (" + languageRefRefFk + ") but refLanguage (" + CdmUtils.Nz(refLanguage) + ") or iso639_2 (" + CdmUtils.Nz(refLanguageIso639_2) + ") was not null for common name ("+ commonNameId +")");
 348                                 }
 349
 350                                 //status
 351                                 if (CdmUtils.isNotEmpty(status)){
 352                                         AnnotationType statusAnnotationType = getAnnotationType( state, STATUS_ANNOTATION_UUID, "status","The status of this object","status", null);
 353                                         Annotation annotation = Annotation.NewInstance(status, statusAnnotationType, Language.DEFAULT());
 354                                         for (CommonTaxonName commonTaxonName : commonTaxonNames){
 355                                                 commonTaxonName.addAnnotation(annotation);
 356                                         }
 357
 358                                 }
 359
 360                                 //Notes
 361                                 for (CommonTaxonName commonTaxonName : commonTaxonNames){
 362                                         doIdCreatedUpdatedNotes(state, commonTaxonName, rs, String.valueOf(commonNameId), NAMESPACE);
 363                                 }
 364                                 partitioner.startDoSave();
 365                                 taxaToSave.add(taxon);
 366
 367                         }
 368                 } catch (SQLException e) {
 369                         logger.error("SQLException:" +  e);
 370                         return false;
 371                 } catch (ClassCastException e) {
 372                         e.printStackTrace();
 373                 }
 374
 375                 //      logger.info( i + " names handled");
 376                 getTaxonService().save(taxaToSave);
 377                 return success;
 378
 379         }
 380
 381
 382
 383         /**
 384          * Not used anymore. Use MisappliedName RIdentifier instead
 385          * @param biblioRefMap
 386          * @param nomRefMap
 387          * @param misNameRefFk
 388          * @param taxon
 389          */
 390         private boolean isFirstMisappliedName = true;
 391         private Taxon getMisappliedName(Map<String, Reference> biblioRefMap, Map<String, Reference> nomRefMap, Object misNameRefFk, Taxon taxon) {
 392                 Taxon misappliedTaxon = null;
 393                 Reference<?> misNameRef = getReferenceOnlyFromMaps(biblioRefMap, nomRefMap, String.valueOf(misNameRefFk));
 394                 misappliedTaxon = Taxon.NewInstance(taxon.getName(), misNameRef);
 395                 Set<String> includeProperty = new HashSet<String>();
 396                 try {
 397 //                      //IMatchStrategy matchStrategy = DefaultMatchStrategy.NewInstance(TaxonBase.class);
 398 //                      //List<TaxonBase> misappliedList1 = getCommonService().findMatching(misappliedTaxon, matchStrategy);
 399                         List<TaxonBase> misappliedList = getTaxonService().list(misappliedTaxon, includeProperty, null, null, null, null);
 400                         if (misappliedList.size() > 0){
 401                                 misappliedTaxon = CdmBase.deproxy(misappliedList.get(0), Taxon.class);
 402                         }
 403                 } catch (ClassCastException e) {
 404                         logger.error(e.getMessage());
 405                         if (isFirstMisappliedName){
 406                                 e.printStackTrace();
 407                                 isFirstMisappliedName = false;
 408                         }
 409                 }
 410                 return misappliedTaxon;
 411         }
 412
 413
 414
 415         /**
 416          * @param iso6392Map
 417          * @param iso639_2
 418          * @param languageString
 419          * @param originalLanguageString
 420          * @param state
 421          * @return
 422          */
 423         private Language getAndHandleLanguage(Map<String, Language> iso639Map,  String iso639_2, String iso639_1, String languageString, String originalLanguageString, BerlinModelImportState state) {
 424                 Language language;
 425                 if (CdmUtils.isNotEmpty(iso639_2)|| CdmUtils.isNotEmpty(iso639_1)  ){
 426                         //TODO test performance, implement in state
 427                         language = getLanguageFromIsoMap(iso639Map, iso639_2, iso639_1);
 428
 429                         if (language == null){
 430                                 language = getTermService().getLanguageByIso(iso639_2);
 431                                 iso639Map.put(iso639_2, language);
 432                                 if (language == null){
 433                                         language = getTermService().getLanguageByIso(iso639_1);
 434                                         iso639Map.put(iso639_1, language);
 435                                 }
 436                                 if (language == null){
 437                                         logger.warn("Language for code ISO693-2 '" + iso639_2 + "' and ISO693-1 '" + iso639_1 + "' was not found");
 438                                 }
 439                         }
 440                 } else if ("unknown".equals(languageString)){
 441                         language = Language.UNKNOWN_LANGUAGE();
 442                 } else if ("mallorquín".equalsIgnoreCase(languageString)){
 443                         language = getLanguage(state, BerlinModelTransformer.uuidLangMajorcan, "Majorcan", "Majorcan (original 'mallorquín')", null);
 444                 }else{
 445                         logger.warn("language ISO 639_1 and ISO 639_2 were empty for " + languageString);
 446                         language = null;
 447                 }
 448                 addOriginalLanguage(language, originalLanguageString);
 449                 return language;
 450         }
 451
 452
 453         /**
 454          * @param iso639Map
 455          * @param iso639_2
 456          * @param iso639_1
 457          * @return
 458          */
 459         private Language getLanguageFromIsoMap(Map<String, Language> iso639Map, String iso639_2, String iso639_1) {
 460                 Language language;
 461                 language = iso639Map.get(iso639_2);
 462                 if (language == null){
 463                         language = iso639Map.get(iso639_1);
 464                 }
 465                 return language;
 466         }
 467
 468         /**
 469          * @param language
 470          * @param originalLanguageString
 471          */
 472         private void addOriginalLanguage(Language language,     String originalLanguageString) {
 473                 if (CdmUtils.isBlank(originalLanguageString)){
 474                         return;
 475                 }else if (language == null){
 476                         logger.warn("Language could not be defined, but originalLanguageString exists: " + originalLanguageString);
 477                 }else {
 478                         Representation representation = language.getRepresentation(language);
 479                         if (representation == null){
 480                                 language.addRepresentation(Representation.NewInstance(originalLanguageString, originalLanguageString, originalLanguageString, language));
 481                                 getTermService().saveOrUpdate(language);
 482                         }
 483                 }
 484
 485         }
 486
 487
 488
 489         /**
 490          * @param state
 491          * @param regionFks
 492          * @param source
 493          * @return
 494          * @throws SQLException
 495          */
 496         private void getRegionFks(BerlinModelImportState state, SortedSet<Integer> regionFks, Source source) throws SQLException {
 497                 String sql = " SELECT DISTINCT RegionFks FROM emCommonName";
 498                 if (state.getConfig().getCommonNameFilter() != null){
 499                         sql += " WHERE " + state.getConfig().getCommonNameFilter();
 500                 }
 501
 502                 ResultSet rs = source.getResultSet(sql);
 503                 while (rs.next()){
 504                         String strRegionFks = rs.getString("RegionFks");
 505                         String[] regionFkArray = strRegionFks.split(",");
 506                         for (String regionFk: regionFkArray){
 507                                 regionFk = regionFk.trim();
 508                                 if (! StringUtils.isNumeric(regionFk) || "".equals(regionFk)  ){
 509                                         state.setUnsuccessfull();
 510                                         logger.warn("RegionFk is not numeric: " + regionFk);
 511                                 }else{
 512                                         regionFks.add(Integer.valueOf(regionFk));
 513                                 }
 514                         }
 515                 }
 516                 return;
 517         }
 518
 519
 520
 521         /**
 522          * @param state
 523          * @param sqlWhere
 524          * @param emTdwgMap
 525          * @throws SQLException
 526          */
 527         private void fillRegionMap(BerlinModelImportState state, String sqlWhere,
 528                         Map<String, String> emTdwgMap) throws SQLException {
 529                 Source source = state.getConfig().getSource();
 530                 String sql;
 531                 ResultSet rs;
 532                 sql = " SELECT RegionId, Region FROM emLanguageRegion WHERE RegionId IN ("+ sqlWhere+ ") ";
 533                 rs = source.getResultSet(sql);
 534                 while (rs.next()){
 535                         Object regionId = rs.getObject("RegionId");
 536                         String region = rs.getString("Region");
 537                         String[] splitRegion = region.split("-");
 538                         if (splitRegion.length <= 1){
 539                                 NamedArea newArea = getNamedArea(state, null, region, "Language region '" + region + "'", null, null, null);
 540 //                              getTermService().save(newArea);
 541                                 regionMap.put(String.valueOf(regionId), newArea);
 542                                 logger.warn("Found new area: " +  region);
 543                         }else if (splitRegion.length == 2){
 544                                 String emCode = splitRegion[1].trim();
 545                                 String tdwgCode = emTdwgMap.get(emCode);
 546                                 if (StringUtils.isNotBlank(tdwgCode) ){
 547                                         NamedArea tdwgArea = getNamedArea(state, tdwgCode);
 548                                         regionMap.put(String.valueOf(regionId), tdwgArea);
 549                                 }else{
 550                                         logger.warn("emCode did not map to valid tdwgCode: " +  CdmUtils.Nz(emCode) + "->" + CdmUtils.Nz(tdwgCode));
 551                                 }
 552                         }
 553                 }
 554         }
 555
 556
 557         /**
 558          * @param state
 559          * @param tdwgCode
 560          */
 561         private NamedArea getNamedArea(BerlinModelImportState state, String tdwgCode) {
 562                 NamedArea area;
 563                 if (tdwgCode.equalsIgnoreCase("Ab")){
 564                         area = getNamedArea(state, BerlinModelTransformer.uuidAzerbaijanNakhichevan, "Azerbaijan & Nakhichevan", "Azerbaijan (including Nakhichevan)",  "Ab", null, null);
 565                 }else if (tdwgCode.equalsIgnoreCase("Rf")){
 566                         area = WaterbodyOrCountry.RUSSIANFEDERATION();
 567 //                      getTermService().save(area);
 568                 }else if (tdwgCode.equalsIgnoreCase("Uk")){
 569                         area = getNamedArea(state, BerlinModelTransformer.uuidUkraineAndCrimea , "Ukraine & Crimea", "Ukraine (including Crimea)", "Uk", null, null);
 570                 }else{
 571                         area = TdwgArea.getAreaByTdwgAbbreviation(tdwgCode);
 572                 }
 573                 if (area == null){
 574                         logger.warn("Area is null for " + tdwgCode);
 575                 }
 576                 return area;
 577         }
 578
 579
 580
 581         /**
 582          * @param regionFks
 583          * @return
 584          */
 585         private String getSqlWhere(SortedSet<Integer> regionFks) {
 586                 String sqlWhere = "";
 587                 for (Integer regionFk : regionFks){
 588                         sqlWhere += regionFk + ",";
 589                 }
 590                 sqlWhere = sqlWhere.substring(0, sqlWhere.length()-1);
 591                 return sqlWhere;
 592         }
 593
 594
 595
 596         /**
 597          * @param source
 598          * @throws SQLException
 599          */
 600         private Map<String, String> getEmTdwgMap(Source source) throws SQLException {
 601                 String sql;
 602                 ResultSet rs;
 603                 Map<String, String> emTdwgMap = new HashMap<String, String>();
 604                 sql = " SELECT EmCode, TDWGCode FROM emArea ";
 605                 rs = source.getResultSet(sql);
 606                 while (rs.next()){
 607                         String emCode = rs.getString("EMCode");
 608                         String TDWGCode = rs.getString("TDWGCode");
 609                         if (StringUtils.isNotBlank(emCode) ){
 610                                 emCode = emCode.trim();
 611                                 if (emCode.equalsIgnoreCase("Ab") || emCode.equalsIgnoreCase("Rf")|| emCode.equalsIgnoreCase("Uk") ){
 612                                         emTdwgMap.put(emCode, emCode);
 613                                 }else if (StringUtils.isNotBlank(TDWGCode)){
 614                                         emTdwgMap.put(emCode, TDWGCode.trim());
 615                                 }
 616                         }
 617                 }
 618                 emTdwgMap.put("Ab / Ab(A)", "Ab");
 619                 emTdwgMap.put("Ga / Ga(F)", "FRA-FR");
 620                 emTdwgMap.put("It / It(I)", "ITA");
 621                 emTdwgMap.put("Uk / Uk(U)", "Uk");
 622                 emTdwgMap.put("Ar / Ar(A)", "TCS-AR");
 623                 return emTdwgMap;
 624         }
 625
 626
 627
 628
 629         /**
 630          * Returns the first non-image gallery description. Creates a new one if no description exists.
 631          * @param taxon
 632          * @return
 633          */
 634         private TaxonDescription getDescription(Taxon taxon) {
 635                 TaxonDescription result = null;
 636                 for (TaxonDescription taxonDescription : taxon.getDescriptions()){
 637                         if (! taxonDescription.isImageGallery()){
 638                                 result = taxonDescription;
 639                         }
 640                 }
 641                 if (result == null){
 642                         result = TaxonDescription.NewInstance(taxon);
 643                 }
 644                 return result;
 645         }
 646
 647         /* (non-Javadoc)
 648          * @see eu.etaxonomy.cdm.io.berlinModel.in.IPartitionedIO#getRelatedObjectsForPartition(java.sql.ResultSet)
 649          */
 650         public Map<Object, Map<String, ? extends CdmBase>> getRelatedObjectsForPartition(ResultSet rs) {
 651                 String nameSpace;
 652                 Class<?> cdmClass;
 653                 Set<String> idSet;
 654                 Map<Object, Map<String, ? extends CdmBase>> result = new HashMap<Object, Map<String, ? extends CdmBase>>();
 655
 656                 try{
 657                         Set<String> taxonIdSet = new HashSet<String>();
 658                         Set<String> nameIdSet = new HashSet<String>();
 659                         Set<String> referenceIdSet = new HashSet<String>();
 660                         while (rs.next()){
 661                                 handleForeignKey(rs, taxonIdSet, "taxonId");
 662                                 handleForeignKey(rs, taxonIdSet, "misappliedTaxonId");
 663                                 handleForeignKey(rs, referenceIdSet, "refId");
 664                                 handleForeignKey(rs, referenceIdSet, "languageRefRefFk");
 665                                 handleForeignKey(rs, nameIdSet, "NameInSourceFk");
 666                                 handleForeignKey(rs, nameIdSet, "PTNameFk");
 667                                 handleForeignKey(rs, referenceIdSet, "MisNameRefFk");
 668                         }
 669
 670                         //name map
 671                         nameSpace = BerlinModelTaxonNameImport.NAMESPACE;
 672                         cdmClass = TaxonNameBase.class;
 673                         idSet = nameIdSet;
 674                         Map<String, TaxonNameBase<?,?>> nameMap = (Map<String, TaxonNameBase<?,?>>)getCommonService().getSourcedObjectsByIdInSource(cdmClass, idSet, nameSpace);
 675                         result.put(nameSpace, nameMap);
 676
 677                         //taxon map
 678                         nameSpace = BerlinModelTaxonImport.NAMESPACE;
 679                         cdmClass = TaxonBase.class;
 680                         idSet = taxonIdSet;
 681                         Map<String, TaxonBase<?>> taxonMap = (Map<String, TaxonBase<?>>)getCommonService().getSourcedObjectsByIdInSource(cdmClass, idSet, nameSpace);
 682                         result.put(nameSpace, taxonMap);
 683
 684                         //nom reference map
 685                         nameSpace = BerlinModelReferenceImport.NOM_REFERENCE_NAMESPACE;
 686                         cdmClass = Reference.class;
 687                         idSet = referenceIdSet;
 688                         Map<String, Reference> nomReferenceMap = (Map<String, Reference>)getCommonService().getSourcedObjectsByIdInSource(cdmClass, idSet, nameSpace);
 689                         result.put(nameSpace, nomReferenceMap);
 690
 691                         //biblio reference map
 692                         nameSpace = BerlinModelReferenceImport.BIBLIO_REFERENCE_NAMESPACE;
 693                         cdmClass = Reference.class;
 694                         idSet = referenceIdSet;
 695                         Map<String, Reference<?>> biblioReferenceMap = (Map<String, Reference<?>>)getCommonService().getSourcedObjectsByIdInSource(cdmClass, idSet, nameSpace);
 696                         result.put(nameSpace, biblioReferenceMap);
 697
 698                 } catch (SQLException e) {
 699                         throw new RuntimeException(e);
 700                 } catch (NullPointerException nep){
 701                         logger.error("NullPointerException in getRelatedObjectsForPartition()");
 702                 }
 703                 return result;
 704         }
 705
 706
 707
 708         /* (non-Javadoc)
 709          * @see eu.etaxonomy.cdm.io.common.CdmIoBase#doCheck(eu.etaxonomy.cdm.io.common.IoStateBase)
 710          */
 711         @Override
 712         protected boolean doCheck(BerlinModelImportState state){
 713                 IOValidator<BerlinModelImportState> validator = new BerlinModelCommonNamesImportValidator();
 714                 return validator.validate(state);
 715         }
 716
 717
 718         /* (non-Javadoc)
 719          * @see eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelImportBase#getTableName()
 720          */
 721         @Override
 722         protected String getTableName() {
 723                 return dbTableName;
 724         }
 725
 726         /* (non-Javadoc)
 727          * @see eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelImportBase#getPluralString()
 728          */
 729         @Override
 730         public String getPluralString() {
 731                 return pluralString;
 732         }
 733
 734         /* (non-Javadoc)
 735          * @see eu.etaxonomy.cdm.io.common.CdmIoBase#isIgnore(eu.etaxonomy.cdm.io.common.IImportConfigurator)
 736          */
 737         protected boolean isIgnore(BerlinModelImportState state){
 738                 return ! state.getConfig().isDoCommonNames();
 739         }
 740
 741 }