Project

General

Profile

« Previous | Next » 

Revision a77d754b

Added by Andreas Müller almost 5 years ago

final changes to E+M import

View differences:

app-import/src/main/java/eu/etaxonomy/cdm/io/berlinModel/in/BerlinModelAreaImport.java
378 378
	private OrderedTermVocabulary<NamedArea> makeEmptyEuroMedVocabulary() {
379 379
		TermType type = TermType.NamedArea;
380 380
		String description = "Euro+Med area vocabulary";
381
		String label = "E+M areas";
381
		String label = "E+M Areas";
382 382
		String abbrev = null;
383 383
		URI termSourceUri = null;
384 384
		OrderedTermVocabulary<NamedArea> result = OrderedTermVocabulary.NewInstance(type, description, label, abbrev, termSourceUri);
app-import/src/main/java/eu/etaxonomy/cdm/io/berlinModel/in/BerlinModelAuthorTeamImport.java
234 234
     */
235 235
    private TeamOrPersonBase<?> handleTeam(BerlinModelImportState state, Team team, String authorTeamCache,
236 236
            String fullAuthorTeamCache, boolean preliminaryFlag, int authorTeamId) {
237

  
237 238
        if (!team.getTeamMembers().isEmpty()){
238 239
            return team;
239 240
        }
app-import/src/main/java/eu/etaxonomy/cdm/io/berlinModel/in/BerlinModelCommonNamesImport.java
49 49
import eu.etaxonomy.cdm.model.taxon.TaxonBase;
50 50
import eu.etaxonomy.cdm.model.term.OrderedTermVocabulary;
51 51
import eu.etaxonomy.cdm.model.term.Representation;
52
import eu.etaxonomy.cdm.model.term.TermType;
52 53
import eu.etaxonomy.cdm.model.term.TermVocabulary;
53 54

  
54 55
/**
......
87 88
		if (isNotBlank(state.getConfig().getCommonNameFilter())){
88 89
			result += " AND " + state.getConfig().getCommonNameFilter();
89 90
		}
90
		result += " ORDER BY CommonNameId ";
91
		result += " ORDER BY PTNameFk, CommonNameId ";
91 92

  
92 93
		return result;
93 94
	}
......
118 119
                         "     LEFT OUTER JOIN PTaxon acc ON rel.PTNameFk2 = acc.PTNameFk AND rel.PTRefFk2 = acc.PTRefFk " +
119 120
                         "     LEFT OUTER JOIN Name accName ON accName.NameId = acc.PTNameFk " +
120 121
			        " WHERE cn.CommonNameId IN (" + ID_LIST_TOKEN + ") " +
121
			        " ORDER BY cn.CommonNameId ";
122
			        " ORDER BY cn.PTNameFk, cn.CommonNameId ";
122 123

  
123 124
		return recordQuery;
124 125
	}
......
172 173
	}
173 174

  
174 175

  
176
    private Map<String, Language> iso6392Map = new HashMap<>();
177

  
175 178
    @Override
176 179
	public boolean doPartition(@SuppressWarnings("rawtypes") ResultSetPartitioner partitioner, BerlinModelImportState state)  {
177 180
		boolean success = true ;
......
185 188
		@SuppressWarnings("unchecked")
186 189
        Map<String, Reference> refMap = partitioner.getObjectMap(BerlinModelReferenceImport.REFERENCE_NAMESPACE);
187 190

  
188
		Map<String, Language> iso6392Map = new HashMap<>();
189 191

  
190 192
	//	logger.warn("MisappliedNameRefFk  not yet implemented for Common Names");
191 193

  
......
563 565
	 * @param emTdwgMap
564 566
	 * @throws SQLException
565 567
	 */
566
	private void fillRegionMap(BerlinModelImportState state, String sqlWhere,
568
	private void fillRegionMap_old(BerlinModelImportState state, String sqlWhere,
567 569
			Map<String, NamedArea> emCodeToAreaMap) throws SQLException {
568 570

  
569 571
	    Source source = state.getConfig().getSource();
......
607 609
		}
608 610
	}
609 611

  
612
   private void fillRegionMap(BerlinModelImportState state, String sqlWhere,
613
            Map<String, NamedArea> emCodeToAreaMap) throws SQLException {
614

  
615
        OrderedTermVocabulary<NamedArea> voc = areaVoc = OrderedTermVocabulary.NewInstance(TermType.NamedArea, "Euro+Med common name areas", "E+M Common Name Areas", null, null);
616
        getVocabularyService().save(areaVoc);
617

  
618
        Map<String,NamedArea> existingAreas = new HashMap<>();
619
        Source source = state.getConfig().getSource();
620
        String sql =
621
              " SELECT RegionId, Region "
622
            + " FROM  emLanguageRegion "
623
            + " WHERE RegionId IN ("+ sqlWhere+ ") "
624
            + " ORDER BY Region ";
625
        ResultSet rs = source.getResultSet(sql);
626
        while (rs.next()){
627
            Object regionId = rs.getObject("RegionId");
628
            String region = rs.getString("Region");
629

  
630
            NamedArea area;
631
            if (existingAreas.containsKey(region)){
632
                area = existingAreas.get(region);
633
            }else{
634

  
635
                String[] splitRegion = region.split("-");
636

  
637
                String emMapping = "None";
638
                if (splitRegion.length == 2){
639
                    String emCode = splitRegion[1].trim().replace(" ", "");
640

  
641
                    NamedArea emArea = emCodeToAreaMap.get(emCode);
642
                    if (emArea == null){
643
                        emArea = normalizeAmbigousAreas(emCode, emCodeToAreaMap);
644
                    }
645
                    if (emArea == null){
646

  
647
                        String[] splits = emCode.split("/");
648
                        if (splits.length == 2){
649
                            emArea = emCodeToAreaMap.get(splits[0]);
650
                        }
651
                        if (emArea != null){
652
                            logger.warn("emCode ambigous. This should not happen anymore due to normalization! Use larger area as default: " +  CdmUtils.Nz(emCode) + "->" + regionId);
653
                        }else{
654
                            logger.warn("emCode not recognized. Region not defined: " +  CdmUtils.Nz(emCode) + "->" + regionId);
655
                        }
656
                    }
657
                    emMapping = emArea == null? "not recognized": emArea.getIdInVocabulary();
658
                }
659

  
660
                String label = splitRegion[0].trim();
661
                String description = "Language region '" + region + "'; EM Area Mapping: " + emMapping ;
662
                area = getNamedArea(state, null, label, description, null, null, null, voc, null);
663
                existingAreas.put(region, area);
664
            }
665
            regionFkToAreaMap.put(String.valueOf(regionId), area);
666
        }
667
    }
668

  
610 669
	/**
611 670
     * Use area according to mail ERS 2018-09-24
612 671
	 * @param emCodeToAreaMap
......
723 782
    private NamedArea getAreaByAreaId(int areaId) {
724 783
        NamedArea result = null;
725 784
        String areaIdStr = String.valueOf(areaId);
726
        OrderedTermVocabulary<NamedArea> voc = getAreaVoc();
785
        OrderedTermVocabulary<NamedArea> voc = getEmAreaVoc();
727 786
        getVocabularyService().update(voc);
728 787
        for (NamedArea area : voc.getTerms()){
729 788
            for (IdentifiableSource source : area.getSources()){
......
740 799

  
741 800
    private OrderedTermVocabulary<NamedArea> areaVoc;
742 801
    @SuppressWarnings("unchecked")
743
    private OrderedTermVocabulary<NamedArea> getAreaVoc(){
802
    private OrderedTermVocabulary<NamedArea> getEmAreaVoc(){
744 803
        if (areaVoc == null){
745 804
            areaVoc = (OrderedTermVocabulary<NamedArea>)getVocabularyService().find(BerlinModelTransformer.uuidVocEuroMedAreas);
746 805
        }
app-import/src/main/java/eu/etaxonomy/cdm/io/berlinModel/in/BerlinModelImportBase.java
142 142
			annotatableEntity.addAnnotation(annotation);
143 143
		}else if (config.getEditor().equals(EDITOR.EDITOR_AS_EDITOR)){
144 144
		    User creator;
145
		    boolean xmlSourceAdded= addXmlSource(state, annotatableEntity, createdWho, false);
145
		    boolean xmlSourceAdded= addXmlSource(state, rs, annotatableEntity, createdWho, false);
146 146
		    if (xmlSourceAdded){
147 147
		        creator = getXmlImporter(state);
148 148
		    }else{
......
151 151
		    annotatableEntity.setCreatedBy(creator);
152 152

  
153 153
		    User updator;
154
		    xmlSourceAdded = addXmlSource(state, annotatableEntity, updatedWho, xmlSourceAdded);
154
		    xmlSourceAdded = addXmlSource(state, rs, annotatableEntity, updatedWho, xmlSourceAdded);
155 155
		    if (xmlSourceAdded){
156 156
		        updator = getXmlImporter(state);
157 157
		    }else{
......
187 187

  
188 188
    /**
189 189
     * @param state
190
     * @param rs
190 191
	 * @param annotatableEntity
191 192
	 * @param xmlSourceAdded
192 193
     * @return
194
     * @throws SQLException
193 195
     */
194
    private boolean addXmlSource(BerlinModelImportState state, AnnotatableEntity annotatableEntity, String username, boolean existsAlready) {
195
        if (!state.getConfig().isEuroMed() || !isXmlImport(username)){
196
    private boolean addXmlSource(BerlinModelImportState state, ResultSet rs, AnnotatableEntity annotatableEntity, String username, boolean existsAlready) throws SQLException {
197
        if (!state.getConfig().isEuroMed()){
196 198
            return false;
197 199
        }
198 200
        if (isXmlImport(username) && existsAlready){
199 201
            return true;
200 202
        }
201
        Reference ref = getXmlRef(state, username);
202
        if (annotatableEntity.isInstanceOf(SourcedEntityBase.class)){
203
            SourcedEntityBase<?> sourcedEntity = CdmBase.deproxy(annotatableEntity, SourcedEntityBase.class);
204
            sourcedEntity.addImportSource(null, null, ref, null);
205
        }else if (annotatableEntity.isInstanceOf(DescriptionElementBase.class)){
206
            DescriptionElementBase descriptionElement = CdmBase.deproxy(annotatableEntity, DescriptionElementBase.class);
207
            descriptionElement.addImportSource(null, null, ref, null);
208
        }else{
209
            return false;
203
        String idInSource = getIdInSource(state, rs);
204

  
205
        boolean isXmlUser = isXmlImport(username);
206
        Reference ref = isXmlUser? getXmlRef(state, username): null;
207
        if (ref != null || isNotBlank(idInSource)){
208
            if (annotatableEntity.isInstanceOf(SourcedEntityBase.class)){
209
                SourcedEntityBase<?> sourcedEntity = CdmBase.deproxy(annotatableEntity, SourcedEntityBase.class);
210
                sourcedEntity.addImportSource(idInSource, null, ref, null);
211
            }else if (annotatableEntity.isInstanceOf(DescriptionElementBase.class)){
212
                DescriptionElementBase descriptionElement = CdmBase.deproxy(annotatableEntity, DescriptionElementBase.class);
213
                descriptionElement.addImportSource(idInSource, null, ref, null);
214
            }else {
215
                return false;
216
            }
210 217
        }
211
        return true;
218
        return isXmlUser;
219
    }
220

  
221
    //can be overriden
222
    protected String getIdInSource(BerlinModelImportState state, ResultSet rs) throws SQLException {
223
        return null;
212 224
    }
213 225

  
214 226

  
......
237 249
                ref = ReferenceFactory.newDatabase();
238 250
                ref.setTitleCache(username, true);
239 251
                ref.setTitle(username);
252
                ref.addImportSource(null, this.getTableName(), state.getTransactionalSourceReference(), null);
240 253
                getReferenceService().save(ref);
241 254
            }
242 255
            state.addRelatedObject(namespace, username, ref);
......
256 269
            return false;
257 270
        }
258 271
        return username.matches(".*\\.xml")
259
                || username.equals("J.Li: import from pandora")
260
                || username.equals("J.Li imported from pandora")
272
                ||username.equals("MCL-Import, a.mueller")
273
                ||username.equals("pandora import (J.Li)")
261 274
                ||username.equals("Import from Kew Checklist 2010")
262
                ||username.equals("Import from ILDIS 2010");
275
                ||username.equals("Import from ILDIS 2010")
276
                ||username.equals("s_em_ImportCastroviejoReferences")
277
                ||username.equals("s_em_ImportUotilaReferences")
278
                ||username.equals("s_em_ImportValdesReferences")
279
                ||username.equals("s_em_NewImportNomReferences")
280
                ||username.equals("Import from Anthos 2010");
263 281
    }
264 282

  
265 283

  
......
527 545
                    username = username.substring(0, username.length()-2);
528 546
                }else if(username.matches("(mariam[1-4]|palermo|palma|paltar)")){
529 547
                    username = "mariam";
548
                }else if (username.matches("a.mueller.*") || "AM".equals(username)){
549
                    username = "a.mueller";
550
                }else if (username.matches("sh,.*")){
551
                    username = "sh";
552
                }else if (username.matches("J.Li.*pandora")){
553
                    username = "pandora import (J.Li)";
554
                }else if (username.matches("euromed")){
555
                    username = "em";
530 556
                }
531 557
                if(username.matches("kapet")){
532 558
                    username = "kpet";
app-import/src/main/java/eu/etaxonomy/cdm/io/berlinModel/in/BerlinModelTaxonImport.java
40 40
import eu.etaxonomy.cdm.model.common.CdmBase;
41 41
import eu.etaxonomy.cdm.model.common.Extension;
42 42
import eu.etaxonomy.cdm.model.common.ExtensionType;
43
import eu.etaxonomy.cdm.model.common.Identifier;
43 44
import eu.etaxonomy.cdm.model.common.Language;
44 45
import eu.etaxonomy.cdm.model.common.Marker;
45 46
import eu.etaxonomy.cdm.model.common.MarkerType;
......
51 52
import eu.etaxonomy.cdm.model.taxon.Synonym;
52 53
import eu.etaxonomy.cdm.model.taxon.Taxon;
53 54
import eu.etaxonomy.cdm.model.taxon.TaxonBase;
55
import eu.etaxonomy.cdm.model.term.DefinedTerm;
54 56

  
55 57

  
56 58
/**
......
253 255
					//detail
254 256
					String detail = rs.getString("Detail");
255 257
					if (isNotBlank(detail)){
256
						ExtensionType detailExtensionType = getExtensionType(state, BerlinModelTransformer.DETAIL_EXT_UUID, "micro reference","micro reference","micro ref.");
257
						Extension.NewInstance(taxonBase, detail, detailExtensionType);
258
//						ExtensionType detailExtensionType = getExtensionType(state, BerlinModelTransformer.DETAIL_EXT_UUID, "micro reference","micro reference","micro ref.");
259
//						Extension.NewInstance(taxonBase, detail, detailExtensionType);
260
						taxonBase.setSecMicroReference(detail.trim());
258 261
					}
259 262
					//idInSource
260 263
					String idInSource = rs.getString("IdInSource");
261 264
					if (isNotBlank(idInSource)){
262
						ExtensionType detailExtensionType = getExtensionType(state, BerlinModelTransformer.ID_IN_SOURCE_EXT_UUID, "Berlin Model IdInSource","Berlin Model IdInSource","BM source id");
263
						Extension.NewInstance(taxonBase, idInSource, detailExtensionType);
265
						if(!state.getConfig().isEuroMed()){
266
						    ExtensionType detailExtensionType = getExtensionType(state, BerlinModelTransformer.ID_IN_SOURCE_EXT_UUID, "Berlin Model IdInSource","Berlin Model IdInSource","BM source id");
267
						    Extension.NewInstance(taxonBase, idInSource.trim(), detailExtensionType);
268
						}else if(isMclIdentifier(state,rs, idInSource)){
269
						    DefinedTerm identifierType = getIdentiferType(state, BerlinModelTransformer.uuidEM_MCLIdentifierType, "MCL identifier", "Med-Checklist identifier", "MCL ID", null);
270
						    Identifier.NewInstance(taxonBase, idInSource.trim(), identifierType);
271
						}
264 272
					}
265 273
					//namePhrase
266 274
					String namePhrase = rs.getString("NamePhrase");
......
368 376
		return success;
369 377
	}
370 378

  
379
    /**
380
     * @param state
381
     * @param rs
382
     * @param idInSource
383
     * @return
384
     * @throws SQLException
385
     */
386
    private boolean isMclIdentifier(BerlinModelImportState state, ResultSet rs, String idInSource) throws SQLException {
387
        if (idInSource.contains("-")){
388
            return true;
389
        }else if (idInSource.matches("(293|303)")){
390
            String created = rs.getString("Created_Who");
391
            if (created.endsWith(".xml")){
392
                return true;
393
            }
394
        }
395
        return false;
396
    }
397

  
398
    @Override
399
    protected String getIdInSource(BerlinModelImportState state, ResultSet rs) throws SQLException {
400
        String id = rs.getString("idInSource");
401
        return id;
402
    }
403

  
371 404

  
372 405
    /**
373 406
     * @param refFkInt
app-import/src/main/java/eu/etaxonomy/cdm/io/berlinModel/in/BerlinModelTaxonNameImport.java
32 32
import eu.etaxonomy.cdm.io.common.ImportHelper;
33 33
import eu.etaxonomy.cdm.io.common.ResultSetPartitioner;
34 34
import eu.etaxonomy.cdm.io.common.Source;
35
import eu.etaxonomy.cdm.io.common.utils.ImportDeduplicationHelper;
35 36
import eu.etaxonomy.cdm.model.agent.Team;
36 37
import eu.etaxonomy.cdm.model.agent.TeamOrPersonBase;
37 38
import eu.etaxonomy.cdm.model.common.Annotation;
......
78 79

  
79 80
	public static final String NAMESPACE = "TaxonName";
80 81

  
81
	   public static final String NAMESPACE_PRELIM = "RefDetail_Preliminary";
82

  
83
	public static final UUID SOURCE_ACC_UUID = UUID.fromString("c3959b4f-d876-4b7a-a739-9260f4cafd1c");
82
	public static final String NAMESPACE_PRELIM = "RefDetail_Preliminary";
84 83

  
85 84
	private static int modCount = 5000;
86 85
	private static final String pluralString = "TaxonNames";
......
257 256
		Set<TaxonName> namesToSave = new HashSet<>();
258 257
		@SuppressWarnings("unchecked")
259 258
        Map<String, Team> teamMap = partitioner.getObjectMap(BerlinModelAuthorTeamImport.NAMESPACE);
259
		//to dedup preliminary refDetail references
260
		dedupHelper.restartSession(this, null);
261

  
260 262

  
261 263
		ResultSet rs = partitioner.getResultSet();
262 264

  
......
376 378
					if (colExists){
377 379
						String sourceAcc = rs.getString("Source_Acc");
378 380
						if (isNotBlank(sourceAcc)){
379
							ExtensionType sourceAccExtensionType = getExtensionType(state, SOURCE_ACC_UUID, "Source_Acc","Source_Acc","Source_Acc");
381
							ExtensionType sourceAccExtensionType = getExtensionType(state, BerlinModelTransformer.SOURCE_ACC_UUID, "Source_Acc","Source_Acc","Source_Acc");
380 382
							Extension.NewInstance(taxonName, sourceAcc, sourceAccExtensionType);
381 383
						}
382 384
					}
......
911 913
                }else{
912 914
                    System.out.println("Final Candidates empty but author exists - should not happen: " + taxonName.getTitleCache());
913 915
                }
914
                handleNoMatch(state, taxonName, detail, genericCandidate, finalCandidates, fullNomRefCache, parsedCandidates);
916
                handleNoMatch(state, refDetailId, taxonName, detail, genericCandidate, finalCandidates, fullNomRefCache, parsedCandidates);
915 917
//                printResult(MatchType.NO_MATCH, unparsedAndName(fullNomRefCache, taxonName));
916 918
            }else if (hasOnlyUnparsedExemplars(finalCandidates)){
917 919
                printResult(MatchType.UNPARSED, unparsedAndName(fullNomRefCache, taxonName));
918 920
            }else if (hasNoCandidateExemplars(finalCandidates)){
919 921
                //but we can define the ref type here
920
                handleNoMatch(state, taxonName, detail, genericCandidate, finalCandidates, fullNomRefCache, parsedCandidates);
922
                handleNoMatch(state, refDetailId, taxonName, detail, genericCandidate, finalCandidates, fullNomRefCache, parsedCandidates);
921 923
//                printResult(MatchType.NO_MATCH, unparsedAndName(fullNomRefCache, taxonName));
922 924
            }else{
923 925
                String message = resultMessage(fullNomRefCache, finalCandidates, taxonName);
......
925 927
            }
926 928
        }else if (matchingCandidates.size() == 1){
927 929
            ReferenceCandidate single = matchingCandidates.iterator().next().candidate;
928
            addAuthorAndDetail(taxonName, single);
930
            addAuthorAndDetail(state, refDetailId, taxonName, single);
929 931
            if (single.ref.isPersited()){
930 932
                printResult(MatchType.SINGLE_FULL_MATCH, unparsedAndName(fullNomRefCache, taxonName));
931 933
            }else{
932
                single.ref.addImportSource(String.valueOf(refDetailId), BerlinModelRefDetailImport.REFDETAIL_NAMESPACE,
933
                        state.getTransactionalSourceReference(), null);
934
//                single.ref.addImportSource(String.valueOf(refDetailId), BerlinModelRefDetailImport.REFDETAIL_NAMESPACE,
935
//                        state.getTransactionalSourceReference(), null);
934 936
                printResult(MatchType.SINGLE_INREF_MATCH,  unparsedAndName(fullNomRefCache, taxonName));
935 937
            }
936 938
        }else{
937
            FinalCandidate finCand = findBestMatchingFinalCandidate(taxonName, matchingCandidates, fullNomRefCache);
938
            addAuthorAndDetail(taxonName, finCand.candidate);
939
            FinalCandidate finCand = findBestMatchingFinalCandidate(state, refDetailId, taxonName, matchingCandidates, fullNomRefCache);
940
//            addAuthorAndDetail(taxonName, finCand.candidate);
939 941
        }
940 942
    }
941 943

  
......
949 951
     * @param fullNomRefCache
950 952
     * @param parsedCandidates
951 953
     */
952
    private void handleNoMatch(BerlinModelImportState state, TaxonName taxonName, String detail,
954
    private void handleNoMatch(BerlinModelImportState state, int refDetailId, TaxonName taxonName, String detail,
953 955
            Reference genericCandidate, Set<FinalCandidate> finalCandidates, String fullNomRefCache,
954 956
            Set<Reference> parsedCandidatesAsRef) {
955 957
        Set<FinalCandidate> parsedCandidates = getParsedExemplars(finalCandidates);
......
962 964
        }else if (parsedCandidates.size() == 1){
963 965

  
964 966
            ReferenceCandidate refCand = parsedCandidates.iterator().next().exemplar;
965
            addAuthorAndDetail(taxonName, refCand);
967
            addAuthorAndDetail(state, refDetailId, taxonName, refCand);
966 968
            if (refCand.ref.getType() == ReferenceType.Article){
967 969
                if(refCand.ref.getInReference().getAbbrevTitle().contains(",")){
968 970
                    printResult(MatchType.NO_MATCH_SINGLE_PARSE_ARTICLE_WITH_COMMA, unparsedAndName(fullNomRefCache, taxonName));
......
978 980
            }
979 981
        }else{
980 982
            ReferenceCandidate generCandidate = createGenericReference(parsedCandidates, detail);
981
            addAuthorAndDetail(taxonName, generCandidate);
983
            addAuthorAndDetail(state, refDetailId, taxonName, generCandidate);
982 984
            if (generCandidate.ref.getType() == ReferenceType.Article){
983 985
                if(generCandidate.ref.getInReference().getAbbrevTitle().contains(",")){
984 986
                    printResult(MatchType.NO_MATCH_SINGLE_PARSE_ARTICLE_WITH_COMMA, unparsedAndName(fullNomRefCache, taxonName));
......
1154 1156

  
1155 1157

  
1156 1158
    /**
1159
     * @param refDetailId
1160
     * @param state
1157 1161
     * @param taxonName
1158 1162
     * @param single
1159 1163
     */
1160
    private void addAuthorAndDetail(TaxonName taxonName, ReferenceCandidate refCand) {
1164
    private void addAuthorAndDetail(BerlinModelImportState state, int refDetailId, TaxonName taxonName, ReferenceCandidate refCand) {
1161 1165
        if (!CdmUtils.nullSafeEqual(refCand.ref.getAuthorship(), taxonName.getCombinationAuthorship())){
1162
            TeamOrPersonBase<?> refAut = refCand.ref.getAuthorship();
1163
            TeamOrPersonBase<?> nameAut = taxonName.getCombinationAuthorship();
1164
            if(refAut == null || nameAut == null){
1165
                logger.warn("refAut or nameAut was null");  //this happened once
1166
            TeamOrPersonBase<?> refAuthor = refCand.ref.getAuthorship();
1167
            TeamOrPersonBase<?> nameAuthor = taxonName.getCombinationAuthorship();
1168
            if(refAuthor == null || nameAuthor == null){
1169
                logger.warn("refAut or nameAut was null. RefDetailId = " + refDetailId);  //this happened once
1166 1170
                return;
1167 1171
            }
1168 1172
            try {
1169
                MatchResult match = MatchStrategyFactory.NewParsedTeamOrPersonInstance().invoke(refAut, nameAut, true);
1173
                MatchResult match = MatchStrategyFactory.NewParsedTeamOrPersonInstance().invoke(refAuthor, nameAuthor, true);
1170 1174
                if (match.isFailed()){
1171 1175
                    System.out.println("not same author \n"+ match);
1172 1176
                }else{
1173
                    taxonName.setCombinationAuthorship(refAut);
1177
                    taxonName.setCombinationAuthorship(refAuthor);
1174 1178
                }
1175 1179
            } catch (MatchException e) {
1176 1180
                // TODO Auto-generated catch block
1177 1181
                e.printStackTrace();
1178 1182
            }
1179 1183
        }
1180
        //FIXME deduplicate
1181
        taxonName.setNomenclaturalReference(refCand.ref);
1184
        if (refCand.ref.getSources().isEmpty()){
1185
            refCand.ref.addImportSource(String.valueOf(refDetailId), BerlinModelRefDetailImport.REFDETAIL_NAMESPACE, state.getTransactionalSourceReference(), null);
1186
        }
1187
//        if (!refCand.ref.isPersited()){
1188
//            getReferenceService().save(refCand.ref);
1189
//        }
1190

  
1191
        Reference nomRef = dedupHelper.getExistingReference(state, refCand.ref);
1192
        if (nomRef != refCand.ref){
1193
            System.out.println("Ref deduplicated: " + nomRef.getTitleCache() + "(RefDetId: " + refDetailId + ")");
1194
            if (nomRef.isPersited() && !getSession().contains(nomRef)){
1195
                System.out.println("Start load with uuid: " + nomRef.getUuid() + ")");
1196
                nomRef = getReferenceService().find(nomRef.getUuid());
1197
            }
1198
        }
1199

  
1200
        taxonName.setNomenclaturalReference(nomRef);
1182 1201
        taxonName.setNomenclaturalMicroReference(refCand.detail);
1183 1202
    }
1184 1203

  
1204
    private ImportDeduplicationHelper<BerlinModelImportState> dedupHelper = ImportDeduplicationHelper.NewInstance(null, null);
1205

  
1206

  
1185 1207

  
1186 1208

  
1187 1209
//
......
1263 1285

  
1264 1286

  
1265 1287
    /**
1266
     * @param taxonName
1267
     * @param finalCandidates
1268
     * @param exemplars
1269
     * @return
1288
     * return object not really necessary here
1270 1289
     */
1271
    private FinalCandidate findBestMatchingFinalCandidate(TaxonName taxonName,
1290
    private FinalCandidate findBestMatchingFinalCandidate(BerlinModelImportState state, int refDetailId, TaxonName taxonName,
1272 1291
            Set<FinalCandidate> finalCandidates, String fullNomRefCache) {
1273 1292
        try {
1274 1293
            Set<FinalCandidate> persistentMatches = findPersistentMatch(taxonName, finalCandidates);
......
1278 1297
                    Set<FinalCandidate> successCandidatesExacts = getSuccess(exactMatches);
1279 1298
                    if (successCandidatesExacts.size() >= 1){
1280 1299
                        FinalCandidate result = successCandidatesExacts.iterator().next();
1281
                        addAuthorAndDetail(taxonName, result.candidate);
1300
                        addAuthorAndDetail(state, refDetailId, taxonName, result.candidate);
1282 1301
//                        String message = resultMessage(fullNomRefCache, exactMatches, taxonName);
1283 1302
                        if (successCandidatesExacts.size()>1){
1284 1303
                            printResult(MatchType.MULTI_MULTI_PERSISTENT_MULTI_EXACT, unparsedAndName(fullNomRefCache, taxonName));
......
1290 1309
                        String message = resultMessage(fullNomRefCache, successCandidatesExacts, taxonName);
1291 1310
                        printResult(MatchType.MULTI_MULTI_PERSISTENT_NO_EXACT, message);
1292 1311
                        FinalCandidate result = persistentMatches.iterator().next();
1293
                        addAuthorAndDetail(taxonName, result.candidate);
1312
                        addAuthorAndDetail(state, refDetailId, taxonName, result.candidate);
1294 1313
                        return result;
1295 1314
                    }
1296 1315
                }else{
1297 1316
                    FinalCandidate result = persistentMatches.iterator().next();
1298
                    addAuthorAndDetail(taxonName, result.candidate);
1317
                    addAuthorAndDetail(state, refDetailId, taxonName, result.candidate);
1299 1318
                    printResult(MatchType.MULTI_SINGLE_PERSISTENT, taxonName.getFullTitleCache());
1300 1319
                    return result;
1301 1320
                }
......
1304 1323
            Set<FinalCandidate> successCandidatesExacts = getSuccess(exactMatches);
1305 1324
            if (successCandidatesExacts.size() >= 1){
1306 1325
                FinalCandidate result = successCandidatesExacts.iterator().next();
1307
                addAuthorAndDetail(taxonName, result.candidate);
1326
                addAuthorAndDetail(state, refDetailId, taxonName, result.candidate);
1308 1327
                String message = resultMessage(fullNomRefCache, exactMatches, taxonName);
1309 1328
                if (successCandidatesExacts.size()>1){
1310 1329
                    printResult(MatchType.MULTI_NO_PERSISTENT_MULTI_EXACT, message);
......
1315 1334
                return result;
1316 1335
            }else{
1317 1336
                FinalCandidate result = finalCandidates.iterator().next();
1318
                addAuthorAndDetail(taxonName, result.candidate);
1337
                addAuthorAndDetail(state, refDetailId, taxonName, result.candidate);
1319 1338
                String message = resultMessage(fullNomRefCache, exactMatches, taxonName);
1320 1339
                printResult(MatchType.MULTI_NO_PERSISTENT_NO_EXACT, message);
1321 1340
                return result;
1322 1341
            }
1323 1342
        } catch (MatchException e) {
1324 1343
            e.printStackTrace();
1325
            return finalCandidates.iterator().next();
1344
            FinalCandidate result = finalCandidates.iterator().next();
1345
            addAuthorAndDetail(state, refDetailId, taxonName, result.candidate);
1346
            return result;
1326 1347
        }
1327 1348
    }
1328 1349

  
app-import/src/main/java/eu/etaxonomy/cdm/io/berlinModel/in/BerlinModelTaxonRelationImport.java
302 302
                            }
303 303
							Taxon fromTaxon = (Taxon)taxon1;
304 304
							if (relQualifierFk == TAX_REL_IS_INCLUDED_IN){
305
							    taxonRelationship = makeTaxonomicallyIncluded(state, classificationMap, treeRefFk, fromTaxon, toTaxon, citation, microcitation);
305
							    Boolean provisional = rs.getBoolean("Provisional");
306
							    taxonRelationship = makeTaxonomicallyIncluded(state, classificationMap, treeRefFk, fromTaxon, toTaxon, citation, microcitation, provisional);
306 307
							}else if (relQualifierFk == TAX_REL_IS_MISAPPLIED_NAME_OF){
307 308
							    boolean isProParte = "p.p.".equals(notes);
308 309
							    if (isProParte){
......
765 766
	}
766 767

  
767 768
	private TaxonNode makeTaxonomicallyIncluded(BerlinModelImportState state, Map<Integer, Classification> classificationMap,
768
	        int treeRefFk, Taxon child, Taxon parent, Reference citation, String microCitation){
769
	        int treeRefFk, Taxon child, Taxon parent, Reference citation, String microCitation, Boolean provisional){
769 770
		Classification tree = getClassificationTree(state, classificationMap, treeRefFk);
770
		return tree.addParentChild(parent, child, citation, microCitation);
771
		TaxonNode result = tree.addParentChild(parent, child, citation, microCitation);
772
		result.setUnplaced(provisional);
773
		return result;
771 774
	}
772 775

  
773 776
	private Classification getClassificationTree(BerlinModelImportState state, Map<Integer, Classification> classificationMap, int treeRefFk) {
......
777 780
			}else{
778 781
				treeRefFk = 1;
779 782
			}
780

  
781 783
		}
782 784
		Classification tree = classificationMap.get(treeRefFk);
783 785
		if (tree == null){
app-import/src/main/java/eu/etaxonomy/cdm/io/berlinModel/in/validation/BerlinModelTaxonRelationImportValidator.java
335 335
	            rs = source.getResultSet(strSelect + strQueryBase + strOrderBy);
336 336
	            while (rs.next()){
337 337

  
338
	                int secRefFk = rs.getInt("secRefFk");
339
	                String secRef = rs.getString("secRef");
340
	                String nameCache = rs.getString("FullNameCache");
341
	                int nameId = rs.getInt("NameId");
342
	                String status = rs.getString("Status");
343

  
344
	                System.out.println("SecRef:" + secRefFk +
345
	                        "\n secRef: " + secRef + "\n name: " + nameCache + "\n nameId: " + nameId
346
	                        + "\n status: " + status
338
	                int relPTaxonId = rs.getInt("RelPTaxonId");
339

  
340
	                System.out.println("RelPTaxonId:" + relPTaxonId
341
//	                        + "\n secRef: " + secRef + "\n name: " + nameCache + "\n nameId: " + nameId
342
//	                        + "\n status: " + status
347 343
	                    );
348 344
	            }
349 345
	            success = (n == 0);
app-import/src/main/java/eu/etaxonomy/cdm/io/berlinModel/out/BerlinModelTaxonNameExport.java
17 17
import org.springframework.transaction.TransactionStatus;
18 18

  
19 19
import eu.etaxonomy.cdm.io.berlinModel.BerlinModelTransformer;
20
import eu.etaxonomy.cdm.io.berlinModel.in.BerlinModelTaxonNameImport;
21 20
import eu.etaxonomy.cdm.io.berlinModel.out.mapper.RefDetailMapper;
22 21
import eu.etaxonomy.cdm.io.berlinModel.out.mapper.TeamOrPersonMapper;
23 22
import eu.etaxonomy.cdm.io.common.Source;
......
123 122
		mapping.addMapper(DbObjectMapper.NewInstance("nomenclaturalReference", "NomRefFk"));
124 123
		mapping.addMapper(RefDetailMapper.NewInstance("nomenclaturalMicroReference","nomenclaturalReference", "NomRefDetailFk"));
125 124
		mapping.addMapper(CreatedAndNotesMapper.NewInstance(false));
126
		ExtensionType sourceAccExtensionType = (ExtensionType)getTermService().find(BerlinModelTaxonNameImport.SOURCE_ACC_UUID);
125
		ExtensionType sourceAccExtensionType = (ExtensionType)getTermService().find(BerlinModelTransformer.SOURCE_ACC_UUID);
127 126
		if (sourceAccExtensionType != null){
128 127
			mapping.addMapper(DbExtensionMapper.NewInstance(sourceAccExtensionType, "Source_Acc"));
129 128
		}

Also available in: Unified diff