Project

General

Profile

Revision 689cc1b7

ID689cc1b771fbc41e38e76f3891c60fc24ba78a3f
Parent 234b98e6
Child d5240fab

Added by Katja Luther almost 4 years ago

improve exports with progress monitor

View differences:

cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/cdmLight/CdmLightClassificationExport.java
22 22

  
23 23
import eu.etaxonomy.cdm.common.CdmUtils;
24 24
import eu.etaxonomy.cdm.common.monitor.IProgressMonitor;
25
import eu.etaxonomy.cdm.filter.TaxonNodeFilter;
25 26
import eu.etaxonomy.cdm.hibernate.HibernateProxyHelper;
26 27
import eu.etaxonomy.cdm.io.common.CdmExportBase;
27 28
import eu.etaxonomy.cdm.io.common.ExportResult.ExportResultState;
......
101 102

  
102 103
    }
103 104

  
105
    @Override
106
    public long countSteps(CdmLightExportState state) {
107
        TaxonNodeFilter filter = state.getConfig().getTaxonNodeFilter();
108
        return taxonNodeService.count(filter);
109
    }
110

  
111

  
104 112
    /**
105 113
     * {@inheritDoc}
106 114
     */
cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/common/TaxonNodeOutStreamPartitioner.java
164 164
        List<TaxonNode> partition = new ArrayList<>();
165 165
        if (!partList.isEmpty()){
166 166
            monitor.subTask(String.format("Reading partition %d/%d", currentPartition + 1, (totalCount / partitionSize) +1 ));
167
            List<String> propertyPaths = null;
167
            List<String> propertyPaths = new ArrayList<String>();
168
            propertyPaths.add("taxon");
169
            propertyPaths.add("taxon.name");
168 170
            partition = repository.getTaxonNodeService().loadByIds(partList, propertyPaths);
169
            monitor.worked(partition.size() * 1);
171
            monitor.worked(partition.size());
170 172
            currentPartition++;
171 173
            monitor.subTask(String.format("Writing partition %d/%d", currentPartition, (totalCount / partitionSize) +1 ));
172 174
        }
cdmlib-io/src/main/java/eu/etaxonomy/cdm/io/csv/caryophyllales/out/CsvNameExport.java
21 21

  
22 22
import eu.etaxonomy.cdm.api.service.dto.CondensedDistribution;
23 23
import eu.etaxonomy.cdm.api.service.pager.Pager;
24
import eu.etaxonomy.cdm.common.monitor.IProgressMonitor;
24 25
import eu.etaxonomy.cdm.ext.geo.CondensedDistributionRecipe;
25 26
import eu.etaxonomy.cdm.ext.geo.IEditGeoService;
27
import eu.etaxonomy.cdm.filter.TaxonNodeFilter;
28
import eu.etaxonomy.cdm.io.common.TaxonNodeOutStreamPartitioner;
29
import eu.etaxonomy.cdm.io.common.XmlExportState;
26 30
import eu.etaxonomy.cdm.model.common.CdmBase;
27 31
import eu.etaxonomy.cdm.model.common.DefinedTermBase;
28 32
import eu.etaxonomy.cdm.model.common.Language;
......
42 46
import eu.etaxonomy.cdm.model.name.NameTypeDesignationStatus;
43 47
import eu.etaxonomy.cdm.model.name.NomenclaturalStatus;
44 48
import eu.etaxonomy.cdm.model.name.Rank;
45
import eu.etaxonomy.cdm.model.name.TaxonName;
46 49
import eu.etaxonomy.cdm.model.name.TypeDesignationBase;
47 50
import eu.etaxonomy.cdm.model.reference.Reference;
48
import eu.etaxonomy.cdm.model.taxon.Classification;
49 51
import eu.etaxonomy.cdm.model.taxon.HomotypicGroupTaxonComparator;
50 52
import eu.etaxonomy.cdm.model.taxon.Synonym;
51 53
import eu.etaxonomy.cdm.model.taxon.Taxon;
......
65 67
    @Autowired
66 68
    IEditGeoService geoService;
67 69

  
70
    HashMap<UUID, HashMap<String,String>> familyMap = new HashMap();
71
    HashMap<UUID, HashMap<String,String>> genusMap = new HashMap();
72

  
73
    List<HashMap<String,String>> nameRecords = new ArrayList<>();
74

  
68 75
    public CsvNameExport() {
69 76
        super();
70 77
        this.ioName = this.getClass().getSimpleName();
......
139 146
    }
140 147

  
141 148
    @Override
149
    public long countSteps(CsvNameExportState state) {
150
        TaxonNodeFilter filter = state.getConfig().getTaxonNodeFilter();
151
        return taxonNodeService.count(filter);
152
    }
153

  
154

  
155
    @Override
142 156
    protected boolean doCheck(CsvNameExportState state) {
143 157
        boolean result = true;
144 158
        logger.warn("No check implemented for " + this.ioName);
......
152 166

  
153 167

  
154 168
    public List<HashMap<String,String>> getRecordsForPrintPub(CsvNameExportState state){
155
        List<String> propertyPaths = new ArrayList<String>();
156
        propertyPaths.add("childNodes");
157
        txStatus = startTransaction();
158
        Classification classification = getClassificationService().load(state.getConfig().getClassificationUUID());
159
        TaxonNode rootNode;
160
        if (classification != null){
161
            rootNode = classification.getRootNode();
162
        }else{
163
            List<Classification> classifications = getClassificationService().list(Classification.class, 10, 0, null, null);
164
            if (classifications.isEmpty()){
165
                return null;
169
//        List<String> propertyPaths = new ArrayList<String>();
170
//        propertyPaths.add("childNodes");
171
//        txStatus = startTransaction();
172
//        Classification classification = getClassificationService().load(state.getConfig().getClassificationUUID());
173
//        TaxonNode rootNode;
174
//        if (classification != null){
175
//            rootNode = classification.getRootNode();
176
//        }else{
177
//            List<Classification> classifications = getClassificationService().list(Classification.class, 10, 0, null, null);
178
//            if (classifications.isEmpty()){
179
//                return null;
180
//            }
181
//            classification = classifications.get(0);
182
//            rootNode = classification.getRootNode();
183
//        }
184
//        rootNode = getTaxonNodeService().load(rootNode.getUuid(), propertyPaths);
185
//        Set<UUID> childrenUuids = new HashSet<UUID>();
186
//
187
//
188
//        rootNode = CdmBase.deproxy(rootNode);
189
//        rootNode.removeNullValueFromChildren();
190
//        for (TaxonNode child: rootNode.getChildNodes()){
191
//            child = CdmBase.deproxy(child);
192
//            childrenUuids.add(child.getUuid());
193
//        }
194
//        Set<UUID> parentsNodesUUID = new HashSet<UUID>(childrenUuids);
195
//        childrenUuids.clear();
196
//        List<TaxonNode> childrenNodes = new ArrayList<TaxonNode>();
197
//
198
//        findChildren(state, childrenUuids, parentsNodesUUID);
199

  
200
        IProgressMonitor monitor = state.getConfig().getProgressMonitor();
201
         @SuppressWarnings("unchecked")
202
        TaxonNodeOutStreamPartitioner<XmlExportState> partitioner
203
          = TaxonNodeOutStreamPartitioner.NewInstance(
204
                this, state, state.getConfig().getTaxonNodeFilter(),
205
                100, monitor, null);
206

  
207

  
208
            monitor.subTask("Start partitioning");
209
//            List<HashMap<String,String>> nameRecords = new ArrayList<>();
210
            TaxonNode node = partitioner.next();
211
            while (node != null){
212
              HashMap<String, String> nameRecord = createNewRecord(node, state);
213
              if (nameRecord != null){
214
                  nameRecords.add(nameRecord);
215
              }
216
              node = partitioner.next();
166 217
            }
167
            classification = classifications.get(0);
168
            rootNode = classification.getRootNode();
169
        }
170
        rootNode = getTaxonNodeService().load(rootNode.getUuid(), propertyPaths);
171
        Set<UUID> childrenUuids = new HashSet<UUID>();
172 218

  
173 219

  
174
        rootNode = CdmBase.deproxy(rootNode);
175
        rootNode.removeNullValueFromChildren();
176
        for (TaxonNode child: rootNode.getChildNodes()){
177
            child = CdmBase.deproxy(child);
178
            childrenUuids.add(child.getUuid());
179
        }
180
        Set<UUID> parentsNodesUUID = new HashSet<UUID>(childrenUuids);
181
        childrenUuids.clear();
182
        List<TaxonNode> childrenNodes = new ArrayList<TaxonNode>();
183

  
184
        findChildren(state, childrenUuids, parentsNodesUUID);
185 220

  
186 221

  
187
        List<HashMap<String,String>> nameRecords = new ArrayList<>();
188

  
189
        childrenNodes = getTaxonNodeService().find(childrenUuids);
190
        for(TaxonNode genusNode : childrenNodes)   {
191
            nameRecords.add(createNewRecord(genusNode, state));
192
            refreshTransaction();
193
        }
222
//        List<HashMap<String,String>> nameRecords = new ArrayList<>();
223
//
224
//        childrenNodes = getTaxonNodeService().find(childrenUuids);
225
//        for(TaxonNode genusNode : childrenNodes)   {
226
//            nameRecords.add(createNewRecord(genusNode, state));
227
//            refreshTransaction();
228
//        }
194 229

  
195 230
        return nameRecords;
196 231

  
197 232
    }
198 233

  
199
    /**
200
     * @param state
201
     * @param childrenUuids
202
     * @param parentsNodesUUID
203
     * @param familyNode
204
     */
205
    private void findChildren(CsvNameExportState state, Set<UUID> childrenUuids, Set<UUID> parentsNodesUUID) {
206
        TaxonName name;
207
        List<TaxonNode> familyNodes = getTaxonNodeService().find(parentsNodesUUID);
208
        parentsNodesUUID =new HashSet<>();
209
        for (TaxonNode familyNode: familyNodes){
210
            familyNode = CdmBase.deproxy(familyNode);
211
            familyNode.removeNullValueFromChildren();
212
            for (TaxonNode child: familyNode.getChildNodes()){
213
                child = CdmBase.deproxy(child);
214
                Taxon taxon = CdmBase.deproxy(child.getTaxon());
215
                if (taxon != null){
216
                    name = CdmBase.deproxy(taxon.getName());
217
                    if (child.getTaxon().getName().getRank().isLower(state.getConfig().getRank()) ) {
218
                        childrenUuids.add(child.getUuid());
219
                        if (child.hasChildNodes()){
220
                            parentsNodesUUID.add(child.getUuid());
221
                        }
222
                    }else{
223
                        parentsNodesUUID.add(child.getUuid());
224
                    }
225
                }
226
            }
227
            //refreshTransaction();
228
            if (!parentsNodesUUID.isEmpty()){
229
                findChildren(state, childrenUuids, parentsNodesUUID);
230
            }
231
        }
232

  
233
    }
234
//    /**
235
//     * @param state
236
//     * @param childrenUuids
237
//     * @param parentsNodesUUID
238
//     * @param familyNode
239
//     */
240
//    private void findChildren(CsvNameExportState state, Set<UUID> childrenUuids, Set<UUID> parentsNodesUUID) {
241
//        TaxonName name;
242
//        List<TaxonNode> familyNodes = getTaxonNodeService().find(parentsNodesUUID);
243
//        parentsNodesUUID =new HashSet<>();
244
//        for (TaxonNode familyNode: familyNodes){
245
//            familyNode = CdmBase.deproxy(familyNode);
246
//            familyNode.removeNullValueFromChildren();
247
//            for (TaxonNode child: familyNode.getChildNodes()){
248
//                child = CdmBase.deproxy(child);
249
//                Taxon taxon = CdmBase.deproxy(child.getTaxon());
250
//                if (taxon != null){
251
//                    name = CdmBase.deproxy(taxon.getName());
252
//                    if (child.getTaxon().getName().getRank().isLower(state.getConfig().getRank()) ) {
253
//                        childrenUuids.add(child.getUuid());
254
//                        if (child.hasChildNodes()){
255
//                            parentsNodesUUID.add(child.getUuid());
256
//                        }
257
//                    }else{
258
//                        parentsNodesUUID.add(child.getUuid());
259
//                    }
260
//                }
261
//            }
262
//            //refreshTransaction();
263
//            if (!parentsNodesUUID.isEmpty()){
264
//                findChildren(state, childrenUuids, parentsNodesUUID);
265
//            }
266
//        }
267
//
268
//    }
234 269

  
235 270

  
236 271

  
......
247 282
        boolean isInvalidRel = false;
248 283
        for (NameRelationship nameRel: nameRelations){
249 284
           // NameRelationship nameRel = nameRelations.iterator().next();
250
            IBotanicalName fromName = CdmBase.deproxy(nameRel.getFromName());
251

  
252
            nameRel = CdmBase.deproxy(nameRel);
253
            nameRelType = nameRel.getType().getTitleCache();
254
            String relatedNameString = "";
255
            if (fromName.equals(synonymName)){
256
                relatedName = nameRel.getToName();
257
            }else{
258
                relatedName = nameRel.getFromName();
259
            }
260 285
            if (!nameRel.getType().equals(NameRelationshipType.BASIONYM())){
286
                IBotanicalName fromName = CdmBase.deproxy(nameRel.getFromName());
287

  
288
                nameRel = CdmBase.deproxy(nameRel);
289
                nameRelType = nameRel.getType().getTitleCache();
290
                String relatedNameString = "";
291
                if (fromName.equals(synonymName)){
292
                    relatedName = nameRel.getToName();
293
                }else{
294
                    relatedName = nameRel.getFromName();
295
                }
296

  
261 297
                isInvalidRel = getStatus(relatedName);
262 298
                relatedNameString = createTaggedNameString(relatedName, isInvalidRel&&isInvalid);
263 299

  
......
357 393

  
358 394

  
359 395

  
360
    private List<TaxonNode> getGenusNodes (UUID classificationUUID){
361
        Classification classification = getClassificationService().load(classificationUUID);
362
        TaxonNode rootNode = classification.getRootNode();
363
        rootNode = getTaxonNodeService().load(rootNode.getUuid());
364
        Set<UUID> childrenUuids = new HashSet<>();
365

  
366
        for (TaxonNode child: rootNode.getChildNodes()){
367
            child = CdmBase.deproxy(child);
368
            childrenUuids.add(child.getUuid());
369
        }
370

  
371
        List<TaxonNode> familyNodes = getTaxonNodeService().find(childrenUuids);
372
        childrenUuids.clear();
373
        List<TaxonNode> genusNodes = new ArrayList<>();
374
        for (TaxonNode familyNode: familyNodes){
375
            for (TaxonNode child: familyNode.getChildNodes()){
376
                child = CdmBase.deproxy(child);
377
                childrenUuids.add(child.getUuid());
378
            }
379

  
380
            genusNodes = getTaxonNodeService().find(childrenUuids);
381
        }
382
        return genusNodes;
383
    }
396
//    private List<TaxonNode> getGenusNodes (UUID classificationUUID){
397
//        Classification classification = getClassificationService().load(classificationUUID);
398
//        TaxonNode rootNode = classification.getRootNode();
399
//        rootNode = getTaxonNodeService().load(rootNode.getUuid());
400
//        Set<UUID> childrenUuids = new HashSet<>();
401
//
402
//        for (TaxonNode child: rootNode.getChildNodes()){
403
//            child = CdmBase.deproxy(child);
404
//            childrenUuids.add(child.getUuid());
405
//        }
406
//
407
//        List<TaxonNode> familyNodes = getTaxonNodeService().find(childrenUuids);
408
//        childrenUuids.clear();
409
//        List<TaxonNode> genusNodes = new ArrayList<>();
410
//        for (TaxonNode familyNode: familyNodes){
411
//            for (TaxonNode child: familyNode.getChildNodes()){
412
//                child = CdmBase.deproxy(child);
413
//                childrenUuids.add(child.getUuid());
414
//            }
415
//
416
//            genusNodes = getTaxonNodeService().find(childrenUuids);
417
//        }
418
//        return genusNodes;
419
//    }
384 420

  
385 421
    private TaxonNode getHigherNode(TaxonNode node, Rank rank){
386 422

  
......
475 511
    private HashMap<String, String> createNewRecord(TaxonNode childNode, CsvNameExportState state){
476 512
        HashMap<String, String> nameRecord = new HashMap<>();
477 513
        nameRecord.put("classification", childNode.getClassification().getTitleCache());
514
        if (!childNode.getTaxon().getName().getRank().isLower(Rank.GENUS())){
515
            return null;
516
        }
478 517
        TaxonNode familyNode = getHigherNode(childNode, Rank.FAMILY());
479 518
        Taxon taxon;
480 519
        String nameString;
......
487 526
            familyNode = CdmBase.deproxy(familyNode);
488 527
            familyNode.getTaxon().setProtectedTitleCache(true);
489 528
            nameRecord.put("familyTaxon", familyNode.getTaxon().getTitleCache());
529
            if (familyMap.get(familyNode.getTaxon().getUuid()) != null){
530
                nameRecord.putAll(familyMap.get(familyNode.getTaxon().getUuid()));
531
            }else{
532
                taxon = (Taxon)getTaxonService().load(familyNode.getTaxon().getUuid());
533
                taxon = CdmBase.deproxy(taxon);
534
                name = CdmBase.deproxy(taxon.getName());
535
                nameRecord.put("familyName", name.getNameCache());
536
                extractDescriptions(nameRecord, taxon, Feature.INTRODUCTION(), "descriptionsFam", state);
537
                familyMap.put(familyNode.getTaxon().getUuid(), nameRecord);
538
            }
490 539

  
491
            taxon = (Taxon)getTaxonService().load(familyNode.getTaxon().getUuid());
492
            taxon = CdmBase.deproxy(taxon);
493
            //if publish flag is set
494

  
495
            //  if (taxon.isPublish()){
496
            name = CdmBase.deproxy(taxon.getName());
497
            nameRecord.put("familyName", name.getNameCache());
498
            extractDescriptions(nameRecord, taxon, Feature.INTRODUCTION(), "descriptionsFam", state);
499 540
        }
500 541
        TaxonNode genusNode = getHigherNode(childNode, Rank.GENUS());
501 542
        if (genusNode!= null){
502 543
            genusNode = CdmBase.deproxy(genusNode);
503 544
            genusNode.getTaxon().setProtectedTitleCache(true);
504 545
            nameRecord.put("genusTaxon", genusNode.getTaxon().getTitleCache());
505

  
506
            taxon = (Taxon)getTaxonService().load(genusNode.getTaxon().getUuid());
507
            taxon = CdmBase.deproxy(taxon);
508
            //if publish flag is set
509

  
510
            //  if (taxon.isPublish()){
511
            name = CdmBase.deproxy(taxon.getName());
512
            if (name.getNameCache() != null){
513
                nameRecord.put("genusName", name.getNameCache());
546
            if (genusMap.get(genusNode.getTaxon().getUuid()) != null){
547
                nameRecord.putAll(genusMap.get(genusNode.getTaxon().getUuid()));
514 548
            }else{
515
                nameRecord.put("genusName", name.getGenusOrUninomial());
549
                taxon = (Taxon)getTaxonService().load(genusNode.getTaxon().getUuid());
550
                taxon = CdmBase.deproxy(taxon);
551
                name = CdmBase.deproxy(taxon.getName());
552
                if (name.getNameCache() != null){
553
                    nameRecord.put("genusName", name.getNameCache());
554
                }else{
555
                    nameRecord.put("genusName", name.getGenusOrUninomial());
556
                }
557
                extractDescriptions(nameRecord, taxon,getNotesFeature(state), "descriptionsGen", state);
558
                genusMap.put(genusNode.getTaxon().getUuid(), nameRecord);
516 559
            }
517
            extractDescriptions(nameRecord, taxon,getNotesFeature(state), "descriptionsGen", state);
560

  
561

  
518 562
        }else{
519 563
            nameRecord.put("genusTaxon", null);
520 564
            nameRecord.put("genusName", null);
521 565
            nameRecord.put("descriptionsGen", null);
522 566
        }
523 567

  
524
        taxon = (Taxon) getTaxonService().load(childNode.getTaxon().getUuid());
525
        taxon = CdmBase.deproxy(taxon);
568
//        taxon = (Taxon) getTaxonService().load(childNode.getTaxon().getUuid());
569
        taxon = CdmBase.deproxy(childNode.getTaxon());
526 570
        //  if (taxon.isPublish()){
527 571

  
528 572
        INonViralName nonViralName = taxon.getName();
......
537 581

  
538 582
        getTaxonRelations(nameRecord, taxon);
539 583

  
540
        name = CdmBase.deproxy(getNameService().load(taxon.getName().getUuid()));
584
        name = CdmBase.deproxy(taxon.getName());
541 585
        nameRecord.put("childName",nameString);
542 586
        nameRecord.put("nameId", String.valueOf(name.getId()));
543 587
        nameRecord.put("nameCache", name.getNameCache());
......
721 765
                    if (relatedTaxon.getAppendedPhrase() != null){
722 766
                        appendedPhrase = relatedTaxon.getAppendedPhrase();
723 767
                    }
768
                    if (appendedPhrase.equals("sphalm.")){
769
                        String relNameString  = createTaggedNameString(relatedTaxon.getName(), getStatus(relatedTaxon.getName()));
770

  
771
                        nameRecord.put("relatedName", relNameString);
772
                        nameRecord.put("nameRelType", NameRelationshipType.ORTHOGRAPHIC_VARIANT().getTitleCache());
773
                    }
724 774
                    if (secRef == null){
725
                        nameString.append("<misapplied>\"" + createTaggedNameString(relatedTaxon.getName(), false) + "\" " + appendedPhrase);
775
                        nameString.append("<misapplied>" +"\u201C" + createTaggedNameString(relatedTaxon.getName(), false) + "\u201D " + appendedPhrase);
726 776
                    } else if (secRef.getAuthorship() == null){
727
                        nameString.append("<misapplied>\"" + createTaggedNameString(relatedTaxon.getName(), false) + "\" " + appendedPhrase + " sensu " + secRef.getTitleCache());
777
                        nameString.append("<misapplied>"+"\u201C" + createTaggedNameString(relatedTaxon.getName(), false) + "\u201D " + appendedPhrase + " sensu " + secRef.getTitleCache());
728 778
                    } else {
729
                        nameString.append("<misapplied>\"" + createTaggedNameString(relatedTaxon.getName(), false) + "\" " + appendedPhrase + " sensu " + secRef.getAuthorship().getNomenclaturalTitle());
779
                        nameString.append("<misapplied>\"" + createTaggedNameString(relatedTaxon.getName(), false) + "\" " + appendedPhrase + " sensu " + secRef.getAuthorship().getFullTitle());
730 780
                    }
731 781

  
732 782
                }
cdmlib-persistence/src/main/java/eu/etaxonomy/cdm/persistence/dao/hibernate/taxon/TaxonNodeFilterDaoHibernateImpl.java
292 292
            String uuid = singleFilter.getUuid().toString();
293 293
            String op = isFirst ? "" : op2Hql(singleFilter.getOperator());
294 294
            result = String.format("(%s%s(tn.classification.uuid = '%s'))", result, op, uuid);
295
//            System.out.println(result);
295
            System.out.println(result);
296 296
            isFirst = false;
297 297
        }
298 298
        return result;

Also available in: Unified diff

Add picture from clipboard (Maximum size: 40 MB)