Commit b4849c0e authored by Martin Schorb's avatar Martin Schorb
Browse files

Merge branch 'xml' into 'master'

Xml

See merge request schorb/pyem!1
parents 7ba3ce3b 0e10e094
......@@ -34,16 +34,7 @@ newnavf = navfile#[:-4] + '_clean.nav'
print('Navigator items marked with Acquire were duplicated and output is written as: ' + newnavf)
nnf = open(newnavf,'w')
nnf.write("%s\n" % navlines[0])
nnf.write("%s\n" % navlines[1])
# fill the new file
for nitem in newnav:
out = em.itemtonav(nitem,nitem['# Item'])
for item in out: nnf.write("%s\n" % item)
nnf.close()
em.write_navfile(newnavf,newnav,xml=False)
\ No newline at end of file
......@@ -27,29 +27,26 @@ acq = filter(lambda item:item.get('Acquire'),allitems)
acq = list(filter(lambda item:item['Acquire']==['1'],acq))
non_acq = [x for x in allitems if x not in acq]
newnav = non_acq
#newnav = non_acq
# create new file by copying the header of the input file
newnavf = navfile[:-4] + '_clean.nav'
nnf = open(newnavf,'w')
nnf.write("%s\n" % navlines[0])
nnf.write("%s\n" % navlines[1])
outnav = list()
#
#nnf = open(newnavf,'w')
#nnf.write("%s\n" % navlines[0])
#nnf.write("%s\n" % navlines[1])
# fill the new file
for item in acq:
labelstr = item['# Item']
searchstr = labelstr[1:4]
include = list(filter(lambda item:item['# Item'].find(searchstr)>0,allitems))
searchstr = labelstr#[1:4]
include = list(filter(lambda item:item['# Item'].find(searchstr)>0,non_acq))
for navitem in include:
out = em.itemtonav(navitem,navitem['# Item'])
for s_item in out: nnf.write("%s\n" % s_item)
outnav.extend(include)
em.write_navfile(newnavf,outnav,xml=False)
nnf.close()
......@@ -30,16 +30,6 @@ newnavf = navfile[:-4] + '_duplicated.nav'
print('Navigator items marked with Acquire were duplicated and output is written as: ' + newnavf)
nnf = open(newnavf,'w')
nnf.write("%s\n" % navlines[0])
nnf.write("%s\n" % navlines[1])
# fill the new file
for nitem in newnav:
out = em.itemtonav(nitem,nitem['# Item'])
for item in out: nnf.write("%s\n" % item)
nnf.close()
em.write_navfile(newnavf,newnav,xml=False)
......@@ -54,9 +54,9 @@ targetheader = target_merge['mergeheader']
t_mat = em.map_matrix(targetitem)
newnavf = navname[:-4] + '_automaps.nav'
nnf = open(newnavf,'w')
nnf.write("%s\n" % navlines[0])
nnf.write("%s\n" % navlines[1])
#nnf = open(newnavf,'w')
#nnf.write("%s\n" % navlines[0])
#nnf.write("%s\n" % navlines[1])
allitems = em.fullnav(navlines)
......@@ -205,9 +205,5 @@ for idx,acq_item in enumerate(acq):
for nitem in outnav:
newnav.append(nitem)
for nitem in newnav:
out = em.itemtonav(nitem,nitem['# Item'])
for item in out: nnf.write("%s\n" % item)
nnf.close()
em.write_navfile(newnavf,newnav,xml=False)
......@@ -63,9 +63,9 @@ targetheader = target_merge['mergeheader']
t_mat = em.map_matrix(targetitem)
newnavf = navname[:-4] + '_automaps.nav'
nnf = open(newnavf,'w')
nnf.write("%s\n" % navlines[0])
nnf.write("%s\n" % navlines[1])
#nnf = open(newnavf,'w')
#nnf.write("%s\n" % navlines[0])
#nnf.write("%s\n" % navlines[1])
allitems = em.fullnav(navlines)
......@@ -213,10 +213,5 @@ for idx,acq_item in enumerate(acq):
for nitem in outnav:
newnav.append(nitem)
for nitem in newnav:
out = em.itemtonav(nitem,nitem['# Item'])
for item in out: nnf.write("%s\n" % item)
nnf.close()
em.write_navfile(newnavf,newnav,xml=False)
......@@ -30,16 +30,5 @@ newnavf = navfile[:-4] + '_sorted.nav'
print('Navigator file was sorted and output is written as: ' + newnavf)
nnf = open(newnavf,'w')
nnf.write("%s\n" % navlines[0])
nnf.write("%s\n" % navlines[1])
# fill the new file
for nitem in newnav:
out = em.itemtonav(nitem,nitem['# Item'])
for item in out: nnf.write("%s\n" % item)
em.write_navfile(newnavf,newnav,xml=False)
nnf.close()
......@@ -199,9 +199,9 @@ navlines = em.loadtext(navname)
newnavf = navname[:-4] + '_automaps.nav'
nnf = open(newnavf,'w')
nnf.write("%s\n" % navlines[0])
nnf.write("%s\n\n" % navlines[1])
#nnf = open(newnavf,'w')
#nnf.write("%s\n" % navlines[0])
#nnf.write("%s\n\n" % navlines[1])
allitems = em.fullnav(navlines)
......@@ -247,12 +247,8 @@ for idx,acq_item in enumerate(acq):
on1=em.ordernav(outnav1, delim='_')
finalnav = maps['mapnav'].copy()
finalnav.extend(on1.copy())
finalnav.extend(on1.copy())
for nitem in finalnav:
out = em.itemtonav(nitem,nitem['# Item'])
for item in out: nnf.write("%s\n" % item)
nnf.close()
em.write_navfile(newnavf,finalnav,xml=False)
......@@ -2,8 +2,8 @@
# pyEM.py
# Copyright (c) 2017, Martin Schorb
# Copyright (c) 2017, European Molecular Biology Laboratory
# Copyright (c) 2020, Martin Schorb
# Copyright (c) 2020, European Molecular Biology Laboratory
# Produced at the EMBL
# All rights reserved.
......@@ -41,6 +41,7 @@ import time
from operator import itemgetter
import fnmatch
from subprocess import Popen, PIPE
import xml.etree.ElementTree as ET
# get python version
py_ver = sys.version_info
......@@ -83,32 +84,44 @@ def loadtext(fname):
def nav_item(inlines,label):
# extracts the content block of a single navItem of givel label
# reads and parses navigator adoc files version >2 !!
# reads and parses navigator adoc files version >2, also in XML format !!
# returns the first item found as a dictionary and the remaining list with that item removed
# this is useful when multiple items have the exact same label and the function is called from within a loop to retrieve them all.
lines=inlines[:]
if lines[-1] != '':
lines = lines+['']
# search for a navigator item of given label
searchstr = '[Item = ' + label + ']'
if not searchstr in lines:
print('ERROR: Navigator Item ' + label + ' not found!')
result=[]
else:
itemstartline = lines.index(searchstr)+1
itemendline = lines[itemstartline:].index('')
item = lines[itemstartline:itemstartline+itemendline]
result = parse_adoc(item)
# create a dictionary entry that contains the label. Comment # is given in case it is exported directly.
result['# Item']=lines[itemstartline-1][lines[itemstartline-1].find(' = ') + 3:-1]
lines[itemstartline-1:itemstartline+itemendline+1]=[]
if(lines[0]=='<?xml version="1.0" encoding="utf-8"?>'):
#load XML
root = ET.fromstringlist(lines)
el = root.findall('*[@name="%s"]' %label)
root.remove(el[0])
newroot = ET.Element('navigator')
newroot.append(el[0])
result = xmltonav(ET.tostringlist(newroot))[0]
lines = ET.tostringlist(root)
else:
if lines[-1] != '':
lines = lines+['']
# search for a navigator item of given label
searchstr = '[Item = ' + label + ']'
if not searchstr in lines:
print('ERROR: Navigator Item ' + label + ' not found!')
result=[]
else:
itemstartline = lines.index(searchstr)+1
itemendline = lines[itemstartline:].index('')
item = lines[itemstartline:itemstartline+itemendline]
result = parse_adoc(item)
# create a dictionary entry that contains the label. Comment # is given in case it is exported directly.
result['# Item']=lines[itemstartline-1][lines[itemstartline-1].find(' = ') + 3:-1]
lines[itemstartline-1:itemstartline+itemendline+1]=[]
return result,lines
......@@ -252,8 +265,77 @@ def itemtonav(item,name):
dlist.append('')
return dlist
# -------------------------------
#%%
def write_navfile(filename,outitems,xml=False):
# creates a new navigator file from a list of navItems (default is mdoc format)
allitems = copy.deepcopy(outitems)
if xml:
root = ET.Element('navigator')
pd = ET.SubElement(root, 'PreData')
adv = ET.SubElement(pd, 'AdocVersion')
adv.text = '2.00'
lsa = ET.SubElement(pd, 'LastSavedAs')
lsa.text = filename
for item in allitems:
ci = ET.SubElement(root, 'Item')
ci.set('name',item['# Item'])
for key,val in item.items():
if not key == '# Item':
cp = ET.SubElement(ci,key)
cp.text = ' '.join(val)
indent_xml(root)
tree = ET.ElementTree(root)
tree.write(filename,encoding="utf-8",xml_declaration=True)
else:
# MDOC format
head0 = 'AdocVersion = 2.00'
head1 = 'LastSavedAs = '+filename
nnf = open(filename,'w')
nnf.write("%s\n" % head0)
nnf.write("%s\n" % head1)
nnf.write("\n")
# fill the new file
for nitem in allitems:
out = itemtonav(nitem,nitem['# Item'])
for item in out: nnf.write("%s\n" % item)
nnf.close()
# -------------------------------
# pretty print xml, from:
# http://effbot.org/zone/element-lib.htm#prettyprint
def indent_xml(elem, level=0):
i = "\n" + level*" "
if len(elem):
if not elem.text or not elem.text.strip():
elem.text = i + " "
if not elem.tail or not elem.tail.strip():
elem.tail = i
for elem in elem:
indent_xml(elem, level+1)
if not elem.tail or not elem.tail.strip():
elem.tail = i
else:
if level and (not elem.tail or not elem.tail.strip()):
elem.tail = i
# ------------------------------
#%%
def newID(allitems,startid):
......@@ -289,19 +371,44 @@ def newreg(navitems):
def fullnav(inlines,header=False):
# parses a full nav file and returns a list of dictionaries
navlines=inlines[:]
c=[]
for index,item in enumerate(navlines):
if item.find('[')>-1:
if header:
return inlines[:index-1]
(b,navlines)=nav_item(navlines,item[item.find(' = ') + 3:-1])
b['# Item']=item[item.find(' = ') + 3:-1]
c.append(b)
navlines=inlines[:]
if(navlines[0]=='<?xml version="1.0" encoding="utf-8"?>'):
#load XML
c = xmltonav(navlines)
else:
# mdoc format
c=[]
for index,item in enumerate(navlines):
if item.find('[')>-1:
if header:
return inlines[:index-1]
(b,navlines)=nav_item(navlines,item[item.find(' = ') + 3:-1])
b['# Item']=item[item.find(' = ') + 3:-1]
c.append(b)
return c
# -------------------------------
#%%
def xmltonav(navlines):
root = ET.fromstringlist(navlines)
c=[];
for child in root:
if child.tag == 'Item':
item = dict({'# Item':child.attrib['name']})
for prop in child:
item[prop.tag] = prop.text.split(' ')
c.append(item)
return c
# -------------------------------
#%%
......
......@@ -69,6 +69,8 @@ generic functions to parse/manipulate navigator/adoc files
- **nav_selection:** extracts a selection of navigator items into a new navigator, _Acquire_ can be chosen as a default flag, - input: list of items = optional list of item labels
- **navlabel_match:** identifies navigator items whose labels contain the given string
- **ordernav:** re-orders a navigator by its label. It considers the indexing after a delimiter in the string. Example: s01_cell-1,s02_cell-1,s01_cell-02, ... is sorted by cells instead of s. When no delimiter is given (''), the navigator is sorted by its label.
- **xmltonav:** parses a navigator file stored in XML-format - input: text list
- **write_navfile:** writes a new Navigator file from a list of item dictionaries - input:filename(str),list of nav dicts
functions to extract information from a navigator item
......
......@@ -6,7 +6,7 @@ with open("readme.md", "r") as fh:
long_description = fh.read()
setup(name='py-EM',
version='20200409',
version='20200421',
py_modules=['pyEM'],
description='Tools to interact with Serial EM enabling automated transmission electron microscopy.',
long_description=long_description,
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment