applicationIndex.py 16.3 KB
Newer Older
1
2
#!/usr/bin/env python

3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
# -*- coding: utf-8 -*-

# Copyright 2015 Helmholtz-Zentrum Berlin für Materialien und Energie GmbH
# <https://www.helmholtz-berlin.de>
#
# Author: Bernhard Kuner <bernhard.kuner@helmholtz-berlin.de>
#
# This program is free software: you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free Software
# Foundation, either version 3 of the License, or (at your option) any later
# version.
# 
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
# details.
# 
# You should have received a copy of the GNU General Public License along with
# this program.  If not, see <http://www.gnu.org/licenses/>.

23
from optparse import OptionParser
24
import time
25
import sys
26
import re
27
28
29
30
import os
import os.path
import subprocess
import epicsUtils as eU
31
import listOfDict as lod
32
33
import pprint as pp
import canLink as cL
34
import tokParse as tP
35
36
37
38
39
40
41
42
# Searching for Hardware means to look at the DTYP field of a record. So here we define
# the list of known DTYPs get it with:
# find $(TOP)/db -name *.db|xargs perl -e 'while(<>){print "$1\n" if($_=~/DTYP,\s*\"(.*)\"/);}'|sort -u
hardwareDtypList = ['esd AIO16','ADA16','BESSY MuxV','Dyncon','EK IO32','ESRF MuxV',
    	    	    'Highland DDG V85x','OMS MAXv','OMS VME58','Rfmux1366','TDU','V375',
		    'V680','VHQ','Vpdu'
		   ]

43
44
45
46
47
48
49
50
51
def systemCall(cmdArgsList):
    """ Do a system call and return the output
    """
    try:
	return subprocess.Popen(cmdArgsList,stdout=subprocess.PIPE,stderr=subprocess.PIPE).communicate()[0]
    except OSError, e:
	print >>sys.stderr, "Execution failed:",cmdArgsList, e
	return None
    
52
53
54
55
56
57
def getIoc(dbIoc,db):
    """ little html helper
    """
    if dbIoc.has_key(db):   
    	return reduce(lambda y,x:y+str("<A HREF=\"#"+x+"\">"+x+"</A><br>"),dbIoc[db],"")
    else: 
58
    	return None
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76

def getDb(dbApp,db):
    """ little html helper
    """
    if dbApp.has_key(db): 
    	return "<DIV TITLE=\""+dbApp[db]+"\"><A HREF=\"#"+dbApp[db]+"\">"+db+"</A></DIV>"
    else: 
    	return "<DIV TITLE=\"got from support module\">"+db+"</DIV>"

def toCol(dList,tag='TD'): 
    """ create a string of tagged list items, default tag is TD, also possible: TH, LI"""
    def toStr(x):
    	if x is None:
	    return "-"
	else:
	    return str(x)
    return "<"+tag+">"+str("</"+tag+">\n  <"+tag+">").join([toStr(x) for x in dList])+"</"+tag+">"

77
78
79
80
81
def substEnvVariables(param,envDict):
    for name in envDict.keys():
	param = param.replace(r"${"+name+"}",envDict[name])
    return param

82
83
84
85
86
87
88
89
def getIocStartupData(topPath):
    path = topPath+"/GenericBootApp/O.Common"
    iocPy = []
    for item in systemCall(['ls',path]).split("\n"):
	py = eU.matchRe(item,'(IOC.*)\.py')
	if py: iocPy += py
    if len(iocPy) == 0:
    	return (None,None)
90
    if options.verbose is True: print "*** getIocStartupData() for:\n",iocPy
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
    iocDb = {}
    dbIoc = {}
    sys.path.insert(0,path)
    for ioc in iocPy:
    	fileName = path+"/"+ioc+".py"
	myDict = {}
	try:
	    execfile(fileName,myDict)
	except SyntaxError, e:
	    raise SyntaxError("Syntax Error in File: "+fileName+"\n"+str(e))
	iocDb[ioc] = myDict['iocConfig']['loadRecords']
	for db in iocDb[ioc]:
	    db['DB'] = db['DB']+".db"
	    dbFile = db['DB']
	    if not dbIoc.has_key(dbFile):
	    	dbIoc[dbFile] = []
	    dbIoc[dbFile].append(ioc)
108
    return(iocDb,dbIoc)
109
110

def processStCmd(topPath,iocList):
111
112
    iocDb = {}
    dbIoc = {}
113

114
115
116
117
    tokDefList =( ('B_OPEN', (r'\(',) ),\
    	('B_CLOSE',     (r'\)',) ),\
    	('COMMA',    	(r',',) ),\
    	('QSTRING',(r'(?<!\\)"(.*?)(?<!\\)"',) ), \
118
    	('WORD',    	(r'[a-zA-Z0-9_&/\.:=+\-\{\}\$\']+',) ),\
119
120
121
122
123
124
    	('COMMENT',   	(r'#.*',) ),\
    	('LOAD',   	(r'[<|>]',) ),\
    	('SPACE',(r"\s+",))
    	)
    tokReList = tP.compileTokDefList(tokDefList)

125
    if options.verbose is True: print "*** processStCmd() for:\n",iocList
126
127
    for iocName in iocList:
	parseFileName = topPath+"/iocBoot/ioc"+iocName+"/st.cmd"
128
129
130
131
132
133
134
135
136
	if not os.path.isfile(parseFileName): eU.die("File doesn't exist: "+parseFileName)
	try :
	    IN_FILE = open(parseFileName) 
	except IOError: 
	    eU.die("can't open input file: "+parseFileName)
	if options.verbose is True: print "Reading data from "+parseFileName

	dbdFile=topPath+"/dbd/"
 	envDict={}
137
    	lineNr = 0
138
	for line in IN_FILE:
139
140
141
	    lineNr += 1
	    parsedLine = tP.parseStCmd(line,tokReList,lineNr)
	    if not parsedLine: continue
142
143
	    cmd = parsedLine[0]
	    if cmd == "epicsEnvSet":
144
       		#print "epicsEnvSet,PARSE: '%s'" %line, parsedLine
145
146
    		envDict[parsedLine[1]]=parsedLine[2]
	    if cmd == "putenv":
147
       		#print "putenv,PARSE: '%s'" %line, parsedLine
148
149
150
		(name,value)=parsedLine[1].split("=")
		envDict[name]=value
	    if cmd == "dbLoadDatabase":
151
       		#print "dbLoadDatabase,PARSE: '%s'" %line, parsedLine
152
    	    	dbdFile += substEnvVariables(eU.substRe(parsedLine[1],"dbd/",""),envDict)
153
	    if cmd == "dbLoadRecords":
154
       		#print "dbLoadRecords,PARSE: '%s'" %line, parsedLine
155
156
157
158
    		dbFile = substEnvVariables(eU.substRe(parsedLine[1],"db/",""),envDict)
		param = ""
		if len(parsedLine) == 3:
		    param = parsedLine[2]
159
		    #print "param:",param,eU.parseParam(substEnvVariables(param,envDict),',')
160
161
		if not iocDb.has_key(iocName):
	    	    iocDb[iocName] = []
162
		iocDb[iocName].append( {'DB':dbFile,'SUBST':eU.parseParam(substEnvVariables(param,envDict),',')})
163
164
165
		if not dbIoc.has_key(dbFile):
	    	    dbIoc[dbFile] = []
		dbIoc[dbFile].append(iocName)
166
167
168
169
    return (iocDb,dbIoc)

def findApplications(topPath):
    appString = systemCall(['find',topPath,"-name","*.db"])
170
171
    appList = appString.split("\n")
    if options.verbose is True: print "*** FindApplications in top: '"+topPath+"' : \n", appList
172
173
    appDb = {}
    dbApp = {}
174
    for db in appList:
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
	db = eU.substRe(db,"O\..*\/","")
	d = eU.matchRe(db,".*/(.*?App.*)")
	if d is not None:   	# look for something like: myApp/[Db/]myFile.db
	    item = d[0].split("/")
	    if len(item) < 2:
	    	print "Error in findApplications:",d[0]
		sys.exit()
	    appName = item[0]
	    dbFileName = item[-1]
	    if not appDb.has_key(appName):
	    	appDb[appName] = []
	    appDb[appName].append(dbFileName)
	    dbApp[dbFileName] = appName
    return (appDb,dbApp)

190
def hardware(ioc,dbFile,param,iocname,pvname,fieldDict) :
191
192
193
#    print "hardware(",ioc,dbFile
#    print "param",param
#    print iocname,pvname
194
    try:
195
	pvname = eU.substituteVariables(pvname,param)
196
    except AttributeError:
197
    	print ioc,dbFile,iocname,pvname,param
198
	print "Can't substitute variable '"+param+"' from:"
199
200
	pp.pprint(fieldDict)
	sys.exit()
201
    fieldDict.update( {"iocname":iocname,"filename":dbFile,"pvname":pvname} )
202

203
    if fieldDict.has_key('INP'):
204
        link = fieldDict['INP']
205
    elif fieldDict.has_key('OUT'):
206
        link = fieldDict['OUT']
207
    else:
208
209
210
        link = None
    if link and len(param) >0:
	link = eU.substituteVariables(link,param)
211
    if fieldDict['DTYP'] == 'lowcal':
212
213
214
215
216
217
218
219
220
        try:
	    if fieldDict['RTYP'] == 'hwLowcal':
        	link = cL.hwLowcal2canLink(fieldDict)
	    if link:
		fieldDict.update(cL.decode(link))
        except ValueError, e:
	    print "Warning:",dbFile+", '"+fieldDict['pvname']+"': ",e
	    
	if fieldDict.has_key('cid') and fieldDict['cid'] == 2:	# ADA/IO32 card
221
222
223
224
225
226
227
228
            mux = int(fieldDict['multiplexor'])
            fieldDict['CARD'] =  mux/12
            fieldDict['CHAN'] =  mux%12
        if fieldDict.has_key('cid') and fieldDict['cid'] == 4:	# vctrl card
            mux = int(fieldDict['multiplexor'])
            fieldDict['CARD'] =  mux/2
            fieldDict['CHAN'] =  mux%2
    else:
229
230
	vmeLnk = eU.matchRe(link,"#C\s*(\d+)\s*S\s*(\d+)")
	if vmeLnk is not None:
231
232
            fieldDict['CARD'] =  vmeLnk[0]
            fieldDict['CHAN'] =  vmeLnk[1]
233
234
	else:
	    if link.find ("$")>=0:
235
236
	    	print "Warning: Unsubstituted values found in IOC:",iocname,"FILE:",dbFile,"PV:",pvname,"LINK:",link
		sys.exit()
237
    fieldDict['LINK'] = link    	    
238
    return fieldDict
239
240
241

def checkHardwareAccess(iocDb,topPath):
    hwData = []
242
    if options.verbose is True: print "\n*** CheckHardwareAccess"
243
    for ioc in iocDb.keys():
244
245
246
247
248
249
	for dbItem in iocDb[ioc]:
	    dbFile = dbItem['DB']
	    param = {}
	    if dbItem.has_key('SUBST'):
	    	param  = dbItem['SUBST']
	    hw = systemCall(['grepDb.pl','-pH','-th',topPath+"/db/"+dbFile]) # return a perl hash of {PVNAME=> {FIELD=>VALUE}}
250
251
            if not hw:
                continue 
252
	    hw = eU.substRe(hw," => ",":")  	# make it python eval uable
253
254
255
256
257
258
259
260
            hw = eU.substRe(hw,"\$VAR1\s*=","")
            hw = eU.substRe(hw,";","")
            try:
                hwDict = eval(hw)
            except:
                pp.pprint(hw)
                print "ERROR in checkHardwareAccess(",iocDb,topPath,")"
                sys.exit()
261

262
263
264
	    for pv in hwDict.keys():
		hw = hardware(ioc,dbFile,param,ioc,pv,hwDict[pv])
		if hw: 
265
                    hwData.append(hw)
266
267
268
    return hwData
	    	
######## Main #######################################################
269
usage        = "USAGE: applicationIndex.py [options] topPath outFile"
270
271
272
273
274
275
276
277
278
parser = OptionParser(usage=usage,
		 version="%%prog 1.0",
    		 description="USAGE: applictionIndex.py [-h OPTIONS] topPath")
parser.add_option("-v","--verbose",
		 action="store_true", # default: None
		 help="print debug information", 
    		 )
try:
    (options,args) = parser.parse_args()
279
    (topPath,filename) = args[0:2]
280
except:
281
    print "OPTIONS:",options,"\nARGS:",args[:2]
282
283
284
285
    print usage
    sys.exit()

(appDb,dbApp) = findApplications(topPath)
286
287
288
#pp.pprint(appDb)
#pp.pprint(dbApp)
(iocDb,dbIoc) = getIocStartupData(topPath)
289
290
if iocDb == None: iocDb = {}
if dbIoc == None: dbIoc = {}
291
292
293
294
295
296
297
298
299
300
301
302

iocString = systemCall(['ls',topPath+"/iocBoot"])
iocList = []
for ioc in iocString.split("\n"):
    i = eU.matchRe(ioc,"ioc(.*)")
    if i is None:
	continue
    iocName = i[0]
    if not iocDb.has_key(iocName):
    	iocList.append(iocName)

(iD,dI) = processStCmd(topPath,iocList)
303

304
305
if iD: iocDb.update(iD)
if dI: dbIoc.update(dI)
306
#pp.pprint(iocDb)
307
#pp.pprint(dbIoc)
308
#sys.exit()
309
310
311
312
iocHw = None
if iocDb:
    iocHw = checkHardwareAccess(iocDb,topPath)

313
if options.verbose is True: print "*** Process Data"
314

315
316
317
318
######## PRINT DATA #######################################################
htmlHeader = """<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="de" lang="de">
<head>
319
    <TITLE>Application and Hardware Reference</TITLE>
320
    <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
321
    <link rel=stylesheet type="text/css" href="http://help.bessy.de/~kuner/makeDocs/docStyle.css">
322
323
324
325
326
327
328
329
</head>
<body>
<H1>Application and IOC Reference</H1>
"""
htmlFooter = """
</body>
</html>
"""
330
331
332
if not appDb and not iocDb and not iocHw: # nothing to do
    sys.exit()

333
334
335
336
337
338
339
340
try :
    FILE = open(filename,"w") 
except IOError: 
    eU.die("can't open input file: "+filename)

print >> FILE, htmlHeader

# Application Reference:
341
if options.verbose is True: print "*** Write File:", filename
342
print >> FILE, "<P>last update: "+time.strftime("%d.%m.%Y")+"</P>\n"
343
344
print >> FILE, '</P>'
print >> FILE, '<H2>Content</H2>\n<P>&bull <A HREF="#APP_REF">Application Reference</A><br>&bull <A HREF="#IOC_APP">IOC Application Reference</A></P>'
345
346
347
348
349
350
351
352
print >> FILE, '<P>IOC Hardware Reference</A></P>\n'
(forgetThisList,getHw) = lod.filterMatch(iocHw,{'DTYP':['HwClient','Dist Version','IOC stats','Raw Soft Channel','Soft Channel','Soft Timestamp','Soft Timestamp WA','VX stats','VxWorks Boot Parameter']})
for ioc in sorted(iocDb.keys()):
    (iocHwList,getHw) = lod.filterMatch(getHw,{'iocname':ioc})
    if len(iocHwList) == 0: 
    	continue
    print >> FILE, '<LI><A HREF="#HW_'+ioc+'">'+ioc+'</A>'

353
print >> FILE, '</UL>\n<H2><A NAME="APP_REF"></A>Application Reference</H2>\n\n<TABLE BORDER=1>'
354

355
if appDb:
356
    dbNotLoaded = []
357
358
359
360
361
    for app in appDb.keys():
	dbList = appDb[app]
	span = ""
	if len(dbList) > 1:
    	    span = "ROWSPAN=\""+str(len(dbList))+"\" "
362
363
364
365
366
	iocName = getIoc(dbIoc,dbList[0])
	if iocName == None:
	    iocName = "not loaded on IOC"
	    dbNotLoaded.append(dbList[0])
	print >> FILE, "<TR >\n  <TD "+span+"VALIGN=\"TOP\"><A NAME=\""+app+"\">"+app+"</TD>\n  <TD VALIGN=\"TOP\">"+dbList[0]+"</TD>\n  <TD>"+iocName+"</TD></TR>"
367
	for db in dbList[1:]:
368
369
370
371
372
	    iocName = getIoc(dbIoc,db)
	    if iocName == None:
		iocName = "not loaded on IOC"
	    	dbNotLoaded.append(db)
	    print >> FILE,"<TR>\n  <TD VALIGN=\"TOP\">"+db+"</TD>\n  <TD VALIGN=\"TOP\">"+iocName+"</TD></TR>"
373
    print >> FILE, "</TABLE>\n"
374
375
    if options.verbose is True and len(dbNotLoaded) > 0: 
    	print "*** Warning: .db files not loaded on IOC:\n",dbNotLoaded
376

377
if iocDb:
378
    print >> FILE, '<A NAME="IOC_APP"></A>\n<H2>IOC Application Reference</H2>\n\n<TABLE BORDER=1>'
379
    for ioc in sorted(iocDb.keys()):
380
381
382
383
384
	dbList = iocDb[ioc]
	span = ""
	if len(dbList) > 1:
    	    span = "ROWSPAN=\""+str(len(dbList))+"\" "
	setIoc = '<TH '+span+'VALIGN="TOP"><A NAME="'+ioc+'"><A HREF="#HW_'+ioc+'">'+ioc+'</A></TH>\n  '
385
	for dbObj in dbList:
386
	    print >> FILE,'<TR>\n  '+setIoc+'<TD>'+getDb(dbApp,dbObj['DB'])+'</TD></TR>'
387
388
389
	    if len(setIoc) > 0: 	# first item only
		setIoc = ''
    print >> FILE, "</TABLE>\n"
390

391
if iocHw:
392
    print >> FILE, "<H2>IOC Hardware Reference</H2>\n\n"
393
394
395
396
397
    (forgetThisList,getHw) = lod.filterMatch(iocHw,{'DTYP':['HwClient','Dist Version','IOC stats','Raw Soft Channel','Soft Channel','Soft Timestamp','Soft Timestamp WA','VX stats','VxWorks Boot Parameter']})
    for ioc in iocDb.keys():
	(iocHwList,getHw) = lod.filterMatch(getHw,{'iocname':ioc})
	if len(iocHwList) == 0: 
    	    continue
398

399
	print >> FILE, '<H3><A NAME="HW_'+ioc+'"></A>Hardware Channels on '+ioc+'</H3>\n\n'
400

401
	print >> FILE, "<H4>CAN Devices on"+ioc+"</H4>\n\n"
402
403
404
405
	order = ('port','nid','cid','CARD','CHAN','LINK','pvname','filename')
	(canList,otherList) = lod.filterMatch(iocHwList,{'DTYP':['lowcal',],})
	table = lod.orderToTable(canList,order)
	if len(table) > 0:
406
            print >> FILE, '<TABLE BORDER=1>\n<TR>'+toCol(['Process Variable','Port','CAN-Id','Card','Chan','Link','Port, In-, OutCOB, Mux'],'TH')+'\n</TR>'
407
408
409
410
411
            try:
		for (port,nid,cid,CARD,CHAN,LINK,pvname,filename) in table:
		    if LINK is not None:
			c = LINK.split(' ') # c=(@type dataType port in_cob out_cob mux ....)
			t = "In Cob: %d Out Cob: %d Mux: %d" % (int(c[4],16),int(c[5],16),int(c[6],16))
412
			cid ="%d %d %d %d"%(int(c[3],16),int(c[4],16),int(c[5],16),int(c[6],16))
413
414
415
		    else:
			t = "No INP/OUT"
			LINK = '-'
416
			cid ="-"
417
        	    LINK = '<DIV TITLE="'+t+'">'+LINK+'</DIV>'
418

419
420
		    pvname = '<DIV TITLE="IOC: '+ioc+', Application: '+dbApp[filename]+', File: '+filename+'">'+pvname+'</DIV>'
		    print >> FILE, "<TR>"+toCol([pvname,port,nid,CARD,CHAN,LINK,cid])+"\n</TR>"
421
422
            except KeyError:
	    	print "ERROR in print '"+pvname+"', CAN-Devices: '"+filename+"' not found in dbApp"
423
	    print >> FILE, "</TABLE>\n"
424

425
	print >> FILE, "<H4>VME Devices on "+ioc+"</H4>\n\n"
426
	(vmeList,otherList) = lod.filterMatch(otherList,{'DTYP':hardwareDtypList})
427
428
429
430
	order = ('DTYP','CARD','CHAN','pvname','LINK')
	table = lod.orderToTable(vmeList,order)
	if len(table) > 0:
            print >> FILE, "<TABLE BORDER=1>\n<TR>"+toCol(['Process Variable','Card','Chan','DTYP','Link'],'TH')+"\n</TR>"
431
432
433
434
435
	    try:
        	for l in table:
		    (DTYP,CARD,CHAN,pvname,LINK) = l
    	    	    pvname = '<DIV TITLE="IOC: '+ioc+', Application: '+dbApp[filename]+', File: '+filename+'">'+pvname+'</DIV>'
        	    print >> FILE, "<TR>"+toCol([pvname,CARD,CHAN,DTYP,LINK])+"\n</TR>"
436
437
            except KeyError:
	    	print "ERROR in print '"+pvname+"', VME-Devices: '"+filename+"' not found in dbApp"
438
            print >> FILE, "</TABLE>\n"
439

440
	print >> FILE, "<H4>Other Devices on"+ioc+"</H4>\n\n"
441
	order = ('LINK','pvname','filename','DTYP','RTYP')
442
443
444
	table = lod.orderToTable(otherList,order)
	if len(table) > 0:
            print >> FILE, "<TABLE BORDER=1>\n<TR>"+toCol(['Process Variable','Link'],'TH')+"\n</TR>"
445
	    try:
446
447
        	for (link,pvname,filename,dtyp,rtyp) in table:
        	    pvname = '<DIV TITLE="IOC: '+ioc+', Application: '+dbApp[filename]+', File: '+filename+', RTYP'+rtyp+', DTYP'+dtyp+'">'+pvname+'</DIV>'
448
		    print >> FILE, "<TR>"+toCol([pvname,link])+"\n</TR>"
449
450
            except KeyError:
	    	print "ERROR in print '"+pvname+"', Other-Devices: '"+filename+"' not found in dbApp"
451
            print >> FILE, "</TABLE>\n"
452
453
454
455

print >> FILE, htmlFooter
FILE.close()