000031115 - Enable additional meta keys in table-map-custom.xml for enhanced log information in RSA NetWitness Platform

/etc/netwitness/ng/envision/etc/table-map-custom.xml

• Set to transient in the table-map.xml file
• ExtensionKeys in the CEF parser

chmod +x findmissing.sh

 ./findmissing.sh
Additional Meta keys for table-map-custom.xml can be found in /tmp/TOADD.txt
Paste the contents of this file between the <mappings> </mappings> tags
into the file /etc/netwitness/ng/envision/etc/table-map-custom.xml

<!-- BEGIN List of keys Not in table-map-custom.xml -->
<mapping envisionName="cn_acttimeout" nwName="cn_acttimeout" flags="None"/>
<mapping envisionName="cn_asn_dst" nwName="cn_asn_dst" flags="None"/>
<mapping envisionName="cn_asn_src" nwName="cn_asn_src" flags="None"/>
<mapping envisionName="cn_bgpv4nxthop" nwName="cn_bgpv4nxthop" flags="None"/>
<mapping envisionName="cn_ctr_dst_code" nwName="cn_ctr_dst_code" flags="None"/>
<mapping envisionName="cn_dst_tos" nwName="cn_dst_tos" flags="None"/>
<mapping envisionName="cn_dst_vlan" nwName="cn_dst_vlan" flags="None"/>
<mapping envisionName="cn_engine_id" nwName="cn_engine_id" flags="None"/>
<mapping envisionName="cn_engine_type" nwName="cn_engine_type" flags="None"/>
<mapping envisionName="cn_eventver" nwName="cn_eventver" flags="None"/>
<mapping envisionName="cn_f_switch" nwName="cn_f_switch" flags="None"/>
<mapping envisionName="cn_fld" nwName="cn_fld" flags="None"/>
<mapping envisionName="cn_flowsampid" nwName="cn_flowsampid" flags="None"/>
<mapping envisionName="cn_flowsampintv" nwName="cn_flowsampintv" flags="None"/>
<mapping envisionName="cn_flowsampmode" nwName="cn_flowsampmode" flags="None"/>
<mapping envisionName="cn_inacttimeout" nwName="cn_inacttimeout" flags="None"/>
<mapping envisionName="cn_inpermbyts" nwName="cn_inpermbyts" flags="None"/>
<mapping envisionName="cn_inpermpckts" nwName="cn_inpermpckts" flags="None"/>
<mapping envisionName="cn_invalid" nwName="cn_invalid" flags="None"/>
<mapping envisionName="cn_ip_proto_ver" nwName="cn_ip_proto_ver" flags="None"/>
<mapping envisionName="cn_ipv4_ident" nwName="cn_ipv4_ident" flags="None"/>
<mapping envisionName="cn_l_switch" nwName="cn_l_switch" flags="None"/>
<mapping envisionName="cn_log_did" nwName="cn_log_did" flags="None"/>
<mapping envisionName="cn_log_rid" nwName="cn_log_rid" flags="None"/>
<mapping envisionName="cn_max_ttl" nwName="cn_max_ttl" flags="None"/>
<mapping envisionName="cn_maxpcktlen" nwName="cn_maxpcktlen" flags="None"/>
<mapping envisionName="cn_min_ttl" nwName="cn_min_ttl" flags="None"/>
<mapping envisionName="cn_minpcktlen" nwName="cn_minpcktlen" flags="None"/>
<mapping envisionName="cn_mpls_lbl_1" nwName="cn_mpls_lbl_1" flags="None"/>
<mapping envisionName="cn_mpls_lbl_10" nwName="cn_mpls_lbl_10" flags="None"/>
<mapping envisionName="cn_mpls_lbl_2" nwName="cn_mpls_lbl_2" flags="None"/>
<mapping envisionName="cn_mpls_lbl_3" nwName="cn_mpls_lbl_3" flags="None"/>
<mapping envisionName="cn_mpls_lbl_4" nwName="cn_mpls_lbl_4" flags="None"/>
<mapping envisionName="cn_mpls_lbl_5" nwName="cn_mpls_lbl_5" flags="None"/>
<mapping envisionName="cn_mpls_lbl_6" nwName="cn_mpls_lbl_6" flags="None"/>
<mapping envisionName="cn_mpls_lbl_7" nwName="cn_mpls_lbl_7" flags="None"/>
<mapping envisionName="cn_mpls_lbl_8" nwName="cn_mpls_lbl_8" flags="None"/>
<mapping envisionName="cn_mpls_lbl_9" nwName="cn_mpls_lbl_9" flags="None"/>
<mapping envisionName="cn_mplstoplabel" nwName="cn_mplstoplabel" flags="None"/>
<mapping envisionName="cn_mplstoplabip" nwName="cn_mplstoplabip" flags="None"/>
<mapping envisionName="cn_mul_dst_byt" nwName="cn_mul_dst_byt" flags="None"/>
<mapping envisionName="cn_mul_dst_pks" nwName="cn_mul_dst_pks" flags="None"/>
<mapping envisionName="cn_muligmptype" nwName="cn_muligmptype" flags="None"/>
<mapping envisionName="cn_oldfileid" nwName="cn_oldfileid" flags="None"/>
<mapping envisionName="cn_oldfilesize" nwName="cn_oldfilesize" flags="None"/>
<mapping envisionName="cn_rpackets" nwName="cn_rpackets" flags="None"/>
<mapping envisionName="cn_sampalgo" nwName="cn_sampalgo" flags="None"/>
<mapping envisionName="cn_sampint" nwName="cn_sampint" flags="None"/>
<mapping envisionName="cn_seqctr" nwName="cn_seqctr" flags="None"/>
<mapping envisionName="cn_spackets" nwName="cn_spackets" flags="None"/>
<mapping envisionName="cn_src_tos" nwName="cn_src_tos" flags="None"/>
<mapping envisionName="cn_src_vlan" nwName="cn_src_vlan" flags="None"/>
<mapping envisionName="cn_sysuptime" nwName="cn_sysuptime" flags="None"/>
<mapping envisionName="cn_template_id" nwName="cn_template_id" flags="None"/>
<mapping envisionName="cn_totbytsexp" nwName="cn_totbytsexp" flags="None"/>
<mapping envisionName="cn_totflowexp" nwName="cn_totflowexp" flags="None"/>
<mapping envisionName="cn_totpcktsexp" nwName="cn_totpcktsexp" flags="None"/>
<mapping envisionName="cn_unixnanosecs" nwName="cn_unixnanosecs" flags="None"/>
<mapping envisionName="cn_v6flowlabel" nwName="cn_v6flowlabel" flags="None"/>
<mapping envisionName="cn_v6optheaders" nwName="cn_v6optheaders" flags="None"/>
<mapping envisionName="cs_accesskeyid" nwName="cs_accesskeyid" flags="None"/>
<mapping envisionName="cs_accountid" nwName="cs_accountid" flags="None"/>
<mapping envisionName="cs_agency_dst" nwName="cs_agency_dst" flags="None"/>
<mapping envisionName="cs_analyzedby" nwName="cs_analyzedby" flags="None"/>
<mapping envisionName="cs_av_other" nwName="cs_av_other" flags="None"/>
<mapping envisionName="cs_av_primary" nwName="cs_av_primary" flags="None"/>
<mapping envisionName="cs_av_secondary" nwName="cs_av_secondary" flags="None"/>
<mapping envisionName="cs_bgpv6nxthop" nwName="cs_bgpv6nxthop" flags="None"/>
<mapping envisionName="cs_customdate" nwName="cs_customdate" flags="None"/>
<mapping envisionName="cs_datecret" nwName="cs_datecret" flags="None"/>
<mapping envisionName="cs_devfacility" nwName="cs_devfacility" flags="None"/>
<mapping envisionName="cs_devservice" nwName="cs_devservice" flags="None"/>
<mapping envisionName="cs_dst_tld" nwName="cs_dst_tld" flags="None"/>
<mapping envisionName="cs_eth_dst_ven" nwName="cs_eth_dst_ven" flags="None"/>
<mapping envisionName="cs_eth_src_ven" nwName="cs_eth_src_ven" flags="None"/>
<mapping envisionName="cs_event_uuid" nwName="cs_event_uuid" flags="None"/>
<mapping envisionName="cs_filectime" nwName="cs_filectime" flags="None"/>
<mapping envisionName="cs_fileid" nwName="cs_fileid" flags="None"/>
<mapping envisionName="cs_filemtime" nwName="cs_filemtime" flags="None"/>
<mapping envisionName="cs_fileperm" nwName="cs_fileperm" flags="None"/>
<mapping envisionName="cs_fld" nwName="cs_fld" flags="None"/>
<mapping envisionName="cs_frametype" nwName="cs_frametype" flags="None"/>
<mapping envisionName="cs_identityarn" nwName="cs_identityarn" flags="None"/>
<mapping envisionName="cs_if_desc" nwName="cs_if_desc" flags="None"/>
<mapping envisionName="cs_if_name" nwName="cs_if_name" flags="None"/>
<mapping envisionName="cs_ip_next_hop" nwName="cs_ip_next_hop" flags="None"/>
<mapping envisionName="cs_ipv4dstpre" nwName="cs_ipv4dstpre" flags="None"/>
<mapping envisionName="cs_ipv4srcpre" nwName="cs_ipv4srcpre" flags="None"/>
<mapping envisionName="cs_lifetime" nwName="cs_lifetime" flags="None"/>
<mapping envisionName="cs_log_medium" nwName="cs_log_medium" flags="None"/>
<mapping envisionName="cs_loginname" nwName="cs_loginname" flags="None"/>
<mapping envisionName="cs_oldfilectime" nwName="cs_oldfilectime" flags="None"/>
<mapping envisionName="cs_oldfilehash" nwName="cs_oldfilehash" flags="None"/>
<mapping envisionName="cs_oldfilemtime" nwName="cs_oldfilemtime" flags="None"/>
<mapping envisionName="cs_oldfilename" nwName="cs_oldfilename" flags="None"/>
<mapping envisionName="cs_oldfilepath" nwName="cs_oldfilepath" flags="None"/>
<mapping envisionName="cs_oldfileperm" nwName="cs_oldfileperm" flags="None"/>
<mapping envisionName="cs_oldfiletype" nwName="cs_oldfiletype" flags="None"/>
<mapping envisionName="cs_operation" nwName="cs_operation" flags="None"/>
<mapping envisionName="cs_packettype" nwName="cs_packettype" flags="None"/>
<mapping envisionName="cs_paramkey" nwName="cs_paramkey" flags="None"/>
<mapping envisionName="cs_paramvalue" nwName="cs_paramvalue" flags="None"/>
<mapping envisionName="cs_payload" nwName="cs_payload" flags="None"/>
<mapping envisionName="cs_registrant" nwName="cs_registrant" flags="None"/>
<mapping envisionName="cs_registrar" nwName="cs_registrar" flags="None"/>
<mapping envisionName="cs_req_inst_id" nwName="cs_req_inst_id" flags="None"/>
<mapping envisionName="cs_reqcookies" nwName="cs_reqcookies" flags="None"/>
<mapping envisionName="cs_reqid" nwName="cs_reqid" flags="None"/>
<mapping envisionName="cs_resp_acctid" nwName="cs_resp_acctid" flags="None"/>
<mapping envisionName="cs_rpayload" nwName="cs_rpayload" flags="None"/>
<mapping envisionName="cs_sampler_name" nwName="cs_sampler_name" flags="None"/>
<mapping envisionName="cs_streams" nwName="cs_streams" flags="None"/>
<mapping envisionName="cs_tenant" nwName="cs_tenant" flags="None"/>
<mapping envisionName="cs_tenantid" nwName="cs_tenantid" flags="None"/>
<mapping envisionName="cs_transaction" nwName="cs_transaction" flags="None"/>
<mapping envisionName="cs_user" nwName="cs_user" flags="None"/>
<mapping envisionName="cs_v6nxthop" nwName="cs_v6nxthop" flags="None"/>
<mapping envisionName="cs_whois_server" nwName="cs_whois_server" flags="None"/>
<mapping envisionName="dinterface" nwName="dinterface" flags="None" envisionDisplayName="DestinationInterface"/>
<mapping envisionName="dmacaddr" nwName="eth.dst" flags="None" format="MAC" envisionDisplayName="DestMacAddress|DestinationMacAddress"/>
<mapping envisionName="dmask" nwName="dmask" flags="None"/>
<mapping envisionName="dn" nwName="dn" flags="None"/> <mapping envisionName="dst_dn" nwName="dn.dst" flags="None"/> <mapping envisionName="fqdn" nwName="fqdn" flags="None" envisionDisplayName="FQDN"/> <mapping envisionName="src_dn" nwName="dn.src" flags="None"/>
<mapping envisionName="dtransport" nwName="dtransport" flags="None"/>
<mapping envisionName="event_counter" nwName="event.counter" flags="None" format="Int32"/>
<mapping envisionName="filetype" nwName="filetype" flags="None" />
<mapping envisionName="gateway" nwName="gateway" flags="None"/>
<mapping envisionName="hardware_id" nwName="hardware.id" flags="None"/>
<mapping envisionName="icmptype" nwName="icmp.type" flags="None" format="UInt32"/>
<mapping envisionName="location_city" nwName="loc.city" flags="None"/>
<mapping envisionName="dmacaddr" nwName="eth.dst" flags="None" format="MAC" envisionDisplayName="DestMacAddress|DestinationMacAddress"/> <mapping envisionName="macaddr" nwName="eth.host" flags="None" format="MAC" envisionDisplayName="DeviceMacAddress"/> <mapping envisionName="smacaddr" nwName="eth.src" flags="None" format="MAC" envisionDisplayName="SourceMacAddress" nullTokens="Unknown"/>
<mapping envisionName="packets" nwName="packets" flags="None" format="UInt32"/>
<mapping envisionName="param_endtime" nwName="param_endtime" flags="None"/>
<mapping envisionName="param_event_time" nwName="param_event_time" flags="None"/>
<mapping envisionName="param_starttime" nwName="param_starttime" flags="None"/>
<mapping envisionName="privilege" nwName="privilege" flags="None" envisionDisplayName="Privilege|Privileges"/>
<mapping envisionName="process_id_src" nwName="process.id.src" flags="None" format="Int32" envisionDisplayName="SourceProcessId" nullTokens="(null)|-"/>
<mapping envisionName="process_src" nwName="process.src" flags="None" envisionDisplayName="SourceProcess"/>
<mapping envisionName="c_domain" nwName="sdomain" flags="None" envisionDisplayName="C_Domain|ClientDomain"/> <mapping envisionName="sdomain" nwName="sdomain" flags="None"/>
<mapping envisionName="sessionid" nwName="log.session.id" flags="None"/> <mapping envisionName="sessionid1" nwName="log.session.id1" flags="None"/>
<mapping envisionName="sinterface" nwName="sinterface" flags="None" envisionDisplayName="SourceInterface"/>
<mapping envisionName="smacaddr" nwName="eth.src" flags="None" format="MAC" envisionDisplayName="SourceMacAddress" nullTokens="Unknown"/>
<mapping envisionName="smask" nwName="smask" flags="None"/>
<mapping envisionName="timezone" nwName="timezone" flags="None"/>
<mapping envisionName="rule_uid" nwName="rule.uid" flags="None"/> <mapping envisionName="uid" nwName="username" flags="None" envisionDisplayName="UserID|UID|Uid" nullTokens="none|-"/>
<mapping envisionName="user_org" nwName="org" flags="None" envisionDisplayName="UserOrg|UserOrginization"/>
<!-- END List of keys Not in table-map-custom.xml -->

#!/bin/bash
#Script to add show additional meta keys that could be added to table-map-custom.xml file
#David Waugh
if [ -f /tmp/custom_keys_cef ]
then
 rm -rf /tmp/custom_keys_cef
fi

if [ -d /etc/netwitness/ng/envision/etc/devices/cef/ ]
then
 grep ExtensionKey /etc/netwitness/ng/envision/etc/devices/cef/* |sort | uniq |cut -d " " -f 3 |cut -d \" -
f 2|grep -v \< |sort | uniq > /tmp/custom_keys_cef
else
 echo "CEF Parser is not installed. You can install this parser from RSA Live if you wish."
 fi

grep "<mapping " /etc/netwitness/ng/envision/etc/table-map.xml | grep -v None |cut -d \" -f 4 >> /tmp/custo
m_keys_cef

cat /tmp/custom_keys_cef
cat /tmp/custom_keys_cef |sort |uniq >/tmp/custom_keys_cef_sorted
mv /tmp/custom_keys_cef_sorted /tmp/custom_keys_cef
 rm -rf /tmp/TOADD.txt

if [ ! -f /etc/netwitness/ng/envision/etc/table-map-custom.xml ]
then
  echo "You do not have a table-map-custom.xml already defined"
else
  echo "table-map-custom.xml file is present"
fi
echo "<!-- BEGIN List of keys Not in table-map-custom.xml -->" >>/tmp/TOADD.txt
for metakey in $(cat /tmp/custom_keys_cef)
do
  METAKEY=$metakey
  if [ -f /etc/netwitness/ng/envision/etc/table-map-custom.xml ]
  then
    COUNTCUSTOM=$(grep $metakey /etc/netwitness/ng/envision/etc/table-map-custom.xml |wc -l)
  else
    COUNTCUSTOM=0
  fi

 COUNTTABLEMAP=$(grep $metakey /etc/netwitness/ng/envision/etc/table-map.xml|wc -l)
 COUNTISTRANSIENT=$(grep $metakey /etc/netwitness/ng/envision/etc/table-map.xml| grep -v None |wc -l)
  echo $metakey $COUNTCUSTOM $COUNTTABLEMAP
 # Transient Keys that need to be added that are already in table-map.xml
 if [ $COUNTCUSTOM -eq 0 ] && [ $COUNTTABLEMAP -gt 0 ] && [ $COUNTISTRANSIENT -gt 0 ]
 then
  echo $(grep $metakey  /etc/netwitness/ng/envision/etc/table-map.xml| grep "<mapping ") >>/tmp/TOADD.txt
 fi

 # Custom Keys that do not exist in table-map.xml at all and need to be added
 # Add in the standard Format
 if [ $COUNTCUSTOM -eq 0 ] &&  [ $COUNTTABLEMAP -eq 0 ]
 then
   echo \<mapping envisionName=\"$metakey\" nwName=\"$metakey\" flags=\"None\"\/\> >>/tmp/TOADD.txt
 fi

done
 sed -i -- 's/Transient/None/g' /tmp/TOADD.txt
echo "<!-- END List of keys Not in table-map-custom.xml -->" >>/tmp/TOADD.txt

echo "Additional Meta keys for table-map-custom.xml can be found in /tmp/TOADD.txt"
echo "Paste the contents of this file between the <mappings> </mappings> tags"
echo "into the file /etc/netwitness/ng/envision/etc/table-map-custom.xml"