000031115 - Enable additional meta keys in table-map-custom.xml for enhanced log information in RSA NetWitness Platform

Document created by RSA Customer Support Employee on Jun 14, 2016Last modified by RSA Customer Support on Dec 19, 2019
Version 6Show Document
  • View in full screen mode

Article Content

Article Number000031115
Applies ToRSA Product Set: RSA NetWitness Platform
RSA Product/Service Type: Log Decoder
RSA Version/Condition: 11.x
Platform: CentOS
IssueNot all meta keys are enabled in the table-map.xml file which means some information from log parsers may not be captured. Not all this information may be required, but this script is a way to highlight these additional meta keys so that they can be added into the following file.

/etc/netwitness/ng/envision/etc/table-map-custom.xml

The script attached to this solution displays the meta keys to add to this file. The script highlights meta keys that are:

  • Set to transient in the table-map.xml file
  • ExtensionKeys in the CEF parser
ResolutionCopy the attached script to the Log Decoder appliance and make it executable with the command below.

chmod +x findmissing.sh


On running the script the output below will be displayed.

./findmissing.sh
Additional Meta keys for table-map-custom.xml can be found in /tmp/TOADD.txt
Paste the contents of this file between the <mappings> </mappings> tags
into the file /etc/netwitness/ng/envision/etc/table-map-custom.xml


  An example of the output in /tmp/TOADD.txt is shown below.

<!-- BEGIN List of keys Not in table-map-custom.xml -->
<mapping envisionName="cn_acttimeout" nwName="cn_acttimeout" flags="None"/>
<mapping envisionName="cn_asn_dst" nwName="cn_asn_dst" flags="None"/>
<mapping envisionName="cn_asn_src" nwName="cn_asn_src" flags="None"/>
<mapping envisionName="cn_bgpv4nxthop" nwName="cn_bgpv4nxthop" flags="None"/>
<mapping envisionName="cn_ctr_dst_code" nwName="cn_ctr_dst_code" flags="None"/>
<mapping envisionName="cn_dst_tos" nwName="cn_dst_tos" flags="None"/>
<mapping envisionName="cn_dst_vlan" nwName="cn_dst_vlan" flags="None"/>
<mapping envisionName="cn_engine_id" nwName="cn_engine_id" flags="None"/>
<mapping envisionName="cn_engine_type" nwName="cn_engine_type" flags="None"/>
<mapping envisionName="cn_eventver" nwName="cn_eventver" flags="None"/>
<mapping envisionName="cn_f_switch" nwName="cn_f_switch" flags="None"/>
<mapping envisionName="cn_fld" nwName="cn_fld" flags="None"/>
<mapping envisionName="cn_flowsampid" nwName="cn_flowsampid" flags="None"/>
<mapping envisionName="cn_flowsampintv" nwName="cn_flowsampintv" flags="None"/>
<mapping envisionName="cn_flowsampmode" nwName="cn_flowsampmode" flags="None"/>
<mapping envisionName="cn_inacttimeout" nwName="cn_inacttimeout" flags="None"/>
<mapping envisionName="cn_inpermbyts" nwName="cn_inpermbyts" flags="None"/>
<mapping envisionName="cn_inpermpckts" nwName="cn_inpermpckts" flags="None"/>
<mapping envisionName="cn_invalid" nwName="cn_invalid" flags="None"/>
<mapping envisionName="cn_ip_proto_ver" nwName="cn_ip_proto_ver" flags="None"/>
<mapping envisionName="cn_ipv4_ident" nwName="cn_ipv4_ident" flags="None"/>
<mapping envisionName="cn_l_switch" nwName="cn_l_switch" flags="None"/>
<mapping envisionName="cn_log_did" nwName="cn_log_did" flags="None"/>
<mapping envisionName="cn_log_rid" nwName="cn_log_rid" flags="None"/>
<mapping envisionName="cn_max_ttl" nwName="cn_max_ttl" flags="None"/>
<mapping envisionName="cn_maxpcktlen" nwName="cn_maxpcktlen" flags="None"/>
<mapping envisionName="cn_min_ttl" nwName="cn_min_ttl" flags="None"/>
<mapping envisionName="cn_minpcktlen" nwName="cn_minpcktlen" flags="None"/>
<mapping envisionName="cn_mpls_lbl_1" nwName="cn_mpls_lbl_1" flags="None"/>
<mapping envisionName="cn_mpls_lbl_10" nwName="cn_mpls_lbl_10" flags="None"/>
<mapping envisionName="cn_mpls_lbl_2" nwName="cn_mpls_lbl_2" flags="None"/>
<mapping envisionName="cn_mpls_lbl_3" nwName="cn_mpls_lbl_3" flags="None"/>
<mapping envisionName="cn_mpls_lbl_4" nwName="cn_mpls_lbl_4" flags="None"/>
<mapping envisionName="cn_mpls_lbl_5" nwName="cn_mpls_lbl_5" flags="None"/>
<mapping envisionName="cn_mpls_lbl_6" nwName="cn_mpls_lbl_6" flags="None"/>
<mapping envisionName="cn_mpls_lbl_7" nwName="cn_mpls_lbl_7" flags="None"/>
<mapping envisionName="cn_mpls_lbl_8" nwName="cn_mpls_lbl_8" flags="None"/>
<mapping envisionName="cn_mpls_lbl_9" nwName="cn_mpls_lbl_9" flags="None"/>
<mapping envisionName="cn_mplstoplabel" nwName="cn_mplstoplabel" flags="None"/>
<mapping envisionName="cn_mplstoplabip" nwName="cn_mplstoplabip" flags="None"/>
<mapping envisionName="cn_mul_dst_byt" nwName="cn_mul_dst_byt" flags="None"/>
<mapping envisionName="cn_mul_dst_pks" nwName="cn_mul_dst_pks" flags="None"/>
<mapping envisionName="cn_muligmptype" nwName="cn_muligmptype" flags="None"/>
<mapping envisionName="cn_oldfileid" nwName="cn_oldfileid" flags="None"/>
<mapping envisionName="cn_oldfilesize" nwName="cn_oldfilesize" flags="None"/>
<mapping envisionName="cn_rpackets" nwName="cn_rpackets" flags="None"/>
<mapping envisionName="cn_sampalgo" nwName="cn_sampalgo" flags="None"/>
<mapping envisionName="cn_sampint" nwName="cn_sampint" flags="None"/>
<mapping envisionName="cn_seqctr" nwName="cn_seqctr" flags="None"/>
<mapping envisionName="cn_spackets" nwName="cn_spackets" flags="None"/>
<mapping envisionName="cn_src_tos" nwName="cn_src_tos" flags="None"/>
<mapping envisionName="cn_src_vlan" nwName="cn_src_vlan" flags="None"/>
<mapping envisionName="cn_sysuptime" nwName="cn_sysuptime" flags="None"/>
<mapping envisionName="cn_template_id" nwName="cn_template_id" flags="None"/>
<mapping envisionName="cn_totbytsexp" nwName="cn_totbytsexp" flags="None"/>
<mapping envisionName="cn_totflowexp" nwName="cn_totflowexp" flags="None"/>
<mapping envisionName="cn_totpcktsexp" nwName="cn_totpcktsexp" flags="None"/>
<mapping envisionName="cn_unixnanosecs" nwName="cn_unixnanosecs" flags="None"/>
<mapping envisionName="cn_v6flowlabel" nwName="cn_v6flowlabel" flags="None"/>
<mapping envisionName="cn_v6optheaders" nwName="cn_v6optheaders" flags="None"/>
<mapping envisionName="cs_accesskeyid" nwName="cs_accesskeyid" flags="None"/>
<mapping envisionName="cs_accountid" nwName="cs_accountid" flags="None"/>
<mapping envisionName="cs_agency_dst" nwName="cs_agency_dst" flags="None"/>
<mapping envisionName="cs_analyzedby" nwName="cs_analyzedby" flags="None"/>
<mapping envisionName="cs_av_other" nwName="cs_av_other" flags="None"/>
<mapping envisionName="cs_av_primary" nwName="cs_av_primary" flags="None"/>
<mapping envisionName="cs_av_secondary" nwName="cs_av_secondary" flags="None"/>
<mapping envisionName="cs_bgpv6nxthop" nwName="cs_bgpv6nxthop" flags="None"/>
<mapping envisionName="cs_customdate" nwName="cs_customdate" flags="None"/>
<mapping envisionName="cs_datecret" nwName="cs_datecret" flags="None"/>
<mapping envisionName="cs_devfacility" nwName="cs_devfacility" flags="None"/>
<mapping envisionName="cs_devservice" nwName="cs_devservice" flags="None"/>
<mapping envisionName="cs_dst_tld" nwName="cs_dst_tld" flags="None"/>
<mapping envisionName="cs_eth_dst_ven" nwName="cs_eth_dst_ven" flags="None"/>
<mapping envisionName="cs_eth_src_ven" nwName="cs_eth_src_ven" flags="None"/>
<mapping envisionName="cs_event_uuid" nwName="cs_event_uuid" flags="None"/>
<mapping envisionName="cs_filectime" nwName="cs_filectime" flags="None"/>
<mapping envisionName="cs_fileid" nwName="cs_fileid" flags="None"/>
<mapping envisionName="cs_filemtime" nwName="cs_filemtime" flags="None"/>
<mapping envisionName="cs_fileperm" nwName="cs_fileperm" flags="None"/>
<mapping envisionName="cs_fld" nwName="cs_fld" flags="None"/>
<mapping envisionName="cs_frametype" nwName="cs_frametype" flags="None"/>
<mapping envisionName="cs_identityarn" nwName="cs_identityarn" flags="None"/>
<mapping envisionName="cs_if_desc" nwName="cs_if_desc" flags="None"/>
<mapping envisionName="cs_if_name" nwName="cs_if_name" flags="None"/>
<mapping envisionName="cs_ip_next_hop" nwName="cs_ip_next_hop" flags="None"/>
<mapping envisionName="cs_ipv4dstpre" nwName="cs_ipv4dstpre" flags="None"/>
<mapping envisionName="cs_ipv4srcpre" nwName="cs_ipv4srcpre" flags="None"/>
<mapping envisionName="cs_lifetime" nwName="cs_lifetime" flags="None"/>
<mapping envisionName="cs_log_medium" nwName="cs_log_medium" flags="None"/>
<mapping envisionName="cs_loginname" nwName="cs_loginname" flags="None"/>
<mapping envisionName="cs_oldfilectime" nwName="cs_oldfilectime" flags="None"/>
<mapping envisionName="cs_oldfilehash" nwName="cs_oldfilehash" flags="None"/>
<mapping envisionName="cs_oldfilemtime" nwName="cs_oldfilemtime" flags="None"/>
<mapping envisionName="cs_oldfilename" nwName="cs_oldfilename" flags="None"/>
<mapping envisionName="cs_oldfilepath" nwName="cs_oldfilepath" flags="None"/>
<mapping envisionName="cs_oldfileperm" nwName="cs_oldfileperm" flags="None"/>
<mapping envisionName="cs_oldfiletype" nwName="cs_oldfiletype" flags="None"/>
<mapping envisionName="cs_operation" nwName="cs_operation" flags="None"/>
<mapping envisionName="cs_packettype" nwName="cs_packettype" flags="None"/>
<mapping envisionName="cs_paramkey" nwName="cs_paramkey" flags="None"/>
<mapping envisionName="cs_paramvalue" nwName="cs_paramvalue" flags="None"/>
<mapping envisionName="cs_payload" nwName="cs_payload" flags="None"/>
<mapping envisionName="cs_registrant" nwName="cs_registrant" flags="None"/>
<mapping envisionName="cs_registrar" nwName="cs_registrar" flags="None"/>
<mapping envisionName="cs_req_inst_id" nwName="cs_req_inst_id" flags="None"/>
<mapping envisionName="cs_reqcookies" nwName="cs_reqcookies" flags="None"/>
<mapping envisionName="cs_reqid" nwName="cs_reqid" flags="None"/>
<mapping envisionName="cs_resp_acctid" nwName="cs_resp_acctid" flags="None"/>
<mapping envisionName="cs_rpayload" nwName="cs_rpayload" flags="None"/>
<mapping envisionName="cs_sampler_name" nwName="cs_sampler_name" flags="None"/>
<mapping envisionName="cs_streams" nwName="cs_streams" flags="None"/>
<mapping envisionName="cs_tenant" nwName="cs_tenant" flags="None"/>
<mapping envisionName="cs_tenantid" nwName="cs_tenantid" flags="None"/>
<mapping envisionName="cs_transaction" nwName="cs_transaction" flags="None"/>
<mapping envisionName="cs_user" nwName="cs_user" flags="None"/>
<mapping envisionName="cs_v6nxthop" nwName="cs_v6nxthop" flags="None"/>
<mapping envisionName="cs_whois_server" nwName="cs_whois_server" flags="None"/>
<mapping envisionName="dinterface" nwName="dinterface" flags="None" envisionDisplayName="DestinationInterface"/>
<mapping envisionName="dmacaddr" nwName="eth.dst" flags="None" format="MAC" envisionDisplayName="DestMacAddress|DestinationMacAddress"/>
<mapping envisionName="dmask" nwName="dmask" flags="None"/>
<mapping envisionName="dn" nwName="dn" flags="None"/> <mapping envisionName="dst_dn" nwName="dn.dst" flags="None"/> <mapping envisionName="fqdn" nwName="fqdn" flags="None" envisionDisplayName="FQDN"/> <mapping envisionName="src_dn" nwName="dn.src" flags="None"/>
<mapping envisionName="dtransport" nwName="dtransport" flags="None"/>
<mapping envisionName="event_counter" nwName="event.counter" flags="None" format="Int32"/>
<mapping envisionName="filetype" nwName="filetype" flags="None" />
<mapping envisionName="gateway" nwName="gateway" flags="None"/>
<mapping envisionName="hardware_id" nwName="hardware.id" flags="None"/>
<mapping envisionName="icmptype" nwName="icmp.type" flags="None" format="UInt32"/>
<mapping envisionName="location_city" nwName="loc.city" flags="None"/>
<mapping envisionName="dmacaddr" nwName="eth.dst" flags="None" format="MAC" envisionDisplayName="DestMacAddress|DestinationMacAddress"/> <mapping envisionName="macaddr" nwName="eth.host" flags="None" format="MAC" envisionDisplayName="DeviceMacAddress"/> <mapping envisionName="smacaddr" nwName="eth.src" flags="None" format="MAC" envisionDisplayName="SourceMacAddress" nullTokens="Unknown"/>
<mapping envisionName="packets" nwName="packets" flags="None" format="UInt32"/>
<mapping envisionName="param_endtime" nwName="param_endtime" flags="None"/>
<mapping envisionName="param_event_time" nwName="param_event_time" flags="None"/>
<mapping envisionName="param_starttime" nwName="param_starttime" flags="None"/>
<mapping envisionName="privilege" nwName="privilege" flags="None" envisionDisplayName="Privilege|Privileges"/>
<mapping envisionName="process_id_src" nwName="process.id.src" flags="None" format="Int32" envisionDisplayName="SourceProcessId" nullTokens="(null)|-"/>
<mapping envisionName="process_src" nwName="process.src" flags="None" envisionDisplayName="SourceProcess"/>
<mapping envisionName="c_domain" nwName="sdomain" flags="None" envisionDisplayName="C_Domain|ClientDomain"/> <mapping envisionName="sdomain" nwName="sdomain" flags="None"/>
<mapping envisionName="sessionid" nwName="log.session.id" flags="None"/> <mapping envisionName="sessionid1" nwName="log.session.id1" flags="None"/>
<mapping envisionName="sinterface" nwName="sinterface" flags="None" envisionDisplayName="SourceInterface"/>
<mapping envisionName="smacaddr" nwName="eth.src" flags="None" format="MAC" envisionDisplayName="SourceMacAddress" nullTokens="Unknown"/>
<mapping envisionName="smask" nwName="smask" flags="None"/>
<mapping envisionName="timezone" nwName="timezone" flags="None"/>
<mapping envisionName="rule_uid" nwName="rule.uid" flags="None"/> <mapping envisionName="uid" nwName="username" flags="None" envisionDisplayName="UserID|UID|Uid" nullTokens="none|-"/>
<mapping envisionName="user_org" nwName="org" flags="None" envisionDisplayName="UserOrg|UserOrginization"/>
<!-- END List of keys Not in table-map-custom.xml -->


This text should be added to the /etc/netwitness/ng/envision/etc/table-map-custom.xml
After these meta keys are added, the Log Decoder service will need to be restarted.

For more information about the table-map-custom.xml file, see the Maintain Table Map Files in Hosts and Services Getting Started Guide
 
NotesThe contents of the findmissing.sh script are shown below.

#!/bin/bash
#Script to add show additional meta keys that could be added to table-map-custom.xml file
#David Waugh
if [ -f /tmp/custom_keys_cef ]
then
 rm -rf /tmp/custom_keys_cef
fi

if [ -d /etc/netwitness/ng/envision/etc/devices/cef/ ]
then
 grep ExtensionKey /etc/netwitness/ng/envision/etc/devices/cef/* |sort | uniq |cut -d " " -f 3 |cut -d \" -
f 2|grep -v \< |sort | uniq > /tmp/custom_keys_cef
else
 echo "CEF Parser is not installed. You can install this parser from RSA Live if you wish."
 fi

grep "<mapping " /etc/netwitness/ng/envision/etc/table-map.xml | grep -v None |cut -d \" -f 4 >> /tmp/custo
m_keys_cef

cat /tmp/custom_keys_cef
cat /tmp/custom_keys_cef |sort |uniq >/tmp/custom_keys_cef_sorted
mv /tmp/custom_keys_cef_sorted /tmp/custom_keys_cef
 rm -rf /tmp/TOADD.txt

if [ ! -f /etc/netwitness/ng/envision/etc/table-map-custom.xml ]
then
  echo "You do not have a table-map-custom.xml already defined"
else
  echo "table-map-custom.xml file is present"
fi
echo "<!-- BEGIN List of keys Not in table-map-custom.xml -->" >>/tmp/TOADD.txt
for metakey in $(cat /tmp/custom_keys_cef)
do
  METAKEY=$metakey
  if [ -f /etc/netwitness/ng/envision/etc/table-map-custom.xml ]
  then
    COUNTCUSTOM=$(grep $metakey /etc/netwitness/ng/envision/etc/table-map-custom.xml |wc -l)
  else
    COUNTCUSTOM=0
  fi

 COUNTTABLEMAP=$(grep $metakey /etc/netwitness/ng/envision/etc/table-map.xml|wc -l)
 COUNTISTRANSIENT=$(grep $metakey /etc/netwitness/ng/envision/etc/table-map.xml| grep -v None |wc -l)
  echo $metakey $COUNTCUSTOM $COUNTTABLEMAP
 # Transient Keys that need to be added that are already in table-map.xml
 if [ $COUNTCUSTOM -eq 0 ] && [ $COUNTTABLEMAP -gt 0 ] && [ $COUNTISTRANSIENT -gt 0 ]
 then
  echo $(grep $metakey  /etc/netwitness/ng/envision/etc/table-map.xml| grep "<mapping ") >>/tmp/TOADD.txt
 fi

 # Custom Keys that do not exist in table-map.xml at all and need to be added
 # Add in the standard Format
 if [ $COUNTCUSTOM -eq 0 ] &&  [ $COUNTTABLEMAP -eq 0 ]
 then
   echo \<mapping envisionName=\"$metakey\" nwName=\"$metakey\" flags=\"None\"\/\> >>/tmp/TOADD.txt
 fi

done
 sed -i -- 's/Transient/None/g' /tmp/TOADD.txt
echo "<!-- END List of keys Not in table-map-custom.xml -->" >>/tmp/TOADD.txt

echo "Additional Meta keys for table-map-custom.xml can be found in /tmp/TOADD.txt"
echo "Paste the contents of this file between the <mappings> </mappings> tags"
echo "into the file /etc/netwitness/ng/envision/etc/table-map-custom.xml"



 

Attachments

Outcomes