000031115 - Enable additional meta keys in table-map-custom.xml for enhanced log information in RSA Security Analytics

Document created by RSA Customer Support Employee on Jun 14, 2016Last modified by RSA Customer Support Employee on Apr 21, 2017
Version 5Show Document
  • View in full screen mode

Article Content

Article Number000031115
Applies ToRSA Product Set: Security Analytics
RSA Product/Service Type: Log Decoder
RSA Version/Condition: 10.x
Platform: CentOS
IssueOut of the box, not all meta keys are enabled in the table-map.xml file which means some information from log parsers may not be captured. Not all of this information may be required, but this script is a way to highlight these additional meta keys so that they can be added into the following file.
/etc/netwitness/ng/envision/etc/table-map-custom.xml

The script attached to this solution will display the meta keys to add to this file. The script will highlight meta keys that are:
  • Currently set to transient in the table-map.xml file
  • ExtensionKeys in the CEF parser
ResolutionCopy the attached script to the Log Decoder appliance and make it executable with the command below.
chmod +x findmissing.sh

On running the script the output below will be displayed.
 ./findmissing.sh
Additional Meta keys for table-map-custom.xml can be found in /tmp/TOADD.txt
Paste the contents of this file between the <mappings> </mappings> tags
into the file /etc/netwitness/ng/envision/etc/table-map-custom.xml

  An example of the output in /tmp/TOADD.txt is shown below.
<!-- BEGIN List of keys Not in table-map-custom.xml -->
<mapping envisionName="cn_acttimeout" nwName="cn_acttimeout" flags="None"/>
<mapping envisionName="cn_asn_dst" nwName="cn_asn_dst" flags="None"/>
<mapping envisionName="cn_asn_src" nwName="cn_asn_src" flags="None"/>
<mapping envisionName="cn_bgpv4nxthop" nwName="cn_bgpv4nxthop" flags="None"/>
<mapping envisionName="cn_ctr_dst_code" nwName="cn_ctr_dst_code" flags="None"/>
<mapping envisionName="cn_dst_tos" nwName="cn_dst_tos" flags="None"/>
<mapping envisionName="cn_dst_vlan" nwName="cn_dst_vlan" flags="None"/>
<mapping envisionName="cn_engine_id" nwName="cn_engine_id" flags="None"/>
<mapping envisionName="cn_engine_type" nwName="cn_engine_type" flags="None"/>
<mapping envisionName="cn_eventver" nwName="cn_eventver" flags="None"/>
<mapping envisionName="cn_f_switch" nwName="cn_f_switch" flags="None"/>
<mapping envisionName="cn_fld" nwName="cn_fld" flags="None"/>
<mapping envisionName="cn_flowsampid" nwName="cn_flowsampid" flags="None"/>
<mapping envisionName="cn_flowsampintv" nwName="cn_flowsampintv" flags="None"/>
<mapping envisionName="cn_flowsampmode" nwName="cn_flowsampmode" flags="None"/>
<mapping envisionName="cn_inacttimeout" nwName="cn_inacttimeout" flags="None"/>
<mapping envisionName="cn_inpermbyts" nwName="cn_inpermbyts" flags="None"/>
<mapping envisionName="cn_inpermpckts" nwName="cn_inpermpckts" flags="None"/>
<mapping envisionName="cn_invalid" nwName="cn_invalid" flags="None"/>
<mapping envisionName="cn_ip_proto_ver" nwName="cn_ip_proto_ver" flags="None"/>
<mapping envisionName="cn_ipv4_ident" nwName="cn_ipv4_ident" flags="None"/>
<mapping envisionName="cn_l_switch" nwName="cn_l_switch" flags="None"/>
<mapping envisionName="cn_log_did" nwName="cn_log_did" flags="None"/>
<mapping envisionName="cn_log_rid" nwName="cn_log_rid" flags="None"/>
<mapping envisionName="cn_max_ttl" nwName="cn_max_ttl" flags="None"/>
<mapping envisionName="cn_maxpcktlen" nwName="cn_maxpcktlen" flags="None"/>
<mapping envisionName="cn_min_ttl" nwName="cn_min_ttl" flags="None"/>
<mapping envisionName="cn_minpcktlen" nwName="cn_minpcktlen" flags="None"/>
<mapping envisionName="cn_mpls_lbl_1" nwName="cn_mpls_lbl_1" flags="None"/>
<mapping envisionName="cn_mpls_lbl_10" nwName="cn_mpls_lbl_10" flags="None"/>
<mapping envisionName="cn_mpls_lbl_2" nwName="cn_mpls_lbl_2" flags="None"/>
<mapping envisionName="cn_mpls_lbl_3" nwName="cn_mpls_lbl_3" flags="None"/>
<mapping envisionName="cn_mpls_lbl_4" nwName="cn_mpls_lbl_4" flags="None"/>
<mapping envisionName="cn_mpls_lbl_5" nwName="cn_mpls_lbl_5" flags="None"/>
<mapping envisionName="cn_mpls_lbl_6" nwName="cn_mpls_lbl_6" flags="None"/>
<mapping envisionName="cn_mpls_lbl_7" nwName="cn_mpls_lbl_7" flags="None"/>
<mapping envisionName="cn_mpls_lbl_8" nwName="cn_mpls_lbl_8" flags="None"/>
<mapping envisionName="cn_mpls_lbl_9" nwName="cn_mpls_lbl_9" flags="None"/>
<mapping envisionName="cn_mplstoplabel" nwName="cn_mplstoplabel" flags="None"/>
<mapping envisionName="cn_mplstoplabip" nwName="cn_mplstoplabip" flags="None"/>
<mapping envisionName="cn_mul_dst_byt" nwName="cn_mul_dst_byt" flags="None"/>
<mapping envisionName="cn_mul_dst_pks" nwName="cn_mul_dst_pks" flags="None"/>
<mapping envisionName="cn_muligmptype" nwName="cn_muligmptype" flags="None"/>
<mapping envisionName="cn_oldfileid" nwName="cn_oldfileid" flags="None"/>
<mapping envisionName="cn_oldfilesize" nwName="cn_oldfilesize" flags="None"/>
<mapping envisionName="cn_rpackets" nwName="cn_rpackets" flags="None"/>
<mapping envisionName="cn_sampalgo" nwName="cn_sampalgo" flags="None"/>
<mapping envisionName="cn_sampint" nwName="cn_sampint" flags="None"/>
<mapping envisionName="cn_seqctr" nwName="cn_seqctr" flags="None"/>
<mapping envisionName="cn_spackets" nwName="cn_spackets" flags="None"/>
<mapping envisionName="cn_src_tos" nwName="cn_src_tos" flags="None"/>
<mapping envisionName="cn_src_vlan" nwName="cn_src_vlan" flags="None"/>
<mapping envisionName="cn_sysuptime" nwName="cn_sysuptime" flags="None"/>
<mapping envisionName="cn_template_id" nwName="cn_template_id" flags="None"/>
<mapping envisionName="cn_totbytsexp" nwName="cn_totbytsexp" flags="None"/>
<mapping envisionName="cn_totflowexp" nwName="cn_totflowexp" flags="None"/>
<mapping envisionName="cn_totpcktsexp" nwName="cn_totpcktsexp" flags="None"/>
<mapping envisionName="cn_unixnanosecs" nwName="cn_unixnanosecs" flags="None"/>
<mapping envisionName="cn_v6flowlabel" nwName="cn_v6flowlabel" flags="None"/>
<mapping envisionName="cn_v6optheaders" nwName="cn_v6optheaders" flags="None"/>
<mapping envisionName="cs_accesskeyid" nwName="cs_accesskeyid" flags="None"/>
<mapping envisionName="cs_accountid" nwName="cs_accountid" flags="None"/>
<mapping envisionName="cs_agency_dst" nwName="cs_agency_dst" flags="None"/>
<mapping envisionName="cs_analyzedby" nwName="cs_analyzedby" flags="None"/>
<mapping envisionName="cs_av_other" nwName="cs_av_other" flags="None"/>
<mapping envisionName="cs_av_primary" nwName="cs_av_primary" flags="None"/>
<mapping envisionName="cs_av_secondary" nwName="cs_av_secondary" flags="None"/>
<mapping envisionName="cs_bgpv6nxthop" nwName="cs_bgpv6nxthop" flags="None"/>
<mapping envisionName="cs_customdate" nwName="cs_customdate" flags="None"/>
<mapping envisionName="cs_datecret" nwName="cs_datecret" flags="None"/>
<mapping envisionName="cs_devfacility" nwName="cs_devfacility" flags="None"/>
<mapping envisionName="cs_devservice" nwName="cs_devservice" flags="None"/>
<mapping envisionName="cs_dst_tld" nwName="cs_dst_tld" flags="None"/>
<mapping envisionName="cs_eth_dst_ven" nwName="cs_eth_dst_ven" flags="None"/>
<mapping envisionName="cs_eth_src_ven" nwName="cs_eth_src_ven" flags="None"/>
<mapping envisionName="cs_event_uuid" nwName="cs_event_uuid" flags="None"/>
<mapping envisionName="cs_filectime" nwName="cs_filectime" flags="None"/>
<mapping envisionName="cs_fileid" nwName="cs_fileid" flags="None"/>
<mapping envisionName="cs_filemtime" nwName="cs_filemtime" flags="None"/>
<mapping envisionName="cs_fileperm" nwName="cs_fileperm" flags="None"/>
<mapping envisionName="cs_fld" nwName="cs_fld" flags="None"/>
<mapping envisionName="cs_frametype" nwName="cs_frametype" flags="None"/>
<mapping envisionName="cs_identityarn" nwName="cs_identityarn" flags="None"/>
<mapping envisionName="cs_if_desc" nwName="cs_if_desc" flags="None"/>
<mapping envisionName="cs_if_name" nwName="cs_if_name" flags="None"/>
<mapping envisionName="cs_ip_next_hop" nwName="cs_ip_next_hop" flags="None"/>
<mapping envisionName="cs_ipv4dstpre" nwName="cs_ipv4dstpre" flags="None"/>
<mapping envisionName="cs_ipv4srcpre" nwName="cs_ipv4srcpre" flags="None"/>
<mapping envisionName="cs_lifetime" nwName="cs_lifetime" flags="None"/>
<mapping envisionName="cs_log_medium" nwName="cs_log_medium" flags="None"/>
<mapping envisionName="cs_loginname" nwName="cs_loginname" flags="None"/>
<mapping envisionName="cs_oldfilectime" nwName="cs_oldfilectime" flags="None"/>
<mapping envisionName="cs_oldfilehash" nwName="cs_oldfilehash" flags="None"/>
<mapping envisionName="cs_oldfilemtime" nwName="cs_oldfilemtime" flags="None"/>
<mapping envisionName="cs_oldfilename" nwName="cs_oldfilename" flags="None"/>
<mapping envisionName="cs_oldfilepath" nwName="cs_oldfilepath" flags="None"/>
<mapping envisionName="cs_oldfileperm" nwName="cs_oldfileperm" flags="None"/>
<mapping envisionName="cs_oldfiletype" nwName="cs_oldfiletype" flags="None"/>
<mapping envisionName="cs_operation" nwName="cs_operation" flags="None"/>
<mapping envisionName="cs_packettype" nwName="cs_packettype" flags="None"/>
<mapping envisionName="cs_paramkey" nwName="cs_paramkey" flags="None"/>
<mapping envisionName="cs_paramvalue" nwName="cs_paramvalue" flags="None"/>
<mapping envisionName="cs_payload" nwName="cs_payload" flags="None"/>
<mapping envisionName="cs_registrant" nwName="cs_registrant" flags="None"/>
<mapping envisionName="cs_registrar" nwName="cs_registrar" flags="None"/>
<mapping envisionName="cs_req_inst_id" nwName="cs_req_inst_id" flags="None"/>
<mapping envisionName="cs_reqcookies" nwName="cs_reqcookies" flags="None"/>
<mapping envisionName="cs_reqid" nwName="cs_reqid" flags="None"/>
<mapping envisionName="cs_resp_acctid" nwName="cs_resp_acctid" flags="None"/>
<mapping envisionName="cs_rpayload" nwName="cs_rpayload" flags="None"/>
<mapping envisionName="cs_sampler_name" nwName="cs_sampler_name" flags="None"/>
<mapping envisionName="cs_streams" nwName="cs_streams" flags="None"/>
<mapping envisionName="cs_tenant" nwName="cs_tenant" flags="None"/>
<mapping envisionName="cs_tenantid" nwName="cs_tenantid" flags="None"/>
<mapping envisionName="cs_transaction" nwName="cs_transaction" flags="None"/>
<mapping envisionName="cs_user" nwName="cs_user" flags="None"/>
<mapping envisionName="cs_v6nxthop" nwName="cs_v6nxthop" flags="None"/>
<mapping envisionName="cs_whois_server" nwName="cs_whois_server" flags="None"/>
<mapping envisionName="dinterface" nwName="dinterface" flags="None" envisionDisplayName="DestinationInterface"/>
<mapping envisionName="dmacaddr" nwName="eth.dst" flags="None" format="MAC" envisionDisplayName="DestMacAddress|DestinationMacAddress"/>
<mapping envisionName="dmask" nwName="dmask" flags="None"/>
<mapping envisionName="dn" nwName="dn" flags="None"/> <mapping envisionName="dst_dn" nwName="dn.dst" flags="None"/> <mapping envisionName="fqdn" nwName="fqdn" flags="None" envisionDisplayName="FQDN"/> <mapping envisionName="src_dn" nwName="dn.src" flags="None"/>
<mapping envisionName="dtransport" nwName="dtransport" flags="None"/>
<mapping envisionName="event_counter" nwName="event.counter" flags="None" format="Int32"/>
<mapping envisionName="filetype" nwName="filetype" flags="None" />
<mapping envisionName="gateway" nwName="gateway" flags="None"/>
<mapping envisionName="hardware_id" nwName="hardware.id" flags="None"/>
<mapping envisionName="icmptype" nwName="icmp.type" flags="None" format="UInt32"/>
<mapping envisionName="location_city" nwName="loc.city" flags="None"/>
<mapping envisionName="dmacaddr" nwName="eth.dst" flags="None" format="MAC" envisionDisplayName="DestMacAddress|DestinationMacAddress"/> <mapping envisionName="macaddr" nwName="eth.host" flags="None" format="MAC" envisionDisplayName="DeviceMacAddress"/> <mapping envisionName="smacaddr" nwName="eth.src" flags="None" format="MAC" envisionDisplayName="SourceMacAddress" nullTokens="Unknown"/>
<mapping envisionName="packets" nwName="packets" flags="None" format="UInt32"/>
<mapping envisionName="param_endtime" nwName="param_endtime" flags="None"/>
<mapping envisionName="param_event_time" nwName="param_event_time" flags="None"/>
<mapping envisionName="param_starttime" nwName="param_starttime" flags="None"/>
<mapping envisionName="privilege" nwName="privilege" flags="None" envisionDisplayName="Privilege|Privileges"/>
<mapping envisionName="process_id_src" nwName="process.id.src" flags="None" format="Int32" envisionDisplayName="SourceProcessId" nullTokens="(null)|-"/>
<mapping envisionName="process_src" nwName="process.src" flags="None" envisionDisplayName="SourceProcess"/>
<mapping envisionName="c_domain" nwName="sdomain" flags="None" envisionDisplayName="C_Domain|ClientDomain"/> <mapping envisionName="sdomain" nwName="sdomain" flags="None"/>
<mapping envisionName="sessionid" nwName="log.session.id" flags="None"/> <mapping envisionName="sessionid1" nwName="log.session.id1" flags="None"/>
<mapping envisionName="sinterface" nwName="sinterface" flags="None" envisionDisplayName="SourceInterface"/>
<mapping envisionName="smacaddr" nwName="eth.src" flags="None" format="MAC" envisionDisplayName="SourceMacAddress" nullTokens="Unknown"/>
<mapping envisionName="smask" nwName="smask" flags="None"/>
<mapping envisionName="timezone" nwName="timezone" flags="None"/>
<mapping envisionName="rule_uid" nwName="rule.uid" flags="None"/> <mapping envisionName="uid" nwName="username" flags="None" envisionDisplayName="UserID|UID|Uid" nullTokens="none|-"/>
<mapping envisionName="user_org" nwName="org" flags="None" envisionDisplayName="UserOrg|UserOrginization"/>
<!-- END List of keys Not in table-map-custom.xml -->

This text should be added to the /etc/netwitness/ng/envision/etc/table-map-custom.xml
After these meta keys are added, the Log Decoder service will need to be restarted.
For more information on the table-map-custom.xml file, refer to the Security Analytics User Guide.
 
NotesThe contents of the findmissing.sh script are shown below.
#!/bin/bash
#Script to add show additional meta keys that could be added to table-map-custom.xml file
#David Waugh
if [ -f /tmp/custom_keys_cef ]
then
 rm -rf /tmp/custom_keys_cef
fi
if [ -d /etc/netwitness/ng/envision/etc/devices/cef/ ]
then
 grep ExtensionKey /etc/netwitness/ng/envision/etc/devices/cef/* |sort | uniq |cut -d " " -f 3 |cut -d \" -
f 2|grep -v \< |sort | uniq > /tmp/custom_keys_cef
else
 echo "CEF Parser is not installed. You can install this parser from RSA Live if you wish."
 fi
grep "<mapping " /etc/netwitness/ng/envision/etc/table-map.xml | grep -v None |cut -d \" -f 4 >> /tmp/custo
m_keys_cef
cat /tmp/custom_keys_cef
cat /tmp/custom_keys_cef |sort |uniq >/tmp/custom_keys_cef_sorted
mv /tmp/custom_keys_cef_sorted /tmp/custom_keys_cef
 rm -rf /tmp/TOADD.txt
if [ ! -f /etc/netwitness/ng/envision/etc/table-map-custom.xml ]
then
  echo "You do not have a table-map-custom.xml already defined"
else
  echo "table-map-custom.xml file is present"
fi
echo "<!-- BEGIN List of keys Not in table-map-custom.xml -->" >>/tmp/TOADD.txt
for metakey in $(cat /tmp/custom_keys_cef)
do
  METAKEY=$metakey
  if [ -f /etc/netwitness/ng/envision/etc/table-map-custom.xml ]
  then
    COUNTCUSTOM=$(grep $metakey /etc/netwitness/ng/envision/etc/table-map-custom.xml |wc -l)
  else
    COUNTCUSTOM=0
  fi
 COUNTTABLEMAP=$(grep $metakey /etc/netwitness/ng/envision/etc/table-map.xml|wc -l)
 COUNTISTRANSIENT=$(grep $metakey /etc/netwitness/ng/envision/etc/table-map.xml| grep -v None |wc -l)
  echo $metakey $COUNTCUSTOM $COUNTTABLEMAP
 # Transient Keys that need to be added that are already in table-map.xml
 if [ $COUNTCUSTOM -eq 0 ] && [ $COUNTTABLEMAP -gt 0 ] && [ $COUNTISTRANSIENT -gt 0 ]
 then
  echo $(grep $metakey  /etc/netwitness/ng/envision/etc/table-map.xml| grep "<mapping ") >>/tmp/TOADD.txt
 fi
 # Custom Keys that do not exist in table-map.xml at all and need to be added
 # Add in the standard Format
 if [ $COUNTCUSTOM -eq 0 ] &&  [ $COUNTTABLEMAP -eq 0 ]
 then
   echo \<mapping envisionName=\"$metakey\" nwName=\"$metakey\" flags=\"None\"\/\> >>/tmp/TOADD.txt
 fi
done
 sed -i -- 's/Transient/None/g' /tmp/TOADD.txt
echo "<!-- END List of keys Not in table-map-custom.xml -->" >>/tmp/TOADD.txt
echo "Additional Meta keys for table-map-custom.xml can be found in /tmp/TOADD.txt"
echo "Paste the contents of this file between the <mappings> </mappings> tags"
echo "into the file /etc/netwitness/ng/envision/etc/table-map-custom.xml"

 

Attachments

Outcomes