000015662 - 'Received fatal alert: bad_record_mac' when trying to establish an admin API connection to AM 7.1

Document created by RSA Customer Support Employee on Jun 14, 2016Last modified by RSA Customer Support Employee on Apr 21, 2017
Version 2Show Document
  • View in full screen mode

Article Content

Article Number000015662
Issue"Received fatal alert: bad_record_mac" when trying to establish an admin API connection to AM 7.1
Authentication Manager 7.1 with an hotfix released after march 2009 applied to it.
CauseWhen trying to establish a connection to the admin API the error message  

"Received fatal alert: bad_record_mac" 
is usually experienced when the application server running AM 7.1 is configured to only accept SSLv3 handshakes but the client attempt an SSL handshake using SSLv2Hello. 
An AM 7.1 hotfix released around mid 2009 enforces all AM 7.1 services to accept SSLv3 handshakes only. This this issue might suddenly start occurring after applying a recent AM 7.1 hotfix.
Because of this change some application that aren't implementing a pure SSLv3 handshake will fail to connect to the server. Applications using the AM 7.1 SDK or webservices clients (based on Axis for example) are most likely to be affected by this.
ResolutionIn order to correct this issue when using a webservices based client, you'll have to configure your API client to only use SSLv3 handshakes.
For Axis the following two steps are required:
  1. Implement a custom SecureSocketFactory that enforces SSLv3 communication only. See the sample source code in the next section of this solution.
  2. Configure Axis so that the custom SecureSocketFactory is used when setting up SSL connections. In order to achieve this you'll simply have to add the following line to the Axis client application:
AxisProperties.setProperty("axis.socketSecureFactory", "com.rsa.samples.authn.axis.SSLv3Factory3");


As an example, this might be set in the client whilst setting parameters such as an HTTP username and password value or where a maintain session state is set such as:

      CommandServerService sserv = new CommandServerServiceLocator();
      CommandServer cserver = new CommandServerSoapBindingStub(portAddress sserv);

      ( (org.apache.axis.client.Stub) cserver).setMaintainSession(true);
      ( (org.apache.axis.client.Stub) cserver).setUsername(httpuser);
      ( (org.apache.axis.client.Stub) cserver).setPassword(httppwd);
      org.apache.axis.AxisProperties.setProperty("axis.socketSecureFactory","com.rsa.samples.authn.axis.SSLv3Factory3");

Source code of sample SecureSocketFactory enforcing SSLv3 connections. This code is a slightly modified version of Axis' default SecureSocketFactory. Essentially only the lines highlighted in bold have been added to the original source code.
/*
 * Copyright 2001-2004 The Apache Software Foundation.
 * 
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 * 
 * 
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.rsa.samples.authn.axis;
import org.apache.axis.components.net.*;
import org.apache.axis.utils.Messages;
import org.apache.axis.utils.XMLUtils;
import org.apache.axis.utils.StringUtils;
import javax.net.ssl.SSLSocket;
import javax.net.ssl.SSLSocketFactory;
import java.io.BufferedWriter;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.PrintWriter;
import java.net.Socket;
import java.util.Hashtable;
/**
 * SSL socket factory. It _requires_ a valid RSA key and
 * JSSE. (borrowed code from tomcat)
 * 
 * THIS CODE STILL HAS DEPENDENCIES ON sun.* and com.sun.*
 *
 * @author Davanum Srinivas (dims@yahoo.com)
 */
public class SSLv3Factory3 extends DefaultSocketFactory implements SecureSocketFactory {
    /** Field sslFactory           */
    protected SSLSocketFactory sslFactory = null;
    /**
     * Constructor JSSESocketFactory
     *
     * @param attributes
     */
    public SSLv3Factory3(Hashtable attributes) {
        super(attributes);
    }
    /**
     * Initialize the SSLSocketFactory
     * @throws IOException
     */ 
    protected void initFactory() throws IOException {
        sslFactory = (SSLSocketFactory)SSLSocketFactory.getDefault();
    }
    
    /**
     * creates a secure socket
     *
     * @param host
     * @param port
     * @param otherHeaders
     * @param useFullURL
     *
     * @return Socket
     * @throws Exception
     */
    public Socket create(
            String host, int port, StringBuffer otherHeaders, BooleanHolder useFullURL)
            throws Exception {
        if (sslFactory == null) {
            initFactory();
        }
        if (port == -1) {
            port = 443;
        }
        TransportClientProperties tcp = TransportClientPropertiesFactory.create("https");
        boolean hostInNonProxyList = isHostInNonProxyList(host, tcp.getNonProxyHosts());
        SSLSocket sslSocket = null;
        if (tcp.getProxyHost().length() == 0 || hostInNonProxyList) {
            // direct SSL connection
            sslSocket = (SSLSocket) sslFactory.createSocket(host, port);
                    sslSocket.setEnabledProtocols(new String[]{
                    "SSLv3"
                });
        } else {
            // Default proxy port is 80, even for https
            int tunnelPort = (tcp.getProxyPort().length() != 0)
                             ? Integer.parseInt(tcp.getProxyPort())
                             : 80;
            if (tunnelPort < 0)
                tunnelPort = 80;
            // Create the regular socket connection to the proxy
            Socket tunnel = new Socket(tcp.getProxyHost(), tunnelPort);
            // The tunnel handshake method (condensed and made reflexive)
            OutputStream tunnelOutputStream = tunnel.getOutputStream();
            PrintWriter out = new PrintWriter(
                    new BufferedWriter(new OutputStreamWriter(tunnelOutputStream)));
            // More secure version... engage later?
            // PasswordAuthentication pa =
            // Authenticator.requestPasswordAuthentication(
            // InetAddress.getByName(tunnelHost),
            // tunnelPort, "SOCK", "Proxy","HTTP");
            // if(pa == null){
            // printDebug("No Authenticator set.");
            // }else{
            // printDebug("Using Authenticator.");
            // tunnelUser = pa.getUserName();
            // tunnelPassword = new String(pa.getPassword());
            // }
            out.print("CONNECT " + host + ":" + port + " HTTP/1.0\r\n"
                    + "User-Agent: AxisClient");
            if (tcp.getProxyUser().length() != 0 &&
                tcp.getProxyPassword().length() != 0) {
                // add basic authentication header for the proxy
                String encodedPassword = XMLUtils.base64encode((tcp.getProxyUser()
                        + ":"
                        + tcp.getProxyPassword()).getBytes());
                out.print("\nProxy-Authorization: Basic " + encodedPassword);
            }
            out.print("\nContent-Length: 0");
            out.print("\nPragma: no-cache");
            out.print("\r\n\r\n");
            out.flush();
            InputStream tunnelInputStream = tunnel.getInputStream();
            if (log.isDebugEnabled()) {
                log.debug(Messages.getMessage("isNull00", "tunnelInputStream",
                        "" + (tunnelInputStream
                        == null)));
            }
            String replyStr = "";
            // Make sure to read all the response from the proxy to prevent SSL negotiation failure
            // Response message terminated by two sequential newlines
            int newlinesSeen = 0;
            boolean headerDone = false;    /* Done on first newline */
            while (newlinesSeen < 2) {
                int i = tunnelInputStream.read();
                if (i < 0) {
                    throw new IOException("Unexpected EOF from proxy");
                }
                if (i == '\n') {
                    headerDone = true;
                    ++newlinesSeen;
                } else if (i != '\r') {
                    newlinesSeen = 0;
                    if (!headerDone) {
                        replyStr += String.valueOf((char) i);
                    }
                }
            }
            if (StringUtils.startsWithIgnoreWhitespaces("HTTP/1.0 200", replyStr) &&
                    StringUtils.startsWithIgnoreWhitespaces("HTTP/1.1 200", replyStr)) {
                throw new IOException(Messages.getMessage("cantTunnel00",
                        new String[]{
                            tcp.getProxyHost(),
                            "" + tunnelPort,
                            replyStr}));
            }
            // End of condensed reflective tunnel handshake method
            sslSocket = (SSLSocket) sslFactory.createSocket(tunnel, host, port, true);
            if (log.isDebugEnabled()) {
                log.debug(Messages.getMessage("setupTunnel00",
                          tcp.getProxyHost(),
                        "" + tunnelPort));
            }
        }
        ((SSLSocket) sslSocket).startHandshake();
        if (log.isDebugEnabled()) {
            log.debug(Messages.getMessage("createdSSL00"));
        }
        return sslSocket;
    }
}
Legacy Article IDa47586

Attachments

    Outcomes