From 9207290587ea8c8703486d60877731228eed80ce Mon Sep 17 00:00:00 2001

From: Stephan Bergmann <sbergman@redhat.com>

Date: Fri, 24 Aug 2018 10:27:01 +0200

Subject: [PATCH 4/5] iRelated rhbz#1618703: Properly handle failure

 en-/decoding PDF file

MIME-Version: 1.0

Content-Type: text/plain; charset=UTF-8

Content-Transfer-Encoding: 8bit

...when e.g. FIPS mode makes the various calls to rtl_cipher_initARCFOUR fail.

Reviewed-on: https://gerrit.libreoffice.org/59543

Tested-by: Jenkins

Reviewed-by: Stephan Bergmann <sbergman@redhat.com>

(cherry picked from commit 185a14525f114e58b48236284ed8e8644bc40e48)

Reviewed-on: https://gerrit.libreoffice.org/59573

Reviewed-by: Caolán McNamara <caolanm@redhat.com>

Tested-by: Caolán McNamara <caolanm@redhat.com>

(cherry picked from commit 68ffc5c83ca73c58439b7c9935283541f007db44)

Conflicts:

	filter/source/pdf/impdialog.cxx

	sdext/source/pdfimport/pdfparse/pdfentries.cxx

	vcl/source/gdi/pdfwriter_impl2.cxx

Change-Id: Id1b2222249c151470e233ab814b21228f3a8b561

---

 filter/source/pdf/impdialog.cxx               |  7 +++

 .../source/pdfimport/pdfparse/pdfentries.cxx  | 42 +++++++++++-----

 vcl/source/gdi/pdfwriter_impl2.cxx            | 48 +++++++++++--------

 3 files changed, 67 insertions(+), 30 deletions(-)

diff --git a/filter/source/pdf/impdialog.cxx b/filter/source/pdf/impdialog.cxx

index 82aaf012505a..736ac0c15f26 100644

--- a/filter/source/pdf/impdialog.cxx

+++ b/filter/source/pdf/impdialog.cxx

@@ -23,6 +23,7 @@

 #include <strings.hrc>

 #include <bitmaps.hlst>

 #include <officecfg/Office/Common.hxx>

+#include <vcl/errinf.hxx>

 #include <vcl/layout.hxx>

 #include <vcl/settings.hxx>

 #include <vcl/svapp.hxx>

@@ -1351,6 +1352,12 @@ IMPL_LINK_NOARG(ImpPDFTabSecurityPage, ClickmaPbSetPwdHdl, Button*, void)

         mbHaveOwnerPassword = !aOwnerPW.isEmpty();

         mxPreparedPasswords = vcl::PDFWriter::InitEncryption( aOwnerPW, aUserPW, true );

+        if (!mxPreparedPasswords.is()) {

+            OUString msg;

+            ErrorHandler::GetErrorString(ERRCODE_IO_NOTSUPPORTED, msg); //TOOD: handle failure

+            ScopedVclPtrInstance<MessageDialog>(this, msg, VclMessageType::Error)->Execute();

+            return;

+        }

         if( mbHaveOwnerPassword )

         {

diff --git a/sdext/source/pdfimport/pdfparse/pdfentries.cxx b/sdext/source/pdfimport/pdfparse/pdfentries.cxx

index 16563868f25c..074fb669c8da 100644

--- a/sdext/source/pdfimport/pdfparse/pdfentries.cxx

+++ b/sdext/source/pdfimport/pdfparse/pdfentries.cxx

@@ -1159,9 +1159,13 @@ static bool check_user_password( const OString& rPwd, PDFFileImplData* pData )

     {

         // see PDF reference 1.4 Algorithm 3.4

         // encrypt pad string

-        rtl_cipher_initARCFOUR( pData->m_aCipher, rtl_Cipher_DirectionEncode,

-                                aKey, nKeyLen,

-                                nullptr, 0 );

+        if (rtl_cipher_initARCFOUR( pData->m_aCipher, rtl_Cipher_DirectionEncode,

+                                    aKey, nKeyLen,

+                                    nullptr, 0 )

+            != rtl_Cipher_E_None)

+        {

+            return false; //TODO: differentiate "failed to decrypt" from "wrong password"

+        }

         rtl_cipher_encodeARCFOUR( pData->m_aCipher, nPadString, sizeof( nPadString ),

                                   nEncryptedEntry, sizeof( nEncryptedEntry ) );

         bValid = (memcmp( nEncryptedEntry, pData->m_aUEntry, 32 ) == 0);

@@ -1172,8 +1176,12 @@ static bool check_user_password( const OString& rPwd, PDFFileImplData* pData )

         rtl_digest_updateMD5( pData->m_aDigest, nPadString, sizeof( nPadString ) );

         rtl_digest_updateMD5( pData->m_aDigest, pData->m_aDocID.getStr(), pData->m_aDocID.getLength() );

         rtl_digest_getMD5( pData->m_aDigest, nEncryptedEntry, sizeof(nEncryptedEntry) );

-        rtl_cipher_initARCFOUR( pData->m_aCipher, rtl_Cipher_DirectionEncode,

-                                aKey, sizeof(aKey), nullptr, 0 );

+        if (rtl_cipher_initARCFOUR( pData->m_aCipher, rtl_Cipher_DirectionEncode,

+                                    aKey, sizeof(aKey), nullptr, 0 )

+            != rtl_Cipher_E_None)

+        {

+            return false; //TODO: differentiate "failed to decrypt" from "wrong password"

+        }

         rtl_cipher_encodeARCFOUR( pData->m_aCipher,

                                   nEncryptedEntry, 16,

                                   nEncryptedEntry, 16 ); // encrypt in place

@@ -1183,8 +1191,12 @@ static bool check_user_password( const OString& rPwd, PDFFileImplData* pData )

             for( sal_uInt32 j = 0; j < sizeof(aTempKey); j++ )

                 aTempKey[j] = static_cast<sal_uInt8>( aKey[j] ^ i );

-            rtl_cipher_initARCFOUR( pData->m_aCipher, rtl_Cipher_DirectionEncode,

-                                    aTempKey, sizeof(aTempKey), nullptr, 0 );

+            if (rtl_cipher_initARCFOUR( pData->m_aCipher, rtl_Cipher_DirectionEncode,

+                                        aTempKey, sizeof(aTempKey), nullptr, 0 )

+                != rtl_Cipher_E_None)

+            {

+                return false; //TODO: differentiate "failed to decrypt" from "wrong password"

+            }

             rtl_cipher_encodeARCFOUR( pData->m_aCipher,

                                       nEncryptedEntry, 16,

                                       nEncryptedEntry, 16 ); // encrypt in place

@@ -1230,8 +1242,12 @@ bool PDFFile::setupDecryptionData( const OString& rPwd ) const

         sal_uInt32 nKeyLen = password_to_key( rPwd, aKey, m_pData.get(), true );

         if( m_pData->m_nStandardRevision == 2 )

         {

-            rtl_cipher_initARCFOUR( m_pData->m_aCipher, rtl_Cipher_DirectionDecode,

-                                    aKey, nKeyLen, nullptr, 0 );

+            if (rtl_cipher_initARCFOUR( m_pData->m_aCipher, rtl_Cipher_DirectionDecode,

+                                        aKey, nKeyLen, nullptr, 0 )

+                != rtl_Cipher_E_None)

+            {

+                return false; //TODO: differentiate "failed to decrypt" from "wrong password"

+            }

             rtl_cipher_decodeARCFOUR( m_pData->m_aCipher,

                                       m_pData->m_aOEntry, 32,

                                       nPwd, 32 );

@@ -1244,8 +1260,12 @@ bool PDFFile::setupDecryptionData( const OString& rPwd ) const

                 sal_uInt8 nTempKey[ENCRYPTION_KEY_LEN];

                 for( unsigned int j = 0; j < sizeof(nTempKey); j++ )

                     nTempKey[j] = sal_uInt8(aKey[j] ^ i);

-                rtl_cipher_initARCFOUR( m_pData->m_aCipher, rtl_Cipher_DirectionDecode,

-                                        nTempKey, nKeyLen, nullptr, 0 );

+                if (rtl_cipher_initARCFOUR( m_pData->m_aCipher, rtl_Cipher_DirectionDecode,

+                                            nTempKey, nKeyLen, nullptr, 0 )

+                    != rtl_Cipher_E_None)

+                {

+                    return false; //TODO: differentiate "failed to decrypt" from "wrong password"

+                }

                 rtl_cipher_decodeARCFOUR( m_pData->m_aCipher,

                                           nPwd, 32,

                                           nPwd, 32 ); // decrypt inplace

diff --git a/vcl/source/gdi/pdfwriter_impl2.cxx b/vcl/source/gdi/pdfwriter_impl2.cxx

index e4f567d6bfd9..e85cf15e4395 100644

--- a/vcl/source/gdi/pdfwriter_impl2.cxx

+++ b/vcl/source/gdi/pdfwriter_impl2.cxx

@@ -1443,29 +1443,39 @@ bool PDFWriterImpl::computeODictionaryValue( const sal_uInt8* i_pPaddedOwnerPass

             //Step 4, the key is in nMD5Sum

             //step 5 already done, data is in i_pPaddedUserPassword

             //step 6

-            rtl_cipher_initARCFOUR( aCipher, rtl_Cipher_DirectionEncode,

-                                     nMD5Sum, i_nKeyLength , nullptr, 0 );

-            // encrypt the user password using the key set above

-            rtl_cipher_encodeARCFOUR( aCipher, i_pPaddedUserPassword, ENCRYPTED_PWD_SIZE, // the data to be encrypted

-                                      &io_rOValue[0], sal_Int32(io_rOValue.size()) ); //encrypted data

-            //Step 7, only if 128 bit

-            if( i_nKeyLength == SECUR_128BIT_KEY )

+            if (rtl_cipher_initARCFOUR( aCipher, rtl_Cipher_DirectionEncode,

+                                        nMD5Sum, i_nKeyLength , nullptr, 0 )

+                == rtl_Cipher_E_None)

             {

-                sal_uInt32 i, y;

-                sal_uInt8 nLocalKey[ SECUR_128BIT_KEY ]; // 16 = 128 bit key

-

-                for( i = 1; i <= 19; i++ ) // do it 19 times, start with 1

+                // encrypt the user password using the key set above

+                rtl_cipher_encodeARCFOUR( aCipher, i_pPaddedUserPassword, ENCRYPTED_PWD_SIZE, // the data to be encrypted

+                                          &io_rOValue[0], sal_Int32(io_rOValue.size()) ); //encrypted data

+                //Step 7, only if 128 bit

+                if( i_nKeyLength == SECUR_128BIT_KEY )

                 {

-                    for( y = 0; y < sizeof( nLocalKey ); y++ )

-                        nLocalKey[y] = (sal_uInt8)( nMD5Sum[y] ^ i );

-

-                    rtl_cipher_initARCFOUR( aCipher, rtl_Cipher_DirectionEncode,

-                                            nLocalKey, SECUR_128BIT_KEY, nullptr, 0 ); //destination data area, on init can be NULL

-                    rtl_cipher_encodeARCFOUR( aCipher, &io_rOValue[0], sal_Int32(io_rOValue.size()), // the data to be encrypted

-                                              &io_rOValue[0], sal_Int32(io_rOValue.size()) ); // encrypted data, can be the same as the input, encrypt "in place"

-                    //step 8, store in class data member

+                    sal_uInt32 i, y;

+                    sal_uInt8 nLocalKey[ SECUR_128BIT_KEY ]; // 16 = 128 bit key

+

+                    for( i = 1; i <= 19; i++ ) // do it 19 times, start with 1

+                    {

+                        for( y = 0; y < sizeof( nLocalKey ); y++ )

+                            nLocalKey[y] = (sal_uInt8)( nMD5Sum[y] ^ i );

+

+                        if (rtl_cipher_initARCFOUR( aCipher, rtl_Cipher_DirectionEncode,

+                                                    nLocalKey, SECUR_128BIT_KEY, nullptr, 0 ) //destination data area, on init can be NULL

+                            != rtl_Cipher_E_None)

+                        {

+                            bSuccess = false;

+                            break;

+                        }

+                        rtl_cipher_encodeARCFOUR( aCipher, &io_rOValue[0], sal_Int32(io_rOValue.size()), // the data to be encrypted

+                                                  &io_rOValue[0], sal_Int32(io_rOValue.size()) ); // encrypted data, can be the same as the input, encrypt "in place"

+                        //step 8, store in class data member

+                    }

                 }

             }

+            else

+                bSuccess = false;

         }

         else

             bSuccess = false;

-- 

2.17.1