From 9207290587ea8c8703486d60877731228eed80ce Mon Sep 17 00:00:00 2001
From: Stephan Bergmann <sbergman@redhat.com>
Date: Fri, 24 Aug 2018 10:27:01 +0200
Subject: [PATCH 4/5] iRelated rhbz#1618703: Properly handle failure
en-/decoding PDF file
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
...when e.g. FIPS mode makes the various calls to rtl_cipher_initARCFOUR fail.
Reviewed-on: https://gerrit.libreoffice.org/59543
Tested-by: Jenkins
Reviewed-by: Stephan Bergmann <sbergman@redhat.com>
(cherry picked from commit 185a14525f114e58b48236284ed8e8644bc40e48)
Reviewed-on: https://gerrit.libreoffice.org/59573
Reviewed-by: Caolán McNamara <caolanm@redhat.com>
Tested-by: Caolán McNamara <caolanm@redhat.com>
(cherry picked from commit 68ffc5c83ca73c58439b7c9935283541f007db44)
Conflicts:
filter/source/pdf/impdialog.cxx
sdext/source/pdfimport/pdfparse/pdfentries.cxx
vcl/source/gdi/pdfwriter_impl2.cxx
Change-Id: Id1b2222249c151470e233ab814b21228f3a8b561
---
filter/source/pdf/impdialog.cxx | 7 +++
.../source/pdfimport/pdfparse/pdfentries.cxx | 42 +++++++++++-----
vcl/source/gdi/pdfwriter_impl2.cxx | 48 +++++++++++--------
3 files changed, 67 insertions(+), 30 deletions(-)
diff --git a/filter/source/pdf/impdialog.cxx b/filter/source/pdf/impdialog.cxx
index 82aaf012505a..736ac0c15f26 100644
--- a/filter/source/pdf/impdialog.cxx
+++ b/filter/source/pdf/impdialog.cxx
@@ -23,6 +23,7 @@
#include <strings.hrc>
#include <bitmaps.hlst>
#include <officecfg/Office/Common.hxx>
+#include <vcl/errinf.hxx>
#include <vcl/layout.hxx>
#include <vcl/settings.hxx>
#include <vcl/svapp.hxx>
@@ -1351,6 +1352,12 @@ IMPL_LINK_NOARG(ImpPDFTabSecurityPage, ClickmaPbSetPwdHdl, Button*, void)
mbHaveOwnerPassword = !aOwnerPW.isEmpty();
mxPreparedPasswords = vcl::PDFWriter::InitEncryption( aOwnerPW, aUserPW, true );
+ if (!mxPreparedPasswords.is()) {
+ OUString msg;
+ ErrorHandler::GetErrorString(ERRCODE_IO_NOTSUPPORTED, msg); //TOOD: handle failure
+ ScopedVclPtrInstance<MessageDialog>(this, msg, VclMessageType::Error)->Execute();
+ return;
+ }
if( mbHaveOwnerPassword )
{
diff --git a/sdext/source/pdfimport/pdfparse/pdfentries.cxx b/sdext/source/pdfimport/pdfparse/pdfentries.cxx
index 16563868f25c..074fb669c8da 100644
--- a/sdext/source/pdfimport/pdfparse/pdfentries.cxx
+++ b/sdext/source/pdfimport/pdfparse/pdfentries.cxx
@@ -1159,9 +1159,13 @@ static bool check_user_password( const OString& rPwd, PDFFileImplData* pData )
{
// see PDF reference 1.4 Algorithm 3.4
// encrypt pad string
- rtl_cipher_initARCFOUR( pData->m_aCipher, rtl_Cipher_DirectionEncode,
- aKey, nKeyLen,
- nullptr, 0 );
+ if (rtl_cipher_initARCFOUR( pData->m_aCipher, rtl_Cipher_DirectionEncode,
+ aKey, nKeyLen,
+ nullptr, 0 )
+ != rtl_Cipher_E_None)
+ {
+ return false; //TODO: differentiate "failed to decrypt" from "wrong password"
+ }
rtl_cipher_encodeARCFOUR( pData->m_aCipher, nPadString, sizeof( nPadString ),
nEncryptedEntry, sizeof( nEncryptedEntry ) );
bValid = (memcmp( nEncryptedEntry, pData->m_aUEntry, 32 ) == 0);
@@ -1172,8 +1176,12 @@ static bool check_user_password( const OString& rPwd, PDFFileImplData* pData )
rtl_digest_updateMD5( pData->m_aDigest, nPadString, sizeof( nPadString ) );
rtl_digest_updateMD5( pData->m_aDigest, pData->m_aDocID.getStr(), pData->m_aDocID.getLength() );
rtl_digest_getMD5( pData->m_aDigest, nEncryptedEntry, sizeof(nEncryptedEntry) );
- rtl_cipher_initARCFOUR( pData->m_aCipher, rtl_Cipher_DirectionEncode,
- aKey, sizeof(aKey), nullptr, 0 );
+ if (rtl_cipher_initARCFOUR( pData->m_aCipher, rtl_Cipher_DirectionEncode,
+ aKey, sizeof(aKey), nullptr, 0 )
+ != rtl_Cipher_E_None)
+ {
+ return false; //TODO: differentiate "failed to decrypt" from "wrong password"
+ }
rtl_cipher_encodeARCFOUR( pData->m_aCipher,
nEncryptedEntry, 16,
nEncryptedEntry, 16 ); // encrypt in place
@@ -1183,8 +1191,12 @@ static bool check_user_password( const OString& rPwd, PDFFileImplData* pData )
for( sal_uInt32 j = 0; j < sizeof(aTempKey); j++ )
aTempKey[j] = static_cast<sal_uInt8>( aKey[j] ^ i );
- rtl_cipher_initARCFOUR( pData->m_aCipher, rtl_Cipher_DirectionEncode,
- aTempKey, sizeof(aTempKey), nullptr, 0 );
+ if (rtl_cipher_initARCFOUR( pData->m_aCipher, rtl_Cipher_DirectionEncode,
+ aTempKey, sizeof(aTempKey), nullptr, 0 )
+ != rtl_Cipher_E_None)
+ {
+ return false; //TODO: differentiate "failed to decrypt" from "wrong password"
+ }
rtl_cipher_encodeARCFOUR( pData->m_aCipher,
nEncryptedEntry, 16,
nEncryptedEntry, 16 ); // encrypt in place
@@ -1230,8 +1242,12 @@ bool PDFFile::setupDecryptionData( const OString& rPwd ) const
sal_uInt32 nKeyLen = password_to_key( rPwd, aKey, m_pData.get(), true );
if( m_pData->m_nStandardRevision == 2 )
{
- rtl_cipher_initARCFOUR( m_pData->m_aCipher, rtl_Cipher_DirectionDecode,
- aKey, nKeyLen, nullptr, 0 );
+ if (rtl_cipher_initARCFOUR( m_pData->m_aCipher, rtl_Cipher_DirectionDecode,
+ aKey, nKeyLen, nullptr, 0 )
+ != rtl_Cipher_E_None)
+ {
+ return false; //TODO: differentiate "failed to decrypt" from "wrong password"
+ }
rtl_cipher_decodeARCFOUR( m_pData->m_aCipher,
m_pData->m_aOEntry, 32,
nPwd, 32 );
@@ -1244,8 +1260,12 @@ bool PDFFile::setupDecryptionData( const OString& rPwd ) const
sal_uInt8 nTempKey[ENCRYPTION_KEY_LEN];
for( unsigned int j = 0; j < sizeof(nTempKey); j++ )
nTempKey[j] = sal_uInt8(aKey[j] ^ i);
- rtl_cipher_initARCFOUR( m_pData->m_aCipher, rtl_Cipher_DirectionDecode,
- nTempKey, nKeyLen, nullptr, 0 );
+ if (rtl_cipher_initARCFOUR( m_pData->m_aCipher, rtl_Cipher_DirectionDecode,
+ nTempKey, nKeyLen, nullptr, 0 )
+ != rtl_Cipher_E_None)
+ {
+ return false; //TODO: differentiate "failed to decrypt" from "wrong password"
+ }
rtl_cipher_decodeARCFOUR( m_pData->m_aCipher,
nPwd, 32,
nPwd, 32 ); // decrypt inplace
diff --git a/vcl/source/gdi/pdfwriter_impl2.cxx b/vcl/source/gdi/pdfwriter_impl2.cxx
index e4f567d6bfd9..e85cf15e4395 100644
--- a/vcl/source/gdi/pdfwriter_impl2.cxx
+++ b/vcl/source/gdi/pdfwriter_impl2.cxx
@@ -1443,29 +1443,39 @@ bool PDFWriterImpl::computeODictionaryValue( const sal_uInt8* i_pPaddedOwnerPass
//Step 4, the key is in nMD5Sum
//step 5 already done, data is in i_pPaddedUserPassword
//step 6
- rtl_cipher_initARCFOUR( aCipher, rtl_Cipher_DirectionEncode,
- nMD5Sum, i_nKeyLength , nullptr, 0 );
- // encrypt the user password using the key set above
- rtl_cipher_encodeARCFOUR( aCipher, i_pPaddedUserPassword, ENCRYPTED_PWD_SIZE, // the data to be encrypted
- &io_rOValue[0], sal_Int32(io_rOValue.size()) ); //encrypted data
- //Step 7, only if 128 bit
- if( i_nKeyLength == SECUR_128BIT_KEY )
+ if (rtl_cipher_initARCFOUR( aCipher, rtl_Cipher_DirectionEncode,
+ nMD5Sum, i_nKeyLength , nullptr, 0 )
+ == rtl_Cipher_E_None)
{
- sal_uInt32 i, y;
- sal_uInt8 nLocalKey[ SECUR_128BIT_KEY ]; // 16 = 128 bit key
-
- for( i = 1; i <= 19; i++ ) // do it 19 times, start with 1
+ // encrypt the user password using the key set above
+ rtl_cipher_encodeARCFOUR( aCipher, i_pPaddedUserPassword, ENCRYPTED_PWD_SIZE, // the data to be encrypted
+ &io_rOValue[0], sal_Int32(io_rOValue.size()) ); //encrypted data
+ //Step 7, only if 128 bit
+ if( i_nKeyLength == SECUR_128BIT_KEY )
{
- for( y = 0; y < sizeof( nLocalKey ); y++ )
- nLocalKey[y] = (sal_uInt8)( nMD5Sum[y] ^ i );
-
- rtl_cipher_initARCFOUR( aCipher, rtl_Cipher_DirectionEncode,
- nLocalKey, SECUR_128BIT_KEY, nullptr, 0 ); //destination data area, on init can be NULL
- rtl_cipher_encodeARCFOUR( aCipher, &io_rOValue[0], sal_Int32(io_rOValue.size()), // the data to be encrypted
- &io_rOValue[0], sal_Int32(io_rOValue.size()) ); // encrypted data, can be the same as the input, encrypt "in place"
- //step 8, store in class data member
+ sal_uInt32 i, y;
+ sal_uInt8 nLocalKey[ SECUR_128BIT_KEY ]; // 16 = 128 bit key
+
+ for( i = 1; i <= 19; i++ ) // do it 19 times, start with 1
+ {
+ for( y = 0; y < sizeof( nLocalKey ); y++ )
+ nLocalKey[y] = (sal_uInt8)( nMD5Sum[y] ^ i );
+
+ if (rtl_cipher_initARCFOUR( aCipher, rtl_Cipher_DirectionEncode,
+ nLocalKey, SECUR_128BIT_KEY, nullptr, 0 ) //destination data area, on init can be NULL
+ != rtl_Cipher_E_None)
+ {
+ bSuccess = false;
+ break;
+ }
+ rtl_cipher_encodeARCFOUR( aCipher, &io_rOValue[0], sal_Int32(io_rOValue.size()), // the data to be encrypted
+ &io_rOValue[0], sal_Int32(io_rOValue.size()) ); // encrypted data, can be the same as the input, encrypt "in place"
+ //step 8, store in class data member
+ }
}
}
+ else
+ bSuccess = false;
}
else
bSuccess = false;
--
2.17.1