Intermiediary commit.

This commit is contained in:
Shyotl
2019-03-16 17:51:00 -05:00
parent 7cf55fa993
commit 2d8fbb89c9
58 changed files with 1857 additions and 1057 deletions

View File

@@ -63,7 +63,6 @@ void LLXfer::init (S32 chunk_size)
mXferSize = 0;
mStatus = e_LL_XFER_UNINITIALIZED;
mNext = NULL;
mWaitingForACK = FALSE;
mCallback = NULL;
@@ -99,7 +98,22 @@ void LLXfer::cleanup ()
S32 LLXfer::startSend (U64 xfer_id, const LLHost &remote_host)
{
LL_WARNS() << "undifferentiated LLXfer::startSend for " << getFileName() << LL_ENDL;
LL_WARNS("Xfer") << "unexpected call to base class LLXfer::startSend for " << getFileName() << LL_ENDL;
return (-1);
}
///////////////////////////////////////////////////////////
void LLXfer::closeFileHandle()
{
LL_WARNS("Xfer") << "unexpected call to base class LLXfer::closeFileHandle for " << getFileName() << LL_ENDL;
}
///////////////////////////////////////////////////////////
S32 LLXfer::reopenFileHandle()
{
LL_WARNS("Xfer") << "unexpected call to base class LLXfer::reopenFileHandle for " << getFileName() << LL_ENDL;
return (-1);
}
@@ -115,7 +129,7 @@ void LLXfer::setXferSize (S32 xfer_size)
S32 LLXfer::startDownload()
{
LL_WARNS() << "undifferentiated LLXfer::startDownload for " << getFileName()
LL_WARNS("Xfer") << "undifferentiated LLXfer::startDownload for " << getFileName()
<< LL_ENDL;
return (-1);
}
@@ -140,7 +154,7 @@ S32 LLXfer::receiveData (char *datap, S32 data_size)
}
else
{
LL_ERRS() << "NULL data passed in receiveData" << LL_ENDL;
LL_ERRS("Xfer") << "NULL data passed in receiveData" << LL_ENDL;
}
}
@@ -163,7 +177,7 @@ S32 LLXfer::flush()
S32 LLXfer::suck(S32 start_position)
{
LL_WARNS() << "Attempted to send a packet outside the buffer bounds in LLXfer::suck()" << LL_ENDL;
LL_WARNS("Xfer") << "Attempted to send a packet outside the buffer bounds in LLXfer::suck()" << LL_ENDL;
return (-1);
}
@@ -196,7 +210,7 @@ void LLXfer::sendPacket(S32 packet_num)
if (fdata_size < 0)
{
LL_WARNS() << "negative data size in xfer send, aborting" << LL_ENDL;
LL_WARNS("Xfer") << "negative data size in xfer send, aborting" << LL_ENDL;
abort(LL_ERR_EOF);
return;
}
@@ -248,7 +262,12 @@ void LLXfer::sendPacket(S32 packet_num)
gMessageSystem->nextBlockFast(_PREHASH_DataPacket);
gMessageSystem->addBinaryDataFast(_PREHASH_Data, &fdata_buf,fdata_size);
gMessageSystem->sendMessage(mRemoteHost);
S32 sent_something = gMessageSystem->sendMessage(mRemoteHost);
if (sent_something == 0)
{
abort(LL_ERR_CIRCUIT_GONE);
return;
}
ACKTimer.reset();
mWaitingForACK = TRUE;
@@ -289,12 +308,12 @@ S32 LLXfer::processEOF()
if (LL_ERR_NOERR == mCallbackResult)
{
LL_INFOS() << "xfer from " << mRemoteHost << " complete: " << getFileName()
LL_INFOS("Xfer") << "xfer from " << mRemoteHost << " complete: " << getFileName()
<< LL_ENDL;
}
else
{
LL_INFOS() << "xfer from " << mRemoteHost << " failed, code "
LL_INFOS("Xfer") << "xfer from " << mRemoteHost << " failed, code "
<< mCallbackResult << ": " << getFileName() << LL_ENDL;
}
@@ -323,15 +342,18 @@ void LLXfer::abort (S32 result_code)
{
mCallbackResult = result_code;
LL_INFOS() << "Aborting xfer from " << mRemoteHost << " named " << getFileName()
LL_INFOS("Xfer") << "Aborting xfer from " << mRemoteHost << " named " << getFileName()
<< " - error: " << result_code << LL_ENDL;
if (result_code != LL_ERR_CIRCUIT_GONE)
{
gMessageSystem->newMessageFast(_PREHASH_AbortXfer);
gMessageSystem->nextBlockFast(_PREHASH_XferID);
gMessageSystem->addU64Fast(_PREHASH_ID, mID);
gMessageSystem->addS32Fast(_PREHASH_Result, result_code);
gMessageSystem->sendMessage(mRemoteHost);
}
mStatus = e_LL_XFER_ABORTED;
}

View File

@@ -54,7 +54,6 @@ class LLXfer
S32 mChunkSize;
public:
LLXfer *mNext;
U64 mID;
S32 mPacketNum;
@@ -90,7 +89,9 @@ class LLXfer
void init(S32 chunk_size);
virtual void cleanup();
virtual S32 startSend (U64 xfer_id, const LLHost &remote_host);
virtual S32 startSend(U64 xfer_id, const LLHost &remote_host);
virtual void closeFileHandle();
virtual S32 reopenFileHandle();
virtual void sendPacket(S32 packet_num);
virtual void sendNextPacket();
virtual void resendLastPacket();

View File

@@ -98,16 +98,16 @@ void LLXfer_File::cleanup ()
mFp = NULL;
}
LLFile::remove(mTempFilename);
LLFile::remove(mTempFilename, ENOENT);
if (mDeleteLocalOnCompletion)
{
LL_DEBUGS() << "Removing file: " << mLocalFilename << LL_ENDL;
LLFile::remove(mLocalFilename);
LL_DEBUGS("Xfer") << "Removing file: " << mLocalFilename << LL_ENDL;
LLFile::remove(mLocalFilename, ENOENT);
}
else
{
LL_DEBUGS() << "Keeping local file: " << mLocalFilename << LL_ENDL;
LL_DEBUGS("Xfer") << "Keeping local file: " << mLocalFilename << LL_ENDL;
}
LLXfer::cleanup();
@@ -139,7 +139,7 @@ S32 LLXfer_File::initializeRequest(U64 xfer_id,
mCallbackDataHandle = user_data;
mCallbackResult = LL_ERR_NOERR;
LL_INFOS() << "Requesting xfer from " << remote_host << " for file: " << mLocalFilename << LL_ENDL;
LL_INFOS("Xfer") << "Requesting xfer from " << remote_host << " for file: " << mLocalFilename << LL_ENDL;
if (mBuffer)
{
@@ -182,7 +182,7 @@ S32 LLXfer_File::startDownload()
}
else
{
LL_WARNS() << "Couldn't create file to be received!" << LL_ENDL;
LL_WARNS("Xfer") << "Couldn't create file to be received!" << LL_ENDL;
retval = -1;
}
@@ -223,7 +223,7 @@ S32 LLXfer_File::startSend (U64 xfer_id, const LLHost &remote_host)
}
else
{
LL_INFOS() << "Warning: " << mLocalFilename << " not found." << LL_ENDL;
LL_INFOS("Xfer") << "Warning: " << mLocalFilename << " not found." << LL_ENDL;
return (LL_ERR_FILE_NOT_FOUND);
}
@@ -232,6 +232,36 @@ S32 LLXfer_File::startSend (U64 xfer_id, const LLHost &remote_host)
return (retval);
}
///////////////////////////////////////////////////////////
void LLXfer_File::closeFileHandle()
{
if (mFp)
{
fclose(mFp);
mFp = NULL;
}
}
///////////////////////////////////////////////////////////
S32 LLXfer_File::reopenFileHandle()
{
S32 retval = LL_ERR_NOERR; // presume success
if (mFp == NULL)
{
mFp = LLFile::fopen(mLocalFilename,"rb"); /* Flawfinder : ignore */
if (mFp == NULL)
{
LL_INFOS("Xfer") << "Warning: " << mLocalFilename << " not found when re-opening file" << LL_ENDL;
retval = LL_ERR_FILE_NOT_FOUND;
}
}
return retval;
}
///////////////////////////////////////////////////////////
S32 LLXfer_File::getMaxBufferSize ()
@@ -279,18 +309,21 @@ S32 LLXfer_File::flush()
{
if (mFp)
{
LL_ERRS() << "Overwriting open file pointer!" << LL_ENDL;
LL_ERRS("Xfer") << "Overwriting open file pointer!" << LL_ENDL;
}
mFp = LLFile::fopen(mTempFilename,"a+b"); /* Flawfinder : ignore */
if (mFp)
{
if (fwrite(mBuffer,1,mBufferLength,mFp) != mBufferLength)
S32 write_size = fwrite(mBuffer,1,mBufferLength,mFp);
if (write_size != mBufferLength)
{
LL_WARNS() << "Short write" << LL_ENDL;
LL_WARNS("Xfer") << "Non-matching write size, requested " << mBufferLength
<< " but wrote " << write_size
<< LL_ENDL;
}
// LL_INFOS() << "******* wrote " << mBufferLength << " bytes of file xfer" << LL_ENDL;
// LL_INFOS("Xfer") << "******* wrote " << mBufferLength << " bytes of file xfer" << LL_ENDL;
fclose(mFp);
mFp = NULL;
@@ -298,7 +331,7 @@ S32 LLXfer_File::flush()
}
else
{
LL_WARNS() << "LLXfer_File::flush() unable to open " << mTempFilename << " for writing!" << LL_ENDL;
LL_WARNS("Xfer") << "LLXfer_File::flush() unable to open " << mTempFilename << " for writing!" << LL_ENDL;
retval = LL_ERR_CANNOT_OPEN_FILE;
}
}
@@ -321,7 +354,7 @@ S32 LLXfer_File::processEOF()
mCallbackResult = flushval;
}
LLFile::remove(mLocalFilename);
LLFile::remove(mLocalFilename, ENOENT);
if (!mCallbackResult)
{
@@ -329,18 +362,18 @@ S32 LLXfer_File::processEOF()
{
#if !LL_WINDOWS
S32 error_number = errno;
LL_INFOS() << "Rename failure (" << error_number << ") - "
LL_INFOS("Xfer") << "Rename failure (" << error_number << ") - "
<< mTempFilename << " to " << mLocalFilename << LL_ENDL;
if(EXDEV == error_number)
{
if(copy_file(mTempFilename, mLocalFilename) == 0)
{
LL_INFOS() << "Rename across mounts; copying+unlinking the file instead." << LL_ENDL;
LL_INFOS("Xfer") << "Rename across mounts; copying+unlinking the file instead." << LL_ENDL;
unlink(mTempFilename.c_str());
}
else
{
LL_WARNS() << "Copy failure - " << mTempFilename << " to "
LL_WARNS("Xfer") << "Copy failure - " << mTempFilename << " to "
<< mLocalFilename << LL_ENDL;
}
}
@@ -354,11 +387,11 @@ S32 LLXfer_File::processEOF()
//LL_WARNS() << "File " << mLocalFilename << " does "
// << (!fp ? "not" : "" ) << " exit." << LL_ENDL;
//if(fp) fclose(fp);
LL_WARNS() << "Rename fatally failed, can only handle EXDEV ("
LL_WARNS("Xfer") << "Rename fatally failed, can only handle EXDEV ("
<< EXDEV << ")" << LL_ENDL;
}
#else
LL_WARNS() << "Rename failure - " << mTempFilename << " to "
LL_WARNS("Xfer") << "Rename failure - " << mTempFilename << " to "
<< mLocalFilename << LL_ENDL;
#endif
}

View File

@@ -61,8 +61,10 @@ class LLXfer_File : public LLXfer
virtual S32 startDownload();
virtual S32 processEOF();
virtual S32 startSend (U64 xfer_id, const LLHost &remote_host);
virtual S32 startSend(U64 xfer_id, const LLHost &remote_host);
virtual void closeFileHandle();
virtual S32 reopenFileHandle();
virtual S32 suck(S32 start_position);
virtual S32 flush();

View File

@@ -80,28 +80,6 @@ void LLXfer_Mem::setXferSize (S32 xfer_size)
///////////////////////////////////////////////////////////
U64 LLXfer_Mem::registerXfer(U64 xfer_id, const void *datap, const S32 length)
{
mID = xfer_id;
if (datap)
{
setXferSize(length);
if (mBuffer)
{
memcpy(mBuffer,datap,length); /* Flawfinder : ignore */
mBufferLength = length;
}
else
{
xfer_id = 0;
}
}
mStatus = e_LL_XFER_REGISTERED;
return (xfer_id);
}
S32 LLXfer_Mem::startSend (U64 xfer_id, const LLHost &remote_host)
{
S32 retval = LL_ERR_NOERR; // presume success

View File

@@ -53,7 +53,6 @@ class LLXfer_Mem : public LLXfer
virtual void cleanup();
virtual S32 startSend (U64 xfer_id, const LLHost &remote_host);
virtual U64 registerXfer(U64 xfer_id, const void *datap, const S32 length);
virtual void setXferSize (S32 data_size);
virtual S32 initializeRequest(U64 xfer_id,

View File

@@ -79,8 +79,20 @@ void LLXfer_VFile::init (LLVFS *vfs, const LLUUID &local_id, LLAssetType::EType
void LLXfer_VFile::cleanup ()
{
LLVFile file(mVFS, mTempID, mType, LLVFile::WRITE);
file.remove();
if (mTempID.notNull() &&
mDeleteTempFile)
{
if (mVFS->getExists(mTempID, mType))
{
LLVFile file(mVFS, mTempID, mType, LLVFile::WRITE);
file.remove();
}
else
{
LL_WARNS("Xfer") << "LLXfer_VFile::cleanup() can't open to delete VFS file " << mTempID << "." << LLAssetType::lookup(mType)
<< ", mRemoteID is " << mRemoteID << LL_ENDL;
}
}
delete mVFile;
mVFile = NULL;
@@ -118,7 +130,7 @@ S32 LLXfer_VFile::initializeRequest(U64 xfer_id,
mName = llformat("VFile %s:%s", id_string.c_str(), LLAssetType::lookup(mType));
LL_INFOS() << "Requesting " << mName << LL_ENDL;
LL_INFOS("Xfer") << "Requesting " << mName << LL_ENDL;
if (mBuffer)
{
@@ -131,6 +143,7 @@ S32 LLXfer_VFile::initializeRequest(U64 xfer_id,
mBufferLength = 0;
mPacketNum = 0;
mTempID.generate();
mDeleteTempFile = TRUE;
mStatus = e_LL_XFER_PENDING;
return retval;
}
@@ -140,7 +153,8 @@ S32 LLXfer_VFile::initializeRequest(U64 xfer_id,
S32 LLXfer_VFile::startDownload()
{
S32 retval = 0; // presume success
LLVFile file(mVFS, mTempID, mType, LLVFile::APPEND);
// Don't need to create the file here, it will happen when data arrives
gMessageSystem->newMessageFast(_PREHASH_RequestXfer);
gMessageSystem->nextBlockFast(_PREHASH_XferID);
@@ -184,6 +198,8 @@ S32 LLXfer_VFile::startSend (U64 xfer_id, const LLHost &remote_host)
if (mVFile->getSize() <= 0)
{
LL_WARNS("Xfer") << "LLXfer_VFile::startSend() VFS file " << mLocalID << "." << LLAssetType::lookup(mType)
<< " has unexpected file size of " << mVFile->getSize() << LL_ENDL;
delete mVFile;
mVFile = NULL;
@@ -198,6 +214,7 @@ S32 LLXfer_VFile::startSend (U64 xfer_id, const LLHost &remote_host)
}
else
{
LL_WARNS("Xfer") << "LLXfer_VFile::startSend() can't read VFS file " << mLocalID << "." << LLAssetType::lookup(mType) << LL_ENDL;
retval = LL_ERR_FILE_NOT_FOUND;
}
@@ -205,6 +222,41 @@ S32 LLXfer_VFile::startSend (U64 xfer_id, const LLHost &remote_host)
}
///////////////////////////////////////////////////////////
void LLXfer_VFile::closeFileHandle()
{
if (mVFile)
{
delete mVFile;
mVFile = NULL;
}
}
///////////////////////////////////////////////////////////
S32 LLXfer_VFile::reopenFileHandle()
{
S32 retval = LL_ERR_NOERR; // presume success
if (mVFile == NULL)
{
if (mVFS->getExists(mLocalID, mType))
{
mVFile = new LLVFile(mVFS, mLocalID, mType, LLVFile::READ);
}
else
{
LL_WARNS("Xfer") << "LLXfer_VFile::reopenFileHandle() can't read VFS file " << mLocalID << "." << LLAssetType::lookup(mType) << LL_ENDL;
retval = LL_ERR_FILE_NOT_FOUND;
}
}
return retval;
}
///////////////////////////////////////////////////////////
void LLXfer_VFile::setXferSize (S32 xfer_size)
{
LLXfer::setXferSize(xfer_size);
@@ -236,8 +288,8 @@ S32 LLXfer_VFile::suck(S32 start_position)
// grab a buffer from the right place in the file
if (! mVFile->seek(start_position, 0))
{
LL_WARNS() << "VFile Xfer Can't seek to position " << start_position << ", file length " << mVFile->getSize() << LL_ENDL;
LL_WARNS() << "While sending file " << mLocalID << LL_ENDL;
LL_WARNS("Xfer") << "VFile Xfer Can't seek to position " << start_position << ", file length " << mVFile->getSize() << LL_ENDL;
LL_WARNS("Xfer") << "While sending file " << mLocalID << LL_ENDL;
return -1;
}
@@ -288,12 +340,31 @@ S32 LLXfer_VFile::processEOF()
if (!mCallbackResult)
{
LLVFile file(mVFS, mTempID, mType, LLVFile::WRITE);
if (! file.rename(mLocalID, mType))
if (mVFS->getExists(mTempID, mType))
{
LL_INFOS() << "copy from temp file failed: unable to rename to " << mLocalID << LL_ENDL;
LLVFile file(mVFS, mTempID, mType, LLVFile::WRITE);
if (!file.rename(mLocalID, mType))
{
LL_WARNS("Xfer") << "VFS rename of temp file failed: unable to rename " << mTempID << " to " << mLocalID << LL_ENDL;
}
else
{
#ifdef VFS_SPAM
// Debugging spam
LL_INFOS("Xfer") << "VFS rename of temp file done: renamed " << mTempID << " to " << mLocalID
<< " LLVFile size is " << file.getSize()
<< LL_ENDL;
#endif
// Rename worked: the original file is gone. Clear mDeleteTempFile
// so we don't attempt to delete the file in cleanup()
mDeleteTempFile = FALSE;
}
}
else
{
LL_WARNS("Xfer") << "LLXfer_VFile::processEOF() can't open for renaming VFS file " << mTempID << "." << LLAssetType::lookup(mType) << LL_ENDL;
}
}
if (mVFile)

View File

@@ -47,6 +47,8 @@ class LLXfer_VFile : public LLXfer
std::string mName;
BOOL mDeleteTempFile;
public:
LLXfer_VFile ();
LLXfer_VFile (LLVFS *vfs, const LLUUID &local_id, LLAssetType::EType type);
@@ -66,8 +68,10 @@ class LLXfer_VFile : public LLXfer
virtual S32 startDownload();
virtual S32 processEOF();
virtual S32 startSend (U64 xfer_id, const LLHost &remote_host);
virtual S32 startSend(U64 xfer_id, const LLHost &remote_host);
virtual void closeFileHandle();
virtual S32 reopenFileHandle();
virtual S32 suck(S32 start_position);
virtual S32 flush();

File diff suppressed because it is too large Load Diff

View File

@@ -77,6 +77,7 @@ class LLXferManager
protected:
S32 mMaxOutgoingXfersPerCircuit;
S32 mHardLimitOutgoingXfersPerCircuit; // At this limit, kill off the connection
S32 mMaxIncomingXfers;
BOOL mUseAckThrottling; // Use ack throttling to cap file xfer bandwidth
@@ -92,19 +93,22 @@ class LLXferManager
HIGH_PRIORITY = TRUE,
};
LLXfer *mSendList;
LLXfer *mReceiveList;
// Linked FIFO list, add to the front and pull from back
typedef std::deque<LLXfer *> xfer_list_t;
xfer_list_t mSendList;
xfer_list_t mReceiveList;
typedef std::list<LLHostStatus*> status_list_t;
status_list_t mOutgoingHosts;
private:
protected:
// implementation methods
virtual void startPendingDownloads();
virtual void addToList(LLXfer* xferp, LLXfer*& head, BOOL is_priority);
virtual void addToList(LLXfer* xferp, xfer_list_t & xfer_list, BOOL is_priority);
std::multiset<std::string> mExpectedTransfers; // files that are authorized to transfer out
std::multiset<std::string> mExpectedRequests; // files that are authorized to be downloaded on top of
std::multiset<std::string> mExpectedVFileTransfers; // files that are authorized to transfer out
std::multiset<std::string> mExpectedVFileRequests; // files that are authorized to be downloaded on top of
public:
LLXferManager(LLVFS *vfs);
@@ -117,14 +121,17 @@ class LLXferManager
void setAckThrottleBPS(const F32 bps);
// list management routines
virtual LLXfer *findXfer(U64 id, LLXfer *list_head);
virtual void removeXfer (LLXfer *delp, LLXfer **list_head);
virtual U32 numActiveListEntries(LLXfer *list_head);
virtual LLXfer *findXferByID(U64 id, xfer_list_t & xfer_list);
virtual void removeXfer (LLXfer *delp, xfer_list_t & xfer_list);
LLHostStatus * findHostStatus(const LLHost &host);
virtual S32 numActiveXfers(const LLHost &host);
virtual S32 numPendingXfers(const LLHost &host);
virtual void changeNumActiveXfers(const LLHost &host, S32 delta);
virtual void setMaxOutgoingXfersPerCircuit (S32 max_num);
virtual void setHardLimitOutgoingXfersPerCircuit(S32 max_num);
virtual void setMaxIncomingXfers(S32 max_num);
virtual void updateHostStatus();
virtual void printHostStatus();
@@ -136,11 +143,9 @@ class LLXferManager
virtual S32 decodePacketNum(S32 packet_num);
virtual BOOL isLastPacket(S32 packet_num);
virtual U64 registerXfer(const void *datap, const S32 length);
// file requesting routines
// .. to file
virtual void requestFile(const std::string& local_filename,
virtual U64 requestFile(const std::string& local_filename,
const std::string& remote_filename,
ELLPath remote_path,
const LLHost& remote_host,
@@ -148,7 +153,7 @@ class LLXferManager
void (*callback)(void**,S32,LLExtStat), void** user_data,
BOOL is_priority = FALSE,
BOOL use_big_packets = FALSE);
/*
// .. to memory
virtual void requestFile(const std::string& remote_filename,
ELLPath remote_path,
@@ -157,7 +162,7 @@ class LLXferManager
void (*callback)(void*, S32, void**, S32, LLExtStat),
void** user_data,
BOOL is_priority = FALSE);
*/
// vfile requesting
// .. to vfile
virtual void requestVFile(const LLUUID &local_id, const LLUUID& remote_id,
@@ -180,18 +185,15 @@ class LLXferManager
virtual void expectFileForRequest(const std::string& filename);
virtual bool validateFileForRequest(const std::string& filename);
/*
// xfer request (may be memory or file)
// .. to file
virtual void requestXfer(const char *local_filename, U64 xfer_id,
BOOL delete_remote_on_completion,
const LLHost &remote_host, void (*callback)(void **,S32),void **user_data);
// .. to memory
virtual void requestXfer(U64 xfer_id,
const LLHost &remote_host,
BOOL delete_remote_on_completion,
void (*callback)(void *, S32, void **, S32),void **user_data);
*/
/**
Same idea but for VFiles, kept separate to avoid namespace overlap
*/
/* Present in fireengine, not used by viewer
virtual void expectVFileForTransfer(const std::string& filename);
virtual bool validateVFileForTransfer(const std::string& filename);
virtual void expectVFileForRequest(const std::string& filename);
virtual bool validateVFileForRequest(const std::string& filename);
*/
virtual void processReceiveData (LLMessageSystem *mesgsys, void **user_data);
virtual void sendConfirmPacket (LLMessageSystem *mesgsys, U64 id, S32 packetnum, const LLHost &remote_host);
@@ -202,7 +204,10 @@ class LLXferManager
virtual void retransmitUnackedPackets ();
// error handling
void abortRequestById(U64 xfer_id, S32 result_code);
virtual void processAbort (LLMessageSystem *mesgsys, void **user_data);
virtual bool isHostFlooded(const LLHost & host);
};
extern LLXferManager* gXferManager;