andreacavalli/netty5
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Fix DeleteOnExitHook cause memory leak (#10560)

Browse Source 
Motivation:

If DeleteOnExitHook is in the open state and the program runs for a long time, the DeleteOnExitHook file keeps increasing.
This results in a memory leak


Modification:

I re-customized a DeleteOnExitHook hook. If DeleteOnExitHook is turned on, this hook will be added when creating a temporary file. After the request ends and the corresponding resources are released, the current file will be removed from this hook, so that it will not increase all the time.

Result:

Fixes https://github.com/netty/netty/issues/10351
This commit is contained in:
Kevin Wu 2020-09-15 22:38:42 +08:00 committed by GitHub
parent e8aaf7f586
commit 95ce1b95ea
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 179 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
codec-http/src/main/java/io/netty/handler/codec/http/multipart/AbstractDiskHttpData.java
View File

@ -30,7 +30,8 @@ import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.nio.charset.Charset;
import static io.netty.buffer.Unpooled.*;
import static io.netty.buffer.Unpooled.EMPTY_BUFFER;
import static io.netty.buffer.Unpooled.wrappedBuffer;
/**
* Abstract Disk HttpData implementation
@ -93,7 +94,8 @@ public abstract class AbstractDiskHttpData extends AbstractHttpData {
getBaseDirectory()));
}
if (deleteOnExit()) {
tmpFile.deleteOnExit();
// See https://github.com/netty/netty/issues/10351
DeleteFileOnExitHook.add(tmpFile.getPath());
}
return tmpFile;
}
@ -270,12 +272,21 @@ public abstract class AbstractDiskHttpData extends AbstractHttpData {
}
fileChannel = null;
}
if (! isRenamed) {
if (!isRenamed) {
String filePath = null;
if (file != null && file.exists()) {
filePath = file.getPath();
if (!file.delete()) {
filePath = null;
logger.warn("Failed to delete: {}", file);
}
}
// If you turn on deleteOnExit make sure it is executed.
if (deleteOnExit() && filePath != null) {
DeleteFileOnExitHook.remove(filePath);
}
file = null;
}
}
@ -378,7 +389,7 @@ public abstract class AbstractDiskHttpData extends AbstractHttpData {
if (chunkSize < size - position) {
chunkSize = size - position;
}
position += in.transferTo(position, chunkSize , out);
position += in.transferTo(position, chunkSize, out);
}
} catch (IOException e) {
exception = e;
@ -430,6 +441,7 @@ public abstract class AbstractDiskHttpData extends AbstractHttpData {
/**
* Utility function
*
* @return the array of bytes
*/
private static byte[] readFrom(File src) throws IOException {

82
codec-http/src/main/java/io/netty/handler/codec/http/multipart/DeleteFileOnExitHook.java Normal file
View File

@ -0,0 +1,82 @@
/*
* Copyright 2020 The Netty Project
*
* The Netty Project licenses this file to you under the Apache License,
* version 2.0 (the "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at:
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package io.netty.handler.codec.http.multipart;
import java.io.File;
import java.util.Collections;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
/**
* DeleteFileOnExitHook.
*/
final class DeleteFileOnExitHook {
private static final Set<String> FILES = Collections.newSetFromMap(new ConcurrentHashMap<String, Boolean>());
private DeleteFileOnExitHook() {
}
static {
// DeleteOnExitHook must be the last shutdown hook to be invoked.
// Application shutdown hooks may add the first file to the
// delete on exit list and cause the DeleteOnExitHook to be
// registered during shutdown in progress.
Runtime.getRuntime().addShutdownHook(new Thread() {
@Override
public void run() {
runHook();
}
});
}
/**
* Remove from the pool to reduce space footprint.
*
* @param file tmp file path
*/
public static void remove(String file) {
FILES.remove(file);
}
/**
* Add to the hook and clean up when the program exits.
*
* @param file tmp file path
*/
public static void add(String file) {
FILES.add(file);
}
/**
* Check in the hook files.
*
* @param file target file
* @return true or false
*/
public static boolean checkFileExist(String file) {
return FILES.contains(file);
}
/**
* Clean up all the files.
*/
static void runHook() {
for (String filename : FILES) {
new File(filename).delete();
}
}
}

81
codec-http/src/test/java/io/netty/handler/codec/http/multipart/DeleteFileOnExitHookTest.java Normal file
View File

@ -0,0 +1,81 @@
/*
* Copyright 2020 The Netty Project
*
* The Netty Project licenses this file to you under the Apache License,
* version 2.0 (the "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at:
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package io.netty.handler.codec.http.multipart;
import io.netty.buffer.Unpooled;
import io.netty.handler.codec.http.DefaultHttpRequest;
import io.netty.handler.codec.http.HttpRequest;
import org.junit.Before;
import org.junit.Test;
import java.io.File;
import java.io.FilenameFilter;
import java.io.IOException;
import static io.netty.handler.codec.http.HttpMethod.POST;
import static io.netty.handler.codec.http.HttpVersion.HTTP_1_1;
import static org.junit.Assert.*;
/**
* Test DeleteFileOnExitHook
*/
public class DeleteFileOnExitHookTest {
private static final HttpRequest REQUEST = new DefaultHttpRequest(HTTP_1_1, POST, "/form");
private static final String HOOK_TEST_TMP = "target/DeleteFileOnExitHookTest/tmp";
private FileUpload fu;
@Before
public void setUp() throws IOException {
DefaultHttpDataFactory defaultHttpDataFactory = new DefaultHttpDataFactory(true);
defaultHttpDataFactory.setBaseDir(HOOK_TEST_TMP);
defaultHttpDataFactory.setDeleteOnExit(true);
File baseDir = new File(HOOK_TEST_TMP);
baseDir.mkdirs(); // we don't need to clean it since it is in volatile files anyway
fu = defaultHttpDataFactory.createFileUpload(
REQUEST, "attribute1", "tmp_f.txt", "text/plain", null, null, 0);
fu.setContent(Unpooled.wrappedBuffer(new byte[]{1, 2, 3, 4}));
assertTrue(fu.getFile().exists());
}
@Test
public void testSimulateTriggerDeleteFileOnExitHook() {
// simulate app exit
DeleteFileOnExitHook.runHook();
File[] files = new File(HOOK_TEST_TMP).listFiles(new FilenameFilter() {
@Override
public boolean accept(File dir, String name) {
return name.startsWith(DiskFileUpload.prefix);
}
});
assertEquals(0, files.length);
}
@Test
public void testAfterHttpDataReleaseCheckFileExist() throws IOException {
String filePath = fu.getFile().getPath();
assertTrue(DeleteFileOnExitHook.checkFileExist(filePath));
fu.release();
assertFalse(DeleteFileOnExitHook.checkFileExist(filePath));
}
}